From 8c1071ab633e29d08c5230fe9af934c0a79ff1f6 Mon Sep 17 00:00:00 2001 From: =?utf8?q?V=C3=A1s=C3=A1ry=20D=C3=A1niel?= Date: Fri, 29 May 2020 14:05:34 +0000 Subject: [PATCH] git-tfs-id: [http://tfs.userrendszerhaz.hu:8080/tfs/DefaultCollection]$/MediaCube;C31857 --- ...perties => mediacube-dev-hirtv.properties} | 22 +- .../mediacube-dev-user.properties | 16 +- .../run-mediacube-server-hirtv.launch | 2 +- .../config/config-worker.xml | 1 + .../jobtemplates/common-copy.xml | 2 +- .../fork-validate-and-archive.xml | 25 +++ .../jobtemplates/nexio-checker.xml | 11 +- .../jobtemplates/validate-and-archive.xml | 89 ++++++++ .../mediacube/steps/DownloadableArchive.java | 7 - .../steps/DownloadableListBuilderStep.java | 81 ------- .../server/steps/DownloadableMedia.java | 7 + .../jobengine/server/steps/FileCopyStep.java | 3 +- .../server/steps/FrameNumberReaderStep.java | 36 +++ .../server}/steps/NEXIOCheckerStep.java | 43 ++-- .../jobengine/server/steps/TSMWriterStep.java | 205 ++++++++++++++++++ .../steps/ValidateAndArchiveForkStep.java | 104 +++++++++ .../user/jobengine/server/IT/AMCTests.java | 52 ++++- 17 files changed, 564 insertions(+), 142 deletions(-) rename server/-configuration/{mediacube-dev-bsh.properties => mediacube-dev-hirtv.properties} (57%) create mode 100644 server/user.jobengine.executors/jobtemplates/fork-validate-and-archive.xml create mode 100644 server/user.jobengine.executors/jobtemplates/validate-and-archive.xml delete mode 100644 server/user.jobengine.executors/src/hu/user/mediacube/steps/DownloadableArchive.java delete mode 100644 server/user.jobengine.executors/src/hu/user/mediacube/steps/DownloadableListBuilderStep.java create mode 100644 server/user.jobengine.executors/src/user/jobengine/server/steps/DownloadableMedia.java create mode 100644 server/user.jobengine.executors/src/user/jobengine/server/steps/FrameNumberReaderStep.java rename server/user.jobengine.executors/src/{hu/user/mediacube => user/jobengine/server}/steps/NEXIOCheckerStep.java (64%) create mode 100644 server/user.jobengine.executors/src/user/jobengine/server/steps/TSMWriterStep.java create mode 100644 server/user.jobengine.executors/src/user/jobengine/server/steps/ValidateAndArchiveForkStep.java diff --git a/server/-configuration/mediacube-dev-bsh.properties b/server/-configuration/mediacube-dev-hirtv.properties similarity index 57% rename from server/-configuration/mediacube-dev-bsh.properties rename to server/-configuration/mediacube-dev-hirtv.properties index aa2884c0..82b95c41 100644 --- a/server/-configuration/mediacube-dev-bsh.properties +++ b/server/-configuration/mediacube-dev-hirtv.properties @@ -1,9 +1,13 @@ -#A user.dir mindig a test projekt mappaja, a relativ hivatkozasok ehhez kepest ertendok. -#JobEngine -jobengine.loglevel=INFO -jobengine.jobsteps.root=../user.jobengine.osgi.server/resources/executors -jobengine.jobtemplates.root=../user.jobengine.osgi.server/resources/templates -jobengine.jobsteps.alternate.root=../user.jobengine.executors/bin/user/jobengine/server/steps +jobengine.jobsteps.root=../user.jobengine.executors/bin +jobengine.jobsteps.config=../user.jobengine.executors/config/config-worker.xml +jobengine.jobtemplates.root=../user.jobengine.executors/jobtemplates +jobengine.jobsteps.groovy.root=../user.jobengine.executors/src/user/jobengine/server/steps + +jetty.home=../-configuration/jetty +jetty.etc.config.urls=etc/user-jetty.xml,etc/user-jetty-ssl.xml,etc/user-jetty-ssl-context.xml,etc/user-jetty-http.xml,etc/user-jetty-https.xml + +log4j.configurationFile=../-configuration/log4j2-test.xml + jobengine.db.url=jdbc:db2://10.10.1.27:50000/mc:retrieveMessagesFromServerOnGetMessage=true; jobengine.db.user=db2admin jobengine.db.password=password @@ -16,8 +20,4 @@ jobengine.nosql.db.password=password jobengine.nexio.db.url=jdbc:sqlserver://10.10.1.59:1433;databaseName=NXDB; jobengine.nexio.db.user=sa jobengine.nexio.db.password=resolve -#System -jetty.home=../user.jobengine.osgi.rest/ -jetty.etc.config.urls=etc/user-jetty.xml,etc/user-jetty-ssl.xml,etc/user-jetty-ssl-context.xml,etc/user-jetty-http.xml,etc/user-jetty-https.xml -log4j.configuration=../-configuration/log4j.properties -log4j.configurationFile=../-configuration/log4j2.xml \ No newline at end of file +nexio.host=10.10.1.55 diff --git a/server/-configuration/mediacube-dev-user.properties b/server/-configuration/mediacube-dev-user.properties index 6f427b53..bd090ceb 100644 --- a/server/-configuration/mediacube-dev-user.properties +++ b/server/-configuration/mediacube-dev-user.properties @@ -1,4 +1,3 @@ -org.slf4j.simpleLogger.defaultLogLevel=debug jobengine.jobsteps.root=../user.jobengine.executors/bin jobengine.jobsteps.config=../user.jobengine.executors/config/config-worker.xml jobengine.jobtemplates.root=../user.jobengine.executors/jobtemplates @@ -9,12 +8,13 @@ jetty.etc.config.urls=etc/user-jetty.xml,etc/user-jetty-ssl.xml,etc/user-jetty-s log4j.configurationFile=../-configuration/log4j2-test.xml -jobengine.db.url=jdbc:db2://10.228.198.1:50000/mediaarc:retrieveMessagesFromServerOnGetMessage=true; -jobengine.db.user=db2admin -jobengine.db.password=password -jobengine.nosql.db.url=jdbc:db2://10.228.198.1:50000/mccache:retrieveMessagesFromServerOnGetMessage=true; -jobengine.nosql.db.user=db2admin -jobengine.nosql.db.password=password +jobengine.db.url=jdbc:db2://10.228.212.42:50000/mc:retrieveMessagesFromServerOnGetMessage=true; +jobengine.db.user=blobtest +jobengine.db.password=blobtest +jobengine.nosql.db.url=jdbc:db2://10.228.212.42:50000/mc:retrieveMessagesFromServerOnGetMessage=true; +jobengine.nosql.db.user=blobtest +jobengine.nosql.db.password=blobtest #jobengine.master.server=http://localhost:8888 -javax.ws.rs.ext.RuntimeDelegate=org.jboss.resteasy.spi.ResteasyProviderFactory \ No newline at end of file +javax.ws.rs.ext.RuntimeDelegate=org.jboss.resteasy.spi.ResteasyProviderFactory +nexio.host=10.10.1.55 \ No newline at end of file diff --git a/server/-configuration/run-mediacube-server-hirtv.launch b/server/-configuration/run-mediacube-server-hirtv.launch index 18fa4600..7cf76446 100644 --- a/server/-configuration/run-mediacube-server-hirtv.launch +++ b/server/-configuration/run-mediacube-server-hirtv.launch @@ -22,7 +22,7 @@ - + diff --git a/server/user.jobengine.executors/config/config-worker.xml b/server/user.jobengine.executors/config/config-worker.xml index d4dee362..c3d9854d 100644 --- a/server/user.jobengine.executors/config/config-worker.xml +++ b/server/user.jobengine.executors/config/config-worker.xml @@ -3,4 +3,5 @@ + \ No newline at end of file diff --git a/server/user.jobengine.executors/jobtemplates/common-copy.xml b/server/user.jobengine.executors/jobtemplates/common-copy.xml index 210b6462..aa9cd0a5 100644 --- a/server/user.jobengine.executors/jobtemplates/common-copy.xml +++ b/server/user.jobengine.executors/jobtemplates/common-copy.xml @@ -12,7 +12,7 @@ - + diff --git a/server/user.jobengine.executors/jobtemplates/fork-validate-and-archive.xml b/server/user.jobengine.executors/jobtemplates/fork-validate-and-archive.xml new file mode 100644 index 00000000..d9ad1edc --- /dev/null +++ b/server/user.jobengine.executors/jobtemplates/fork-validate-and-archive.xml @@ -0,0 +1,25 @@ + + + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/server/user.jobengine.executors/jobtemplates/nexio-checker.xml b/server/user.jobengine.executors/jobtemplates/nexio-checker.xml index 332068af..a2d4d047 100644 --- a/server/user.jobengine.executors/jobtemplates/nexio-checker.xml +++ b/server/user.jobengine.executors/jobtemplates/nexio-checker.xml @@ -1,10 +1,11 @@ - + + - + @@ -13,6 +14,12 @@ + + + + + + diff --git a/server/user.jobengine.executors/jobtemplates/validate-and-archive.xml b/server/user.jobengine.executors/jobtemplates/validate-and-archive.xml new file mode 100644 index 00000000..7344ef50 --- /dev/null +++ b/server/user.jobengine.executors/jobtemplates/validate-and-archive.xml @@ -0,0 +1,89 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/server/user.jobengine.executors/src/hu/user/mediacube/steps/DownloadableArchive.java b/server/user.jobengine.executors/src/hu/user/mediacube/steps/DownloadableArchive.java deleted file mode 100644 index d4a62b12..00000000 --- a/server/user.jobengine.executors/src/hu/user/mediacube/steps/DownloadableArchive.java +++ /dev/null @@ -1,7 +0,0 @@ -package hu.user.mediacube.steps; - -import com.ibm.nosql.json.api.BasicDBObject; - -public class DownloadableArchive extends BasicDBObject { - -} diff --git a/server/user.jobengine.executors/src/hu/user/mediacube/steps/DownloadableListBuilderStep.java b/server/user.jobengine.executors/src/hu/user/mediacube/steps/DownloadableListBuilderStep.java deleted file mode 100644 index a7d507f8..00000000 --- a/server/user.jobengine.executors/src/hu/user/mediacube/steps/DownloadableListBuilderStep.java +++ /dev/null @@ -1,81 +0,0 @@ -package hu.user.mediacube.steps; - -import java.io.File; -import java.io.IOException; -import java.nio.file.DirectoryStream; -import java.nio.file.Files; -import java.nio.file.Path; -import java.nio.file.Paths; - -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; - -import com.ibm.nosql.json.api.BasicDBObject; - -import user.jobengine.server.IJobEngine; -import user.jobengine.server.IJobRuntime; -import user.jobengine.server.steps.JobStep; -import user.jobengine.server.steps.StepEntry; - -public class DownloadableListBuilderStep extends JobStep { - private static final Logger logger = LogManager.getLogger(); - // private static final String UTF8 = "utf-8"; - private static final String STATUSFOLDER = ".STATUS"; - private static final String JSONEXT = ".json"; - private static final String CATCHEDEXT = ".catched"; - - public static final String ITEM_TITLE = "itemTitle"; - public static final String ITEM_HOUSEID = "itemHouseId"; - public static final String ITEM_DESCRIPTION = "itemDescription"; - public static final String MEDIA_HOUSEID = "mediaHouseId"; - public static final String MEDIA_TITLE = "mediaTitle"; - public static final String MEDIA_DESCRIPTION = "mediaDescription"; - public static final String MEDIA_TYPE = "mediaType"; - private static final String DURATION = "duration"; - private static final String EXISTING_MEDIAID = "existingMediaId"; - - @StepEntry - public Object[] execute(String sourcePath, int limit, IJobEngine jobEngine, IJobRuntime jobRuntime) { - DirectoryStream directoryStream = null; - try { - directoryStream = Files.newDirectoryStream(Paths.get(sourcePath)); - for (Path p : directoryStream) { - processPathItem(p); - } - } catch (Exception e) { - logger.error(getSessionMarker(), e.getMessage()); - } finally { - if (directoryStream != null) { - try { - directoryStream.close(); - } catch (IOException e) { - } - } - } - - return null; - } - - private String getMetadata(BasicDBObject dbObject, String fieldName) { - String result = null; - if (dbObject.containsKey(fieldName)) - result = dbObject.getString(fieldName); - return result; - } - - private boolean processPathItem(Path mediaFilePath) { - File mediaFile = mediaFilePath.toFile(); - - // if (mediaFile.length() > 0) - // return false; - - if (mediaFile.isDirectory()) { - return false; - } - - Path dotStorePath = Paths.get(mediaFilePath.getParent().toString(), STATUSFOLDER); - Path catchedFilePath = Paths.get(dotStorePath.toString(), mediaFile.getName() + CATCHEDEXT); - File catchedFile = catchedFilePath.toFile(); - return true; - } -} diff --git a/server/user.jobengine.executors/src/user/jobengine/server/steps/DownloadableMedia.java b/server/user.jobengine.executors/src/user/jobengine/server/steps/DownloadableMedia.java new file mode 100644 index 00000000..a42dec19 --- /dev/null +++ b/server/user.jobengine.executors/src/user/jobengine/server/steps/DownloadableMedia.java @@ -0,0 +1,7 @@ +package user.jobengine.server.steps; + +import com.ibm.nosql.json.api.BasicDBObject; + +public class DownloadableMedia extends BasicDBObject { + +} diff --git a/server/user.jobengine.executors/src/user/jobengine/server/steps/FileCopyStep.java b/server/user.jobengine.executors/src/user/jobengine/server/steps/FileCopyStep.java index 76edb1ce..4d5633a6 100644 --- a/server/user.jobengine.executors/src/user/jobengine/server/steps/FileCopyStep.java +++ b/server/user.jobengine.executors/src/user/jobengine/server/steps/FileCopyStep.java @@ -75,9 +75,8 @@ public class FileCopyStep extends JobStep { }); source.transferFrom(target, sourceFileName, targetFileName); - if (killDateDays > 0) { + if (killDateDays > -1) EscortFiles.createUNCKillDate(targetPath, targetFileName, killDateDays, marker); - } } catch (Exception e) { logger.catching(e); diff --git a/server/user.jobengine.executors/src/user/jobengine/server/steps/FrameNumberReaderStep.java b/server/user.jobengine.executors/src/user/jobengine/server/steps/FrameNumberReaderStep.java new file mode 100644 index 00000000..031a80b6 --- /dev/null +++ b/server/user.jobengine.executors/src/user/jobengine/server/steps/FrameNumberReaderStep.java @@ -0,0 +1,36 @@ +package user.jobengine.server.steps; + +import java.io.FileNotFoundException; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.Arrays; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + +import user.commons.mediatool.MediaInfo; +import user.jobengine.server.steps.JobStep; +import user.jobengine.server.steps.StepEntry; + +public class FrameNumberReaderStep extends JobStep { + private static final Logger logger = LogManager.getLogger(); + + @StepEntry + public Object[] execute(String sourcePath, String fileName) throws Exception { + Object result[] = null; + Path filePath = Paths.get(sourcePath, fileName); + MediaInfo mi = new MediaInfo(filePath); + try { + if (!filePath.toFile().exists()) + throw new FileNotFoundException("File not exists: " + filePath.toString()); + mi.process(); + long frames = mi.getFrames(); + result = Arrays.asList(frames).toArray(); + } catch (Exception e) { + logger.error(e.getMessage()); + throw e; + } + return result; + } + +} diff --git a/server/user.jobengine.executors/src/hu/user/mediacube/steps/NEXIOCheckerStep.java b/server/user.jobengine.executors/src/user/jobengine/server/steps/NEXIOCheckerStep.java similarity index 64% rename from server/user.jobengine.executors/src/hu/user/mediacube/steps/NEXIOCheckerStep.java rename to server/user.jobengine.executors/src/user/jobengine/server/steps/NEXIOCheckerStep.java index ec8177d1..c7ac4c94 100644 --- a/server/user.jobengine.executors/src/hu/user/mediacube/steps/NEXIOCheckerStep.java +++ b/server/user.jobengine.executors/src/user/jobengine/server/steps/NEXIOCheckerStep.java @@ -1,4 +1,4 @@ -package hu.user.mediacube.steps; +package user.jobengine.server.steps; import java.io.IOException; import java.sql.Timestamp; @@ -16,21 +16,17 @@ import user.commons.nexio.server.protocol.ProtocolException; import user.commons.remotestore.RemoteStoreProtocol; import user.jobengine.db.Media; import user.jobengine.db.Store; -import user.jobengine.server.steps.EscortFiles; -import user.jobengine.server.steps.JobStep; -import user.jobengine.server.steps.StepEntry; public class NEXIOCheckerStep extends JobStep { private static final String DOT_MXF = ".MXF"; private static final Logger logger = LogManager.getLogger(); - private DownloadableArchive createArchive(String title, Timestamp modified, Timestamp created, int duration) { - DownloadableArchive archive = new DownloadableArchive(); + private DownloadableMedia createArchive(String title, Timestamp modified, Timestamp created, int duration) { + DownloadableMedia archive = new DownloadableMedia(); archive.append("title", title); archive.append("fileName", title + DOT_MXF); archive.append("modified", modified); archive.append("created", created); - archive.append("duration", duration); return archive; } @@ -48,13 +44,13 @@ public class NEXIOCheckerStep extends JobStep { try { if (storeUri.getPortNumber() == 0) - controller = new Controller(storeUri.getRootPath(), storeUri.getPortNumber()); + controller = new Controller(storeUri.getUri(), storeUri.getPortNumber()); else - controller = new Controller(storeUri.getRootPath()); + controller = new Controller(storeUri.getUri()); controller.connect(); Mediabase mediabase = controller.getMediabase(); Iterator clips = mediabase.getClips(); - processClips(outputPath, limit, clips); + processClips(storeName, outputPath, limit, clips); } catch (Exception e) { logger.error(getSessionMarker(), e.getMessage()); throw e; @@ -66,13 +62,15 @@ public class NEXIOCheckerStep extends JobStep { return null; } - private void processClips(String outputPath, int limit, Iterator clips) throws ClipNotFoundException, IOException, ProtocolException { + private void processClips(String storeName, String outputPath, int limit, Iterator clips) + throws ClipNotFoundException, IOException, ProtocolException { int count = limit; while (clips.hasNext()) { - //megszakithatosag - if (getJobRuntime().isWaitingCancel()) + if (getJobRuntime().isWaitingCancel()) { + cancel(); break; + } Clip clip = clips.next(); String title = clip.getXid().get(); @@ -80,12 +78,20 @@ public class NEXIOCheckerStep extends JobStep { Timestamp created = Timestamp.from(clip.getRecordDateTimestamp().toInstant()); int duration = clip.getDuration(); + //TODO a rogzites alatt allo klippeket kiszurni + Media media = getManager().getMedia(title); - boolean create = media == null || media.getLength() != duration || !media.getModified().equals(modified) || !media.getCreated().equals(created); + + boolean create = media == null; + create |= create || !media.getModified().equals(modified); + create |= create || !media.getCreated().equals(created); + //a regi moziknal nincs hossz mentve + create |= create || (media.getLength() > 0 && media.getLength() != duration); + if (create) { - DownloadableArchive archive = createArchive(title, modified, created, duration); + DownloadableMedia archive = createArchive(title, modified, created, duration); //letezik : mar hozzadva - if (tryAdd(outputPath, archive)) + if (tryAdd(storeName, outputPath, archive, media)) count--; } @@ -98,9 +104,10 @@ public class NEXIOCheckerStep extends JobStep { } } - private boolean tryAdd(String outputPath, DownloadableArchive archive) throws IOException { + private boolean tryAdd(String storeName, String outputPath, DownloadableMedia archive, Media media) throws IOException { boolean result = false; - String fileName = archive.getString("fileName"); + String fileName = storeName + "." + archive.getString("fileName"); + if (!EscortFiles.isMetadataExists(outputPath, fileName)) { String metadata = archive.toPrettyString(""); EscortFiles.createMetadata(outputPath, fileName, metadata); diff --git a/server/user.jobengine.executors/src/user/jobengine/server/steps/TSMWriterStep.java b/server/user.jobengine.executors/src/user/jobengine/server/steps/TSMWriterStep.java new file mode 100644 index 00000000..d933b2b7 --- /dev/null +++ b/server/user.jobengine.executors/src/user/jobengine/server/steps/TSMWriterStep.java @@ -0,0 +1,205 @@ +package user.jobengine.server.steps; + +import java.io.File; +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.attribute.BasicFileAttributes; +import java.sql.Timestamp; +import java.util.Date; +import java.util.List; + +import org.apache.commons.lang.RandomStringUtils; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.apache.logging.log4j.Marker; +import org.apache.logging.log4j.message.Message; +import org.apache.logging.log4j.message.ParameterizedMessage; + +import user.commons.RemoteFile; +import user.commons.StoreUri; +import user.commons.remotestore.IProgressEventListener; +import user.commons.remotestore.IStatusEventListener; +import user.commons.remotestore.ProgressEvent; +import user.commons.remotestore.RemoteStoreProtocol; +import user.commons.remotestore.StatusEvent; +import user.jobengine.db.FileType; +import user.jobengine.db.IItemManager; +import user.jobengine.db.Media; +import user.jobengine.db.MediaFile; +import user.jobengine.db.Store; +import user.jobengine.server.IJobEngine; +import user.jobengine.server.IJobRuntime; +import user.jobengine.server.JobEngineException; + +public class TSMWriterStep extends JobStep { + private static final String MXFEXT = ".MXF"; + private static final Logger logger = LogManager.getLogger(); + private IItemManager manager; + private File sourceMediaFile; + private Store tsmStore; + private StoreUri targetUri; + private FileType fileType; + private Marker marker; + + @StepEntry + public Object[] execute(ArchiveItem archiveItem, Media mediaCubeMedia, int killDateDays) throws Exception { + marker = getSessionMarker(); + + File sourceMediaFile = new File(archiveItem.getMediaFile()); + String sourceFileName = sourceMediaFile.getName(); + String details = String.format("%s (%d bytes)", sourceFileName, sourceMediaFile.length()); + try { + + setAndCheck(archiveItem, mediaCubeMedia, getEngine()); + + getJobRuntime().setDescription(details); + + //TODO mxf helyett az osszes kiterjesztest!!!!! + //A dupla ellenorzes a napon beluli ismetlesek miatt kell + long existingMediaId = archiveItem.getExistingMediaId(); + if (sourceMediaFile.length() == 0 && existingMediaId == 0) { + existingMediaId = ItemManagerExtensions.getExistingRundownMedia(manager, sourceFileName.replace(MXFEXT, "")); + if (existingMediaId == 0) + existingMediaId = -1; + } + + if (existingMediaId == 0) + existingMediaId = ItemManagerExtensions.getExistingRundownMedia(manager, sourceFileName.replace(MXFEXT, "")); + + boolean randomizeName = Boolean.parseBoolean(System.getProperty("jobengine.randomize.archives", "false")); + + String targetFileName; + if (randomizeName) { + // a-z, A-Z, 0-9. For example: WRMcpIk7, s57JwCVA + //veletlenszeru neveket adunk! + targetFileName = String.format("%s-%s", RandomStringUtils.randomAlphanumeric(8), sourceFileName); + } else + targetFileName = sourceFileName; + + if (existingMediaId == 0) { + StoreUri sourceUri = manager.createStoreUri(RemoteStoreProtocol.LOCAL, sourceMediaFile.getParent().toString()); + + final IJobRuntime runtime = getJobRuntime(); + sourceUri.addProgressListener(new IProgressEventListener() { + @Override + public void progressChanged(ProgressEvent evt) { + runtime.incrementProgress(evt.getProgress()); + } + }); + sourceUri.addStatusListener(new IStatusEventListener() { + @Override + public void statusChanged(StatusEvent evt) { + evt.setCancel(!canContinue()); + } + }); + + RemoteFile remoteFile = sourceUri.transferFrom(targetUri, sourceFileName, targetFileName); + } + + if (existingMediaId > 0) + logger.info(marker, "Az '{}' TSM mentése nem szükséges, mert már megtalálható az archívumban.", sourceFileName); + + //Fel kell szabadítani, hogy a kovetkezo archivalaskor is nekifusson + if (existingMediaId == -1) { + logger.info(marker, "Az '{}' mentése jelenleg nem lehetséges, mert a szükséges metaadat még nem található meg az archívumban.", sourceFileName); + if (!archiveItem.removeCatchedFile()) + logger.error(marker, + "Az '{}' állomány .catched jelző állománya nem törölhető. Az újabb archiválási kísérlethez annak kézi eltávolítása szükséges!", + sourceMediaFile.getName()); + mediaCubeMedia.remove(); + } else { + saveMetadata(mediaCubeMedia, sourceMediaFile, targetFileName, existingMediaId); + logger.info(marker, "Az '{}' archiválása sikeres.", sourceFileName); + if (killDateDays != 0) + EscortFiles.createUNCKillDate(sourceMediaFile.getParent(), sourceFileName, killDateDays, marker); + } + + } catch (Exception e) { + logger.catching(e); + Message m = new ParameterizedMessage("Az '{}' állomány archiválása sikertelen. A rendszer hibaüzenete: {}", details, e.getMessage()); + logger.error(marker, m); + if (!archiveItem.removeCatchedFile()) + logger.error(marker, + "Az '{}' állomány .catched jelző állománya nem törölhető. Az újabb archiválási kísérlethez annak kézi eltávolítása szükséges!", + sourceMediaFile.getName()); + throw new Exception(m.getFormattedMessage()); + } + return null; + } + + private void saveMetadata(Media mediaCubeMedia, File sourceFile, String targetFileName, long existingMediaId) { + + if (existingMediaId == 0) { + MediaFile mediaFile = manager.createMediaFile(targetFileName, fileType, tsmStore, mediaCubeMedia); + mediaFile.setHouseId(sourceFile.getName()); + mediaFile.add(); + } else { + Media existingMedia = manager.getMedia(existingMediaId); + List mediaFiles = existingMedia.getMediaFiles(); + if (mediaFiles != null) { + for (MediaFile mf : mediaFiles) { + mf.setPersister(manager); + mf.setId(0); + mf.setMedia(mediaCubeMedia); + mf.add(); + } + } + } + mediaCubeMedia.setPersister(manager); + + try { + BasicFileAttributes attr = Files.readAttributes(sourceFile.toPath(), BasicFileAttributes.class); + mediaCubeMedia.setArchived(new Timestamp(attr.creationTime().toMillis())); + } catch (IOException e) { + logger.catching(e); + mediaCubeMedia.setArchived(new Timestamp(new Date().getTime())); + } + + mediaCubeMedia.modify(); + } + + private void setAndCheck(ArchiveItem archiveItem, Media mediaCubeMedia, IJobEngine jobEngine) throws JobEngineException, IOException { + if (jobEngine == null) { + logger.error(marker, "Az folyamatkezelő réteg nem elérhető."); + throw new NullPointerException("Internal error, missing JobEngine reference."); + } + manager = jobEngine.getItemManager(); + if (manager == null) { + logger.error(marker, "Az adatbáziskezelő réteg nem elérhető."); + throw new NullPointerException("Internal error, missing ItemManager reference."); + } + if (archiveItem == null) { + logger.error(marker, "A folyamat 'archiveItem' bemeneti paramétere üres."); + throw new NullPointerException("System is not configured properly, missing 'archiveItem' input parameter."); + } + sourceMediaFile = new File(archiveItem.getMediaFile()); + if (sourceMediaFile == null) { + logger.error(marker, "A folyamat 'archiveItem' bemeneti paraméter 'mediaFile' értéke üres."); + throw new NullPointerException("System is not configured properly, missing 'mediaFile' value in 'archiveItem' input parameter."); + } + if (!sourceMediaFile.exists()) { + logger.error(marker, "Az {} állomány nem létezik vagy nem érhető el.", sourceMediaFile.getName()); + throw new IOException(String.format("Input file {} not exists or unreachable.", sourceMediaFile.getName())); + } + tsmStore = manager.getSystemStore(false); + if (tsmStore == null) { + logger.error(marker, "A TSM rendszer beállítás nem elérhető."); + throw new NullPointerException("System is not configured properly, missing TSM Store."); + } + targetUri = tsmStore.getSourceStoreUri(RemoteStoreProtocol.TSM); + if (targetUri == null) { + logger.error(marker, "A TSM rendszer beállítás paraméterei nem elérhetőek."); + throw new NullPointerException("System is not configured properly, missing TSM StoreUri."); + } + fileType = manager.getFileType("High-res"); + if (fileType == null) { + logger.error(marker, "Adatbázis bejegyzés hiba, a 'High-res' FileType nem található."); + throw new NullPointerException("System is not configured properly, missing 'High-res' FileType."); + } + if (mediaCubeMedia == null) { + logger.error(marker, "A folyamat 'mediaCubeMedia' bemeneti paramétere üres."); + throw new NullPointerException("System is not configured properly, 'mediaCubeMedia' input parameter missing."); + } + + } +} diff --git a/server/user.jobengine.executors/src/user/jobengine/server/steps/ValidateAndArchiveForkStep.java b/server/user.jobengine.executors/src/user/jobengine/server/steps/ValidateAndArchiveForkStep.java new file mode 100644 index 00000000..cd55002a --- /dev/null +++ b/server/user.jobengine.executors/src/user/jobengine/server/steps/ValidateAndArchiveForkStep.java @@ -0,0 +1,104 @@ +package user.jobengine.server.steps; + +import java.io.IOException; +import java.nio.file.DirectoryStream; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + +import com.ibm.nosql.json.util.JSON; + +import user.jobengine.db.ArchivedMedia; +import user.jobengine.server.IJobEngine; +import user.jobengine.server.IJobRuntime; + +public class ValidateAndArchiveForkStep extends JobStep { + private static final Logger logger = LogManager.getLogger(); + private static final String JOB_TEMPLATE = "validate-and-archive.xml"; + + @StepEntry + public Object[] execute(String template, String sourcePath, int limit) { + DirectoryStream directoryStream = null; + int count = limit; + try { + if (getJobRuntime().forkPrepare()) { + directoryStream = Files.newDirectoryStream(Paths.get(sourcePath)); + if (directoryStream != null) { + for (Path p : directoryStream) { + + if (getJobRuntime().isWaitingCancel()) { + cancel(); + break; + } + + if (processPathItem(template, p)) + count--; + + if (count == 0) + break; + } + } + + getJobRuntime().forkWaitComplete(); + } + + setProgress(100); + } catch (Exception e) { + logger.error(getSessionMarker(), e.getMessage()); + } finally { + if (directoryStream != null) { + try { + directoryStream.close(); + } catch (IOException e) { + } + } + } + + return null; + } + + private boolean processPathItem(String template, Path escortFile) throws IOException { + if (escortFile.toFile().isDirectory()) + return false; + + String fileName = escortFile.getFileName().toString(); + //minimum 3 karakter: STORENAME.FILENAME.MXF.JSON + long dotCount = fileName.chars().filter(ch -> ch == '.').count(); + if (dotCount < 4) + return false; + + byte[] bytes = Files.readAllBytes(escortFile); + String content = new String(bytes); + if (content != null) { + try { + DownloadableMedia archive = (DownloadableMedia) JSON.parse(content); + logger.info("Starting {}", template); + } catch (Exception e) { + logger.error(e.getMessage()); + } + + } + + return true; + } + + public void submit(ArchivedMedia archivedMedia, String recipient, String houseId, String targetPathType, IJobEngine jobEngine, IJobRuntime jobRuntime) + throws Exception { + // try { + // ScheduledJob scheduledJob = jobEngine.getScheduledJob(CHILD_TEMPLATE); + // Map parameters = scheduledJob.getJobParameters(); + // parameters.put(ARCHIVEDMEDIA, archivedMedia); + // parameters.put(HOUSEID, houseId); + // parameters.put(RECIPIENT, recipient); + // parameters.put(TARGET_PATH_TYPE, targetPathType); + // IJobRuntime child = jobEngine.submit(jobRuntime, null, CHILD_TEMPLATE, String.format("Visszatöltés %s részére", recipient), parameters); + // ((MediaCubeMarker) child.getMarker()).setTo(recipient); + // } catch (Exception e) { + // logger.catching(e); + // logger.error(marker, "Hiba a kötegelt visszatöltésben. A rendszer üzenete: {}", e.getMessage()); + // } + } +} diff --git a/server/user.jobengine.osgi.server/test/user/jobengine/server/IT/AMCTests.java b/server/user.jobengine.osgi.server/test/user/jobengine/server/IT/AMCTests.java index 73d157a9..adc30eaa 100644 --- a/server/user.jobengine.osgi.server/test/user/jobengine/server/IT/AMCTests.java +++ b/server/user.jobengine.osgi.server/test/user/jobengine/server/IT/AMCTests.java @@ -1,14 +1,18 @@ package user.jobengine.server.IT; import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotNull; import java.io.FileInputStream; import java.net.URL; +import java.nio.file.Files; +import java.nio.file.Paths; import java.util.HashMap; import java.util.Map; import java.util.Properties; import java.util.concurrent.CountDownLatch; +import org.apache.commons.io.FileUtils; import org.junit.AfterClass; import org.junit.BeforeClass; import org.junit.Test; @@ -26,6 +30,7 @@ import user.jobengine.server.JobEngine; public class AMCTests { private static IItemManager manager; private static IJobEngine jobEngine; + private static final String output = "/opt/NEXIO";; @BeforeClass public static void initialize() throws Exception { @@ -52,32 +57,57 @@ public class AMCTests { } @Test - public void testNexioCheckerStep() throws Exception { - Store nexioStore = manager.createStore("NEXIO", false, false); - nexioStore.add(); + public void testForkValidateAndArchive() throws Exception { + Files.createDirectories(Paths.get(output)); + DownloadableArchive media = new DownloadableArchive(); + } + @Test + public void testNexioChecker() throws Exception { //Tesztelni a HirTV-n lehet - StoreUri storeUri = manager.createStoreUri(RemoteStoreProtocol.NEXIO, "10.10.1.55"); - storeUri.setStoreId(nexioStore.getId()); - storeUri.add(); - + String storeName = "NEXIO"; + Store nexioStore = manager.getStore(storeName); + StoreUri storeUri = null; + if (nexioStore == null) { + nexioStore = manager.createStore(storeName, false, false); + nexioStore.add(); + + storeUri = manager.createStoreUri(RemoteStoreProtocol.NEXIO, System.getProperty("nexio.host")); + storeUri.setSource(true); + storeUri.setTarget(true); + storeUri.setStoreId(nexioStore.getId()); + storeUri.add(); + } else { + storeUri = nexioStore.getSourceStoreUri(RemoteStoreProtocol.NEXIO); + storeUri.setPersister(manager); + nexioStore.setPersister(manager); + } CountDownLatch finishLatch = new CountDownLatch(1); + Files.createDirectories(Paths.get(output)); + Map jobParams = new HashMap<>(); - jobParams.put("storeName", "NEXIO"); - jobParams.put("outputPath", "/opt"); + jobParams.put("storeName", storeName); + jobParams.put("outputPath", output); jobParams.put("limit", 1); + jobEngine.addJobChangedEventListener(e -> { + }); + IJobRuntime jobRuntime = jobEngine.submit(null, e -> { if (e.isRuntimeTerminated()) finishLatch.countDown(); - }, "nexio-check.xml", "NEXIO check", jobParams); + }, "nexio-checker.xml", "NEXIO check", jobParams); - finishLatch.await(); + if (jobRuntime != null) + finishLatch.await(); storeUri.remove(); nexioStore.remove(); + FileUtils.deleteDirectory(Paths.get(output).toFile()); + + assertNotNull(jobRuntime); assertEquals(JobStatus.FINISHED, jobRuntime.getStatus()); } -- 2.54.0