git-tfs-id: [http://tfs.userrendszerhaz.hu:8080/tfs/DefaultCollection]$/MediaCube...
authorvasary.daniel <TFS\vasary.daniel>
Tue, 27 Jul 2021 12:40:45 +0000 (12:40 +0000)
committervasary.daniel <TFS\vasary.daniel>
Tue, 27 Jul 2021 12:40:45 +0000 (12:40 +0000)
server/-product/production/AMC/jobs/steps/PBARCHIVETransferFromStep.java [new file with mode: 0644]
server/-product/production/AMC/jobs/steps/PBArchiveCheckerStep.java [new file with mode: 0644]
server/-product/production/AMC/jobs/templates/PBARCHIVE-archive-checker.xml [new file with mode: 0644]
server/-product/production/AMC/jobs/templates/PBARCHIVE-validate-and-archive.xml [new file with mode: 0644]
server/hu.user.mediacube.executors.tests/src/hu/user/mediacube/executors/tests/SupportMVDeletedFinishedShows.java [new file with mode: 0644]
server/user.jobengine.executors/src/user/jobengine/server/steps/PBARCHIVETransferFromStep.java [new file with mode: 0644]
server/user.jobengine.executors/src/user/jobengine/server/steps/PBArchiveCheckerStep.java [new file with mode: 0644]
server/user.jobengine.osgi.db/migrations/scripts/038_create_pbarchive_store.sql [new file with mode: 0644]

diff --git a/server/-product/production/AMC/jobs/steps/PBARCHIVETransferFromStep.java b/server/-product/production/AMC/jobs/steps/PBARCHIVETransferFromStep.java
new file mode 100644 (file)
index 0000000..d47c2b7
--- /dev/null
@@ -0,0 +1,21 @@
+package user.jobengine.server.steps;\r
+\r
+import java.nio.file.Path;\r
+import java.nio.file.Paths;\r
+\r
+import org.apache.logging.log4j.LogManager;\r
+import org.apache.logging.log4j.Logger;\r
+\r
+import user.commons.StoreUri;\r
+\r
+public class PBARCHIVETransferFromStep extends TransferStep {\r
+       private static final Logger logger = LogManager.getLogger();\r
+\r
+       @Override\r
+       @StepEntry\r
+       public Object[] execute(StoreUri sourceStoreUri, String sourceFileName, StoreUri targetStoreUri, String targetFileName) throws Exception {\r
+               Object[] result = null;\r
+               result = super.execute(sourceStoreUri, sourceFileName, targetStoreUri, targetFileName);\r
+               return result;\r
+       }\r
+}\r
diff --git a/server/-product/production/AMC/jobs/steps/PBArchiveCheckerStep.java b/server/-product/production/AMC/jobs/steps/PBArchiveCheckerStep.java
new file mode 100644 (file)
index 0000000..8beafb2
--- /dev/null
@@ -0,0 +1,187 @@
+package user.jobengine.server.steps;\r
+\r
+import java.io.IOException;\r
+import java.nio.file.Paths;\r
+import java.sql.Timestamp;\r
+import java.text.SimpleDateFormat;\r
+import java.time.Duration;\r
+import java.time.Instant;\r
+import java.util.Iterator;\r
+\r
+import org.apache.logging.log4j.LogManager;\r
+import org.apache.logging.log4j.Logger;\r
+\r
+import user.commons.DownloadableMedia;\r
+import user.commons.StoreUri;\r
+import user.commons.nexio.api.Clip;\r
+import user.commons.nexio.api.ClipNotFoundException;\r
+import user.commons.nexio.api.Controller;\r
+import user.commons.nexio.api.Mediabase;\r
+import user.commons.nexio.server.protocol.ProtocolException;\r
+import user.commons.remotestore.RemoteStoreProtocol;\r
+import user.jobengine.db.Media;\r
+import user.jobengine.db.MediaFile;\r
+\r
+public class NEXIOArchiveCheckerStep extends JobStep {\r
+       private static final String DOT_MXF = ".mxf";\r
+       private static final Logger logger = LogManager.getLogger();\r
+       private SimpleDateFormat df = new SimpleDateFormat("yyyyMMdd-HHmmss");\r
+\r
+       private boolean check(Timestamp modified, Timestamp created, int duration, Media media, int videoFormat, int lastModifiedHours) {\r
+               String title = media.getTitle();\r
+\r
+               //formatum\r
+               if (!(videoFormat == 3 || videoFormat == 19)) {\r
+                       logger.debug(getMarker(), "{} unsupported video format", videoFormat);\r
+                       return false;\r
+               }\r
+\r
+               if (!checkModifiedByTime(title, modified, lastModifiedHours))\r
+                       return false;\r
+\r
+               //utolso modositas\r
+               MediaFile mediaFile = getManager().getSystemMediaFile(media);\r
+               boolean isModified = true;\r
+               if (mediaFile == null) {\r
+                       logger.info(getMarker(), "{} system media file not available", title);\r
+                       return false;\r
+               }\r
+\r
+               String mcMod = df.format(mediaFile.getLastModified());\r
+               String nexioMod = df.format(modified);\r
+               isModified = mcMod.compareTo(nexioMod) < 0;\r
+               if (isModified) {\r
+                       logger.info(getMarker(), "{} modification time: MC {} < NEXIO {}", title, mcMod, nexioMod);\r
+                       return true;\r
+               }\r
+\r
+               boolean isLengthChanged = media.getLength() > 0 && Math.abs(media.getLength() - duration) > 3;\r
+               if (isLengthChanged) {\r
+                       logger.info(getMarker(), "{} length: MC {} != NEXIO {}", title, media.getLength(), duration);\r
+                       return true;\r
+               }\r
+\r
+               return false;\r
+       }\r
+\r
+       private boolean checkModifiedByTime(String title, Timestamp modified, int lastModifiedHours) {\r
+               //utolso modositas a mosthoz kepest\r
+               //The result of this method can be a negative period if the end is before the start.\r
+               long modifiedHours = Duration.between(modified.toInstant(), Instant.now()).toHours();\r
+               if (modifiedHours < 0) {\r
+                       logger.info(getMarker(), "{} modification time greather than now", title);\r
+                       return false;\r
+               }\r
+               if (modifiedHours < lastModifiedHours) {\r
+                       logger.debug(getMarker(), "{} modification time is too cloose for now", title);\r
+                       return false;\r
+               }\r
+\r
+               return true;\r
+       }\r
+\r
+       @StepEntry\r
+       public Object[] execute(String sourceStoreName, String escortStoreName, int lastModifiedHours) throws Exception {\r
+               Controller controller = null;\r
+\r
+               try {\r
+                       getJobRuntime().setRelated(sourceStoreName + " archive");\r
+\r
+                       StoreUri nexioStoreUri = getManager().getStoreUri(sourceStoreName, RemoteStoreProtocol.NEXIO);\r
+                       StoreUri sourceStoreUri = getManager().getStoreUri(sourceStoreName, RemoteStoreProtocol.FTP);\r
+                       StoreUri targetStoreUri = getManager().getStoreUri("TSM", RemoteStoreProtocol.TSM);\r
+                       StoreUri escortStoreUri = getManager().getStoreUri(escortStoreName, RemoteStoreProtocol.LOCAL);\r
+                       String outputPath = Paths.get(escortStoreUri.toString(true)).toString();\r
+\r
+                       logger.info(getMarker(), "Connecting mediabase {}", nexioStoreUri.toString());\r
+                       if (nexioStoreUri.getPortNumber() == 0)\r
+                               controller = new Controller(nexioStoreUri.getUri(), nexioStoreUri.getPortNumber());\r
+                       else\r
+                               controller = new Controller(nexioStoreUri.getUri());\r
+                       controller.connect();\r
+                       Mediabase mediabase = controller.getMediabase();\r
+\r
+                       Iterator<Clip> clips = mediabase.getClips();\r
+                       logger.info(getMarker(), "Processing clips");\r
+\r
+                       processClips(sourceStoreName, sourceStoreUri, targetStoreUri, outputPath, clips, lastModifiedHours);\r
+               } catch (Exception e) {\r
+                       logger.error(getSessionMarker(), e.getMessage() == null ? "Unknown error" : e.getMessage());\r
+                       throw e;\r
+               } finally {\r
+                       setProgress(100);\r
+                       if (controller != null)\r
+                               controller.disconnect();\r
+               }\r
+\r
+               return null;\r
+       }\r
+\r
+       private void processClips(String storeName, StoreUri sourceStoreUri, StoreUri targetStoreUri, String outputPath, Iterator<Clip> clips,\r
+                       int lastModifiedHours) throws ClipNotFoundException, IOException, ProtocolException {\r
+               int processed = 0;\r
+               while (clips.hasNext()) {\r
+\r
+                       if (getJobRuntime().isWaitingCancel()) {\r
+                               cancel();\r
+                               break;\r
+                       }\r
+\r
+                       Clip clip = clips.next();\r
+\r
+                       try {\r
+                               String title = null;\r
+                               Timestamp created = null;\r
+                               Timestamp modified = null;\r
+                               int frames = 0;\r
+                               int videoFormat = 0;\r
+                               frames = clip.getDuration();\r
+\r
+                               //ures clip\r
+                               if (frames == 1)\r
+                                       continue;\r
+\r
+                               title = clip.getXid().get();\r
+                               created = Timestamp.from(clip.getRecordDateTimestamp().toInstant());\r
+                               modified = Timestamp.from(clip.getModifiedTimestamp().toInstant());\r
+                               videoFormat = clip.getVideoFormat();\r
+\r
+                               Media media = getManager().getMedia(title);\r
+                               logger.debug(getMarker(), "Checking {}", title);\r
+                               boolean createEscort = false;\r
+\r
+                               long mediaId = 0;\r
+                               if (media == null) {\r
+                                       if (checkModifiedByTime(title, modified, lastModifiedHours)) {\r
+                                               createEscort = true;\r
+                                               logger.info(getMarker(), "{} not archived yet", title);\r
+                                       }\r
+                               } else {\r
+                                       mediaId = media.getId();\r
+                                       createEscort = check(modified, created, frames, media, videoFormat, lastModifiedHours);\r
+                               }\r
+\r
+                               if (createEscort && !title.contains("*")) {\r
+                                       String fileName = title + DOT_MXF;\r
+                                       DownloadableMedia downloadable = DownloadableMedia.create(title, fileName, modified, created, frames, 0, sourceStoreUri.getId(),\r
+                                                       targetStoreUri.getId(), mediaId);\r
+                                       String escortFileName = storeName + "." + downloadable.getString("fileName");\r
+                                       if (EscortFiles.createMetadataIfNotExists(outputPath, escortFileName, downloadable.toPrettyString(""))) {\r
+                                               logger.info(getMarker(), "Archive status file created for {}", fileName);\r
+                                       } else {\r
+                                               logger.info(getMarker(), "Archive status file already exists for {}", fileName);\r
+                                       }\r
+\r
+                               }\r
+\r
+                               processed++;\r
+                               if (processed % 100 == 0)\r
+                                       logger.info(getMarker(), "Processed {} records", processed);\r
+                       } catch (Exception e) {\r
+                               logger.error(getSessionMarker(), e.getMessage());\r
+                               continue;\r
+                       }\r
+               }\r
+       }\r
+\r
+}\r
diff --git a/server/-product/production/AMC/jobs/templates/PBARCHIVE-archive-checker.xml b/server/-product/production/AMC/jobs/templates/PBARCHIVE-archive-checker.xml
new file mode 100644 (file)
index 0000000..54d3606
--- /dev/null
@@ -0,0 +1,26 @@
+<?xml version="1.0" encoding="UTF-8"?>\r
+<!-- A PB ARCHIVE mappat vizsgalva, megkeresi azokat a fajlokat, amik archivalhatoak. Ezekre archivalasi bejegyzest keszit.  -->\r
+<jobtemplate useSessionLog="false">\r
+<declarations>\r
+       <parameters>\r
+               <parameter name="sourceStoreName" type="java.lang.String"/>\r
+               <parameter name="escortStoreName" type="java.lang.String"/>\r
+               <parameter name="lastModifiedHours" type="java.lang.Integer"/>\r
+       </parameters>\r
+</declarations>\r
+<commands>\r
+       <calljobstep type="PBArchiveCheckerStep.java" weight="1">\r
+               <inputs>\r
+                       <input>\r
+                               <parameter name="sourceStoreName" />\r
+                       </input>\r
+                       <input>\r
+                               <parameter name="escortStoreName" />\r
+                       </input>\r
+                       <input>\r
+                               <parameter name="lastModifiedHours" />\r
+                       </input>\r
+               </inputs>\r
+       </calljobstep>\r
+</commands>\r
+</jobtemplate>
\ No newline at end of file
diff --git a/server/-product/production/AMC/jobs/templates/PBARCHIVE-validate-and-archive.xml b/server/-product/production/AMC/jobs/templates/PBARCHIVE-validate-and-archive.xml
new file mode 100644 (file)
index 0000000..2b4a7d7
--- /dev/null
@@ -0,0 +1,94 @@
+<?xml version="1.0" encoding="UTF-8"?>\r
+<jobtemplate multiInstance="true" useSessionLog="false">\r
+       <declarations>\r
+               <parameters>\r
+                       <parameter name="sourceStoreUri" type="user.commons.StoreUri"/>\r
+                       <parameter name="fileName" type="java.lang.String"/>\r
+                       <parameter name="tempStoreUri" type="user.commons.StoreUri"/>\r
+                       <parameter name="targetStoreUri" type="user.commons.StoreUri"/>\r
+                       <parameter name="expectedFrameNumber" type="java.lang.Long"/>\r
+                       <parameter name="expectedSize" type="java.lang.Long"/>\r
+                       <parameter name="expectedColorSpace" type="java.lang.String"/>\r
+                       <parameter name="escortFile" type="java.lang.String"/>\r
+               </parameters>\r
+       </declarations>\r
+       <commands>\r
+               <calljobstep type="PBARCHIVETransferFromStep.java" weight="5">\r
+                       <inputs>\r
+                               <input>\r
+                                       <parameter name="sourceStoreUri" />\r
+                               </input>\r
+                               <input>\r
+                                       <parameter name="fileName" />\r
+                               </input>\r
+                               <input>\r
+                                       <parameter name="tempStoreUri" />\r
+                               </input>\r
+                               <input>\r
+                                       <parameter name="fileName" />\r
+                               </input>\r
+                       </inputs>\r
+               </calljobstep>\r
+               <calljobstep type="FileValidatorStep.java" weight="1">\r
+                       <inputs>\r
+                               <input>\r
+                                       <parameter name="tempStoreUri" />\r
+                               </input>\r
+                               <input>\r
+                                       <parameter name="fileName" />\r
+                               </input>\r
+                               <input>\r
+                                       <parameter name="expectedFrameNumber" />\r
+                               </input>\r
+                               <input>\r
+                                       <parameter name="expectedSize" />\r
+                               </input>\r
+                               <input>\r
+                                       <parameter name="expectedColorSpace" />\r
+                               </input>\r
+                               <input>\r
+                                       <parameter name="escortFile" />\r
+                               </input>\r
+                       </inputs>\r
+               </calljobstep>\r
+               <calljobstep type="TSMTransferToStep.java" weight="5">\r
+                       <inputs>\r
+                               <input>\r
+                                       <parameter name="tempStoreUri" />\r
+                               </input>\r
+                               <input>\r
+                                       <parameter name="fileName" />\r
+                               </input>\r
+                               <input>\r
+                                       <parameter name="targetStoreUri" />\r
+                               </input>\r
+                               <input>\r
+                                       <parameter name="fileName" />\r
+                               </input>\r
+                       </inputs>\r
+               </calljobstep>\r
+               <calljobstep type="MetadataPersisterStep.java" weight="5">\r
+                       <inputs>\r
+                               <input>\r
+                                       <parameter name="targetStoreUri" />\r
+                               </input>\r
+                               <input>\r
+                                       <parameter name="escortFile" />\r
+                               </input>\r
+                       </inputs>\r
+               </calljobstep>\r
+               <calljobstep type="FileCleanupStep.java" weight="5">\r
+                       <inputs>\r
+                               <input>\r
+                                       <parameter name="tempStoreUri" />\r
+                               </input>\r
+                               <input>\r
+                                       <parameter name="fileName" />\r
+                               </input>\r
+                               <input>\r
+                                       <parameter name="escortFile" />\r
+                               </input>\r
+                       </inputs>\r
+               </calljobstep>\r
+       </commands>\r
+</jobtemplate>
\ No newline at end of file
diff --git a/server/hu.user.mediacube.executors.tests/src/hu/user/mediacube/executors/tests/SupportMVDeletedFinishedShows.java b/server/hu.user.mediacube.executors.tests/src/hu/user/mediacube/executors/tests/SupportMVDeletedFinishedShows.java
new file mode 100644 (file)
index 0000000..f8bcd86
--- /dev/null
@@ -0,0 +1,72 @@
+package hu.user.mediacube.executors.tests;\r
+\r
+import java.nio.file.Files;\r
+import java.nio.file.Path;\r
+import java.nio.file.Paths;\r
+import java.util.List;\r
+\r
+import org.apache.logging.log4j.LogManager;\r
+import org.apache.logging.log4j.Logger;\r
+import org.junit.AfterClass;\r
+import org.junit.BeforeClass;\r
+import org.junit.Test;\r
+\r
+import user.jobengine.db.IItemManager;\r
+import user.jobengine.db.ItemManager;\r
+import user.jobengine.db.Media;\r
+\r
+/***\r
+ * MediaVivantis delete-materials.log-bol kepzett file vizsgalata\r
+ * pl. cat delete-materials-12-30-2020-1.log delete-materials-04-16-2021-1.log | grep 'killdate bejegyzés alapján sikeresen' > deleted_finished_shows.txt \r
+ * @author elgekko\r
+ */\r
+public class SupportMVDeletedFinishedShows {\r
+       Logger logger = LogManager.getLogger();\r
+\r
+       protected static IItemManager manager = null;\r
+\r
+       @BeforeClass\r
+       static public void setUpConnection() throws Exception {\r
+               manager = new ItemManager();\r
+               manager.connect();\r
+       }\r
+\r
+       @AfterClass\r
+       static public void tearDownConnection() throws Exception {\r
+               manager.disconnect();\r
+       }\r
+\r
+       @Test\r
+       public void checkDeletedFinishedShows() throws Exception {\r
+               //              Store systemStore = manager.getSystemStore(false);\r
+               //              logger.info(systemStore.getName());\r
+               Path src = Paths.get("/opt/mv/deleted_finished_shows.txt");\r
+               List<String> lines = Files.readAllLines(src);\r
+\r
+               int p = 0;\r
+               for (String line : lines) {\r
+                       String mediaFileName = line.substring(62, line.indexOf(' ', 62));\r
+                       //logger.info(media);\r
+\r
+                       String sql = "select mediaid from mediafile where houseid=?";\r
+\r
+                       manager.executeQuery(sql, rs -> {\r
+                               long mediaid = rs.getLong(1);\r
+                               //logger.info("{} {}", mediaFileName, mediaid);\r
+                               Media media = manager.getMedia(mediaid);\r
+                               if (media.getMediaFilesCount() != 2) {\r
+                                       logger.info("{} {}", mediaFileName, mediaid);\r
+                               }\r
+                               return true;\r
+                       }, st -> {\r
+                               st.setString(1, mediaFileName);\r
+                       });\r
+                       p++;\r
+\r
+                       if (p % 100 == 0)\r
+                               logger.info("Processed {}", p);\r
+               }\r
+\r
+               logger.info("Processed {} lines", lines.size());\r
+       }\r
+}\r
diff --git a/server/user.jobengine.executors/src/user/jobengine/server/steps/PBARCHIVETransferFromStep.java b/server/user.jobengine.executors/src/user/jobengine/server/steps/PBARCHIVETransferFromStep.java
new file mode 100644 (file)
index 0000000..d47c2b7
--- /dev/null
@@ -0,0 +1,21 @@
+package user.jobengine.server.steps;\r
+\r
+import java.nio.file.Path;\r
+import java.nio.file.Paths;\r
+\r
+import org.apache.logging.log4j.LogManager;\r
+import org.apache.logging.log4j.Logger;\r
+\r
+import user.commons.StoreUri;\r
+\r
+public class PBARCHIVETransferFromStep extends TransferStep {\r
+       private static final Logger logger = LogManager.getLogger();\r
+\r
+       @Override\r
+       @StepEntry\r
+       public Object[] execute(StoreUri sourceStoreUri, String sourceFileName, StoreUri targetStoreUri, String targetFileName) throws Exception {\r
+               Object[] result = null;\r
+               result = super.execute(sourceStoreUri, sourceFileName, targetStoreUri, targetFileName);\r
+               return result;\r
+       }\r
+}\r
diff --git a/server/user.jobengine.executors/src/user/jobengine/server/steps/PBArchiveCheckerStep.java b/server/user.jobengine.executors/src/user/jobengine/server/steps/PBArchiveCheckerStep.java
new file mode 100644 (file)
index 0000000..071ed7c
--- /dev/null
@@ -0,0 +1,161 @@
+package user.jobengine.server.steps;\r
+\r
+import java.io.IOException;\r
+import java.nio.file.Paths;\r
+import java.sql.Timestamp;\r
+import java.text.SimpleDateFormat;\r
+import java.time.Duration;\r
+import java.time.Instant;\r
+import java.util.ArrayList;\r
+import java.util.Iterator;\r
+import java.util.List;\r
+\r
+import org.apache.commons.net.ftp.FTPClient;\r
+import org.apache.logging.log4j.LogManager;\r
+import org.apache.logging.log4j.Logger;\r
+import org.w3c.dom.stylesheets.LinkStyle;\r
+\r
+import user.commons.DownloadableMedia;\r
+import user.commons.RemoteFile;\r
+import user.commons.StoreUri;\r
+import user.commons.nexio.api.Clip;\r
+import user.commons.nexio.api.ClipNotFoundException;\r
+import user.commons.nexio.api.Controller;\r
+import user.commons.nexio.api.Mediabase;\r
+import user.commons.nexio.server.protocol.ProtocolException;\r
+import user.commons.remotestore.FtpDirectoryLister;\r
+import user.commons.remotestore.IDirectoryLister;\r
+import user.commons.remotestore.RemoteStoreProtocol;\r
+import user.jobengine.db.Media;\r
+import user.jobengine.db.MediaFile;\r
+import user.jobengine.server.steps.shared.EscortFiles;\r
+\r
+public class PBArchiveCheckerStep extends JobStep {\r
+       private static final String DOT_MXF = ".mxf";\r
+       private static final Logger logger = LogManager.getLogger();\r
+       private SimpleDateFormat df = new SimpleDateFormat("yyyyMMdd-HHmmss");\r
+\r
+\r
+       private boolean checkModifiedByTime(String title, Timestamp modified, int lastModifiedHours) {\r
+               long modifiedHours = Duration.between(modified.toInstant(), Instant.now()).toHours();\r
+               if (modifiedHours < 0) {\r
+                       logger.info(getMarker(), "{} modification time greather than now", title);\r
+                       return false;\r
+               }\r
+               if (modifiedHours < lastModifiedHours) {\r
+                       logger.debug(getMarker(), "{} modification time is too cloose for now", title);\r
+                       return false;\r
+               }\r
+\r
+               return true;\r
+       }\r
+\r
+       @StepEntry\r
+       public Object[] execute(String sourceStoreName, String escortStoreName, int lastModifiedHours) throws Exception {\r
+               logger.info(getMarker(), "Started {}", sourceStoreName);\r
+               StoreUri sourceStoreUri = null;\r
+               try {\r
+                       getJobRuntime().setRelated(sourceStoreName + " archive");\r
+\r
+                       sourceStoreUri = getManager().getStoreUri(sourceStoreName, RemoteStoreProtocol.FTP);\r
+\r
+                       StoreUri targetStoreUri = getManager().getStoreUri("TSM", RemoteStoreProtocol.TSM);\r
+                       StoreUri escortStoreUri = getManager().getStoreUri(escortStoreName, RemoteStoreProtocol.LOCAL);\r
+                       String outputPath = Paths.get(escortStoreUri.toString(true)).toString();\r
+\r
+                       FtpDirectoryLister lister = (FtpDirectoryLister) sourceStoreUri.getLister();\r
+                       FTPClient client = lister.connect();\r
+                       //a StoreUri alapu listazas nem mukodik, miota a working directory beallitas fix\r
+                       List<RemoteFile> remoteFiles = new ArrayList<>();\r
+                       \r
+                       String[] listNames = client.listNames();\r
+                       for (String f : listNames) {\r
+                               logger.info(getMarker(), "File name is {}", f);\r
+                               \r
+                               try {\r
+                                       RemoteFile remoteFile = lister.get(f);\r
+                                       if (remoteFile == null)\r
+                                               logger.info(getMarker(), "Remote file for {} is not available", f);\r
+                                       else\r
+                                               remoteFiles.add(remoteFile);\r
+                                       \r
+                               } catch (Exception iex){\r
+                                       logger.error(getMarker(), iex.getMessage());\r
+                               }\r
+                       }\r
+                       \r
+                       logger.info(getMarker(), "Found {} files in working dir {}", remoteFiles.size(), client.printWorkingDirectory());\r
+                       processClips(sourceStoreName, sourceStoreUri, targetStoreUri, outputPath, remoteFiles, lastModifiedHours);\r
+\r
+               } catch (Exception e) {\r
+                       logger.error(getMarker(), e.getMessage());\r
+                       logger.error(getSessionMarker(), e.getMessage() == null ? "Unknown error" : e.getMessage());\r
+                       throw e;\r
+               } finally {\r
+                       if (sourceStoreUri != null)\r
+                               sourceStoreUri.cleanUp();\r
+                       setProgress(100);\r
+               }\r
+\r
+               return null;\r
+       }\r
+\r
+       private void processClips(String storeName, StoreUri sourceStoreUri, StoreUri targetStoreUri, String outputPath, List<RemoteFile> remoteFiles,\r
+                       int lastModifiedHours) throws ClipNotFoundException, IOException, ProtocolException {\r
+               int processed = 0;\r
+               for (RemoteFile remoteFile : remoteFiles) {\r
+\r
+                       if (getJobRuntime().isWaitingCancel()) {\r
+                               //logger.info("CANCELED");\r
+                               cancel();\r
+                               break;\r
+                       }\r
+                       try {\r
+                               \r
+                               String title = null;\r
+                               Timestamp created = null;\r
+                               Timestamp modified = null;\r
+                               title = remoteFile.getName();\r
+                               created = remoteFile.getCreated();\r
+                               modified = remoteFile.getModify();\r
+\r
+                               logger.info(getMarker(), "{} with extension {} created {} modified {}", title, remoteFile.getExtension(), created, modified);\r
+                               Media media = getManager().getMedia(title);\r
+                               boolean createEscort = false;\r
+\r
+                               if (!createEscort)\r
+                                       continue;\r
+                               \r
+                               if (checkModifiedByTime(title, modified, lastModifiedHours)) {\r
+                                       createEscort = true;\r
+                                       logger.info(getMarker(), "{} not archived yet", title);\r
+                               }\r
+\r
+                               long mediaId = 0;\r
+                               if (media != null)\r
+                                       mediaId = media.getId();\r
+\r
+                               if (createEscort) {\r
+                                       String fileName = title + DOT_MXF;\r
+                                       DownloadableMedia downloadable = DownloadableMedia.create(title, fileName, modified, created, 0, 0, sourceStoreUri.getId(),\r
+                                                       targetStoreUri.getId(), mediaId);\r
+                                       String escortFileName = storeName + "." + downloadable.getString("fileName");\r
+                                       if (EscortFiles.createMetadataIfNotExists(outputPath, escortFileName, downloadable.toPrettyString(""))) {\r
+                                               logger.info(getMarker(), "Archive status file created for {}", fileName);\r
+                                       } else {\r
+                                               logger.info(getMarker(), "Archive status file already exists for {}", fileName);\r
+                                       }\r
+\r
+                               }\r
+\r
+                               processed++;\r
+                               if (processed % 100 == 0)\r
+                                       logger.info(getMarker(), "Processed {} records", processed);\r
+                       } catch (Exception e) {\r
+                               logger.error(getSessionMarker(), e.getMessage());\r
+                               continue;\r
+                       }\r
+               }\r
+       }\r
+\r
+}\r
diff --git a/server/user.jobengine.osgi.db/migrations/scripts/038_create_pbarchive_store.sql b/server/user.jobengine.osgi.db/migrations/scripts/038_create_pbarchive_store.sql
new file mode 100644 (file)
index 0000000..8eb79b6
--- /dev/null
@@ -0,0 +1,20 @@
+-- // Create VOD store
+-- Migration SQL that makes the change goes here.
+
+INSERT INTO STORE (NAME, ISSYSTEM, ISLOWRES) VALUES ('PBARCHIVE', 'N', 'N')
+@
+
+
+
+-- @DELIMITER ;
+INSERT INTO STOREURI(STOREID,PROTOCOL,DELIVERY,URI,ISSTREAM,ISSOURCE,ISTARGET,USERNAME,PASSWORD,ROOTPATH,PORTNUMBER) VALUES
+((SELECT ID FROM STORE WHERE NAME='PBARCHIVE'),'FTP',null,'isilon.bcplay.local','N','Y','N','mcube','K6NV5aFemp','ARCHIVE',null)
+;
+-- @DELIMITER @
+
+-- //@UNDO
+-- SQL to undo the change goes here.
+DELETE FROM STOREURI WHERE STOREID=(SELECT ID FROM STORE WHERE NAME='PBARCHIVE')
+@
+DELETE FROM STORE WHERE NAME='PBARCHIVE'
+@