<intAttribute key="default_start_level" value="4"/>\r
<setAttribute key="deselected_workspace_bundles"/>\r
<booleanAttribute key="includeOptional" value="false"/>\r
+ <mapAttribute key="org.eclipse.debug.core.environmentVariables">\r
+ <mapEntry key="DSMI_CONFIG" value="${workspace_loc}\-product\production\MEDIAVIVANTIS\settings\dsm.opt"/>\r
+ <mapEntry key="DSMI_DIR" value="c:\apps\TSM\baclient"/>\r
+ <mapEntry key="DSMI_LOG" value="c:\temp"/>\r
+ <mapEntry key="DSM_CONFIG" value="${workspace_loc}\-product\production\MEDIAVIVANTIS\settings\dsm.opt"/>\r
+ <mapEntry key="DSM_DIR" value="c:\apps\TSM\baclient"/>\r
+ <mapEntry key="LD_LIBRARY_PATH" value="c:\apps\TSM\baclient;${workspace_loc}"/>\r
+ <mapEntry key="LIBPATH" value="c:\apps\TSM\baclient"/>\r
+ <mapEntry key="SHLIB_PATH" value="c:\apps\TSM\baclient"/>\r
+ </mapAttribute>\r
<listAttribute key="org.eclipse.debug.ui.favoriteGroups">\r
<listEntry value="org.eclipse.debug.ui.launchGroup.debug"/>\r
<listEntry value="org.eclipse.debug.ui.launchGroup.run"/>\r
{"joblist":[\r
- {\r
- "template": "remote-transcode.xml",\r
- "parameters": [ \r
- {"name": "remoteServer", "value": "http://localhost:9080", "type": "java.lang.String"}, \r
- {"name": "transcodeTemplate", "value": "create-proxy-ffmpeg.xml", "type": "java.lang.String"}, \r
- {"name": "jobName", "value": "Create proxy (slave process)", "type": "java.lang.String"}, \r
- {"name": "profileName", "value": "proxy", "type": "java.lang.String"},\r
- {"name": "tempStoreName", "value": "LOCAL", "type": "java.lang.String"},\r
- {"name": "tempStoreProtocol", "value": "LOCAL", "type": "java.lang.String"}\r
- ]\r
- },\r
- {\r
- "template": "create-proxy-ffmpeg.xml",\r
- "parameters": [ \r
- {"name": "input", "value": "c:/_video/in.mxf", "type": "java.lang.String"}, \r
- {"name": "output", "value": "c:/_video/out.mp4", "type": "java.lang.String"}, \r
- {"name": "profile", "value": "proxy", "type": "java.lang.String"} \r
- ]\r
- },\r
- {\r
- "template": "sync-subtitles.xml",\r
- "parameters": [ \r
- {"name": "sourceFolder", "value": "/windows", "type": "java.lang.String"}, \r
- { \r
- "name": "filter", \r
- "value": {\r
- "fileName": ".*\\.(sh|ini|conf|vhost|xml|php)$"\r
- }, \r
- "type": "com.ibm.nosql.json.api.BasicDBObject"\r
- } \r
- ]\r
- },\r
- {\r
- "template": "test-fork-cancelable.xml",\r
- "parameters": [ {"name": "itemID", "value": 1, "type": "java.lang.Long"} ]\r
- },\r
- {\r
- "active": true,\r
- "executeimmediate": true,\r
- "name" : "Proxy generálás",\r
- "template": "sys-recreate-lowres.xml",\r
- "cronexpression": "0 */1 * * * ?",\r
- "parameters": [ \r
- {"name": "globalRetrievePath", "value": "file://10.11.1.90/data", "type": "java.lang.String"},\r
- {"name": "localRetrievePath", "value": "/mediacube/data", "type": "java.lang.String"},\r
- {"name": "localHiresPath", "value": "/mediacube/data", "type": "java.lang.String" },\r
- {"name": "globalHiresPath", "value": "\\\\10.11.1.90\\data", "type": "java.lang.String" },\r
- {"name": "localLowresPath", "value": "/mediacube/data/TRANSCODER_OUT", "type": "java.lang.String" },\r
- {"name": "transcoderAddress", "value": "http://10.11.1.111:65445/api/json/v1/", "type": "java.lang.String"},\r
- {"name": "transcoderTemplateName", "value": "MP4", "type": "java.lang.String"}\r
- ]\r
- },\r
- {\r
- "active": false,\r
- "executeimmediate": false,\r
- "name" : "Rekurzív archiválás a POLC/FINISHED_SHOWS mappából",\r
- "template": "archive-recursive.xml",\r
- "cronexpression": "0 */1 * * * ?",\r
- "parameters": [ \r
- {"name": "sourcePath", "value": "/mnt/POLC/FINISHED_SHOWS", "type": "java.lang.String"},\r
- {"name": "killDateDays", "value": 1, "type": "java.lang.Integer"},\r
- {"name": "limit", "value": 1, "type": "java.lang.Integer"}\r
- ]\r
- },\r
- {\r
- "active": false,\r
- "executeimmediate": true,\r
- "name" : "Limitált archiválás az PROMISE/ARCHIVE mappából",\r
- "template": "archive-limited.xml",\r
- "cronexpression": "0 */10 * * * ?",\r
- "parameters": [ \r
- {"name": "sourcePath", "value": "/mnt/PROMISE/ARCHIVE", "type": "java.lang.String"},\r
- {"name": "globalSourcePath", "value": "\\\\10.11.1.100\\PROMISE\\ARCHIVE", "type": "java.lang.String"},\r
- {"name": "transcoderTargetPath", "value": "/mnt/PROMISE/TRANSCODER/FFASTRANSCODER/Out", "type": "java.lang.String"},\r
- {"name": "killDateDays", "value": -1, "type": "java.lang.Integer"},\r
- {"name": "limit", "value": 10, "type": "java.lang.Integer"}\r
- ]\r
- },\r
- {\r
- "active": false,\r
- "executeimmediate": false,\r
- "name" : "Párhuzamosított teszt folyamat",\r
- "template": "fake-noparams.xml",\r
- "cronexpression": "0 40 22 * * ?",\r
- "parameters": [ {"name": "itemID", "value": 1, "type": "java.lang.Long"} ]\r
- },\r
- {\r
- "template": "cancelable.xml"\r
- }, \r
- {\r
- "active": false,\r
- "executeimmediate": false,\r
- "name" : "HSM migrálás",\r
- "template": "migrate-hsm.xml",\r
- "cronexpression": "0 0 */1 * * ?",\r
- "parameters": [ \r
- {"name": "sourceLocation", "value": "", "type": "java.lang.String"},\r
- {"name": "targetLocation", "value": "/mediacube/data/ARCHIVE", "type": "java.lang.String"}\r
- ]\r
- },\r
+\r
{\r
"active": false,\r
"executeimmediate": true,\r
- "name" : "Limitált archiválás a /mediacube/data/ARCHIVE mappából",\r
+ "name" : "Limitált archiválás",\r
"template": "archive-limited.xml",\r
"cronexpression": "0 */10 * * * ?",\r
"parameters": [ \r
- {"name": "sourcePath", "value": "/mediacube/data/ARCHIVE", "type": "java.lang.String"},\r
+ {"name": "sourcePath", "value": "/opt/mediacube/ARCHIVE", "type": "java.lang.String"},\r
{"name": "globalSourcePath", "value": "\\\\10.11.1.90\\data\\ARCHIVE", "type": "java.lang.String"},\r
{"name": "transcoderTargetPath", "value": "/mnt/PROMISE/TRANSCODER/FFASTRANSCODER/Out", "type": "java.lang.String"},\r
{"name": "killDateDays", "value": -1, "type": "java.lang.Integer"},\r
{"name": "limit", "value": 1000, "type": "java.lang.Integer"}\r
]\r
},\r
- {\r
- "active": false,\r
- "executeimmediate": true,\r
- "name" : "Lejárt /mediacube/data/ARCHIVE anyagok törlése",\r
- "template": "delete-materials.xml",\r
- "cronexpression": "0 */10 * * * ?",\r
- "parameters": [ \r
- {"name": "sourcePath", "value": "/mediacube/data/ARCHIVE", "type": "java.lang.String"}\r
- ]\r
- }, \r
- {\r
- "active": false,\r
- "executeimmediate": true,\r
- "name" : "Lejárt /mediacube/data/OMARCHIVE anyagok törlése",\r
- "template": "delete-materials.xml",\r
- "cronexpression": "0 */10 * * * ?",\r
- "parameters": [ \r
- {"name": "sourcePath", "value": "/mediacube/data/OMARCHIVE", "type": "java.lang.String"}\r
- ]\r
- }, \r
- {\r
- "active": false,\r
- "executeimmediate": true,\r
- "name" : "Lejárt /PROMISE/ARCHIVE anyagok törlése",\r
- "template": "delete-materials.xml",\r
- "cronexpression": "0 */10 * * * ?",\r
- "parameters": [ \r
- {"name": "sourcePath", "value": "/mnt/PROMISE/ARCHIVE", "type": "java.lang.String"}\r
- ]\r
- }, \r
- {\r
- "active": false,\r
- "executeimmediate": false,\r
- "name" : "Lejárt /POLC/FINISHED_SHOWS anyagok törlése",\r
- "template": "delete-materials.xml",\r
- "cronexpression": "0 * 11 * * ?",\r
- "parameters": [ \r
- {"name": "sourcePath", "value": "/mnt/POLC/FINISHED_SHOWS", "type": "java.lang.String"}\r
- ]\r
- }, \r
{\r
"name" : "SYS: batch-retrieve-ondemand",\r
"template": "batch-retrieve-ondemand.xml",\r
"active": false,\r
"executeimmediate": false\r
- },\r
- {\r
- "name" : "SYS: retrieve-ondemand",\r
- "template": "retrieve-ondemand.xml",\r
- "active": false,\r
- "executeimmediate": false,\r
- "parameters": [ \r
- {"name": "globalRetrievePath", "value": "file://10.11.1.100", "type": "java.lang.String"},\r
- {"name": "localRetrievePath", "value": "/mnt/PROMISE/", "type": "java.lang.String"},\r
- {"name": "materialOutputFolder", "value": "PLAYOUT_NLE", "type": "java.lang.String"},\r
- {"name": "promoOutputFolder", "value": "PROMO_NLE", "type": "java.lang.String"},\r
- {"name": "advertisementOutputFolder", "value": "REKLAM_NLE", "type": "java.lang.String"},\r
- {"name": "octopusOutputFolder", "value": "OCTOPUS", "type": "java.lang.String"},\r
- {"name": "genericOutputFolder", "value": "ARCHIVE_RESTORE", "type": "java.lang.String"},\r
- {"name": "onlineOutputFolder", "value": "ONLINE", "type": "java.lang.String"},\r
- {"name": "killDateDays", "value": 7, "type": "java.lang.Integer"},\r
- {"name": "nexioAgency", "value": "ARCHIVE_RESTORE", "type": "java.lang.String"},\r
- {"name": "nexioPort", "value": 2098, "type": "java.lang.Integer"},\r
- {"name": "nexioUserName", "value": "administrator", "type": "java.lang.String"},\r
- {"name": "nexioPassword", "value": "system", "type": "java.lang.String"}\r
- ]\r
- },\r
- {\r
- "name" : "SYS: common-copy",\r
- "template": "common-copy.xml",\r
- "active": false,\r
- "executeimmediate": false\r
- },\r
- {\r
- "template": "create-lowres-ondemand.xml",\r
- "active": false,\r
- "executeimmediate": false,\r
- "parameters": [ \r
- {"name": "globalRetrievePath", "value": "file://10.11.1.90/data", "type": "java.lang.String"},\r
- {"name": "localRetrievePath", "value": "/mediacube/data", "type": "java.lang.String"},\r
- {"name": "localHiresPath", "value": "/mediacube/data", "type": "java.lang.String" },\r
- {"name": "globalHiresPath", "value": "\\\\10.11.1.90\\data", "type": "java.lang.String" },\r
- {"name": "localLowresPath", "value": "/mediacube/data/TRANSCODER_OUT", "type": "java.lang.String" },\r
- {"name": "transcoderAddress", "value": "http://10.11.1.111:65445/api/json/v1/", "type": "java.lang.String"},\r
- {"name": "transcoderTemplateName", "value": "MP4", "type": "java.lang.String"}\r
- ]\r
- },\r
- {\r
- "active": false,\r
- "executeimmediate": false,\r
- "template": "cancelable.xml",\r
- "parameters": [ \r
- {"name": "param", "value": 1, "type": "java.lang.Integer"}\r
- ]\r
}\r
]}\r
--- /dev/null
+package user.jobengine.server.steps;\r
+\r
+import java.io.File;\r
+import java.io.IOException;\r
+import java.nio.file.DirectoryStream;\r
+import java.nio.file.Files;\r
+import java.nio.file.Path;\r
+import java.nio.file.Paths;\r
+import java.util.LinkedList;\r
+import java.util.List;\r
+\r
+import org.apache.commons.lang.StringUtils;\r
+import org.apache.logging.log4j.LogManager;\r
+import org.apache.logging.log4j.Logger;\r
+import org.apache.logging.log4j.Marker;\r
+\r
+import user.jobengine.server.IJobEngine;\r
+import user.jobengine.server.IJobRuntime;\r
+\r
+/**\r
+ * Az archivalhato mediak listazasa MediaFileWrapper objektumokban. A listazott media allomanyokat megjeloli .catched signal allomannyal, hogy a legkozelebbi\r
+ * listazas figyelmen kivul hagyja.\r
+ *\r
+ * @author robi\r
+ */\r
+public class ArchiveListBuilderStep extends JobStep {\r
+ private static final Logger logger = LogManager.getLogger();\r
+ // private static final String UTF8 = "utf-8";\r
+ private static final String STATUSFOLDER = ".STATUS";\r
+ private static final String JSONEXT = ".json";\r
+ private static final String CATCHEDEXT = ".catched";\r
+\r
+ public static final String ITEM_TITLE = "itemTitle";\r
+ public static final String ITEM_HOUSEID = "itemHouseId";\r
+ public static final String ITEM_DESCRIPTION = "itemDescription";\r
+ public static final String MEDIA_HOUSEID = "mediaHouseId";\r
+ public static final String MEDIA_TITLE = "mediaTitle";\r
+ public static final String MEDIA_DESCRIPTION = "mediaDescription";\r
+ public static final String MEDIA_TYPE = "mediaType";\r
+ private static final String DURATION = "duration";\r
+ private static final String EXISTING_MEDIAID = "existingMediaId";\r
+ private static final String TAGS = "tags";\r
+\r
+ private Marker marker;\r
+\r
+ private ArchiveItem createArchiveItem(Path jsonFilePath, Path mediaFilePath, Path catchedFilePath) {\r
+ ArchiveItem result = null;\r
+ try {\r
+ result = ArchiveItem.fromFile(jsonFilePath);\r
+ result.setMediaFile(mediaFilePath.toString());\r
+ result.setCatchedFile(catchedFilePath.toString());\r
+ } catch (Exception e) {\r
+ logger.catching(e);\r
+ }\r
+\r
+ return result;\r
+ }\r
+\r
+ private void createCatchedFile(Path catchedFilePath) {\r
+ try {\r
+ Files.createFile(catchedFilePath);\r
+ //Files.write(catchedFilePath, CATCHED.getBytes(UTF8), StandardOpenOption.CREATE);\r
+ } catch (Exception e) {\r
+ logger.catching(e);\r
+ }\r
+ }\r
+\r
+ @StepEntry\r
+ public Object[] execute(String sourcePath, int limit, IJobEngine jobEngine, IJobRuntime jobRuntime) {\r
+ marker = jobRuntime.getSessionMarker();\r
+ List<ArchiveItem> archiveList = new LinkedList<ArchiveItem>();\r
+ DirectoryStream<Path> directoryStream = null;\r
+ try {\r
+ DirectoryStream<Path> stream = Files.newDirectoryStream(Paths.get(sourcePath));\r
+ for (Path p : stream) {\r
+ processPathItem(p, archiveList);\r
+ }\r
+ } catch (Exception e) {\r
+ logger.catching(e);\r
+ logger.error(marker, "Az '{}' mappa elérése sikertelen. A rendszer hibaüzenete: {}", e.getMessage());\r
+ } finally {\r
+ if (directoryStream != null) {\r
+ try {\r
+ directoryStream.close();\r
+ } catch (IOException e) {\r
+ }\r
+ }\r
+ }\r
+\r
+ if (limit > 0 && archiveList.size() > limit) {\r
+ archiveList = archiveList.subList(0, limit);\r
+ logger.info(marker, "A folyamat alkalmazza a beállított {} limitet.", limit);\r
+ }\r
+\r
+ if (archiveList.size() == 0)\r
+ logger.info(marker, "Nincs archiválandó anyag.");\r
+ else\r
+ logger.info(marker, "Az archiváló folyamat {} új anyagot érzékelt.", archiveList == null ? 0 : archiveList.size());\r
+\r
+ for (ArchiveItem archiveItem : archiveList) {\r
+ createCatchedFile(Paths.get(archiveItem.getCatchedFile()));\r
+ }\r
+\r
+ return new Object[] { archiveList };\r
+ }\r
+\r
+ private boolean processPathItem(Path mediaFilePath, final List<ArchiveItem> archiveList) {\r
+ File mediaFile = mediaFilePath.toFile();\r
+\r
+ // if (mediaFile.length() > 0)\r
+ // return false;\r
+\r
+ if (mediaFile.isDirectory()) {\r
+ return false;\r
+ }\r
+\r
+ Path dotStorePath = Paths.get(mediaFilePath.getParent().toString(), STATUSFOLDER);\r
+ Path catchedFilePath = Paths.get(dotStorePath.toString(), mediaFile.getName() + CATCHEDEXT);\r
+ File catchedFile = catchedFilePath.toFile();\r
+ if (catchedFile.exists()) {\r
+ logger.warn("{} file is already catched.", mediaFile.getName());\r
+ return false;\r
+ }\r
+\r
+ Path jsonFilePath = Paths.get(dotStorePath.toString(), mediaFile.getName() + JSONEXT);\r
+ File jsonFile = jsonFilePath.toFile();\r
+ if (!jsonFile.exists()) {\r
+ logger.warn("{} has no json metadata.", mediaFile.getName());\r
+ return false;\r
+ }\r
+\r
+ ArchiveItem archiveItem = createArchiveItem(jsonFilePath, mediaFilePath, catchedFilePath);\r
+\r
+ if (archiveItem == null) {\r
+ logger.warn("{} has no metadata specified.", mediaFile.getName());\r
+ return false;\r
+ }\r
+\r
+ if (StringUtils.isBlank(archiveItem.getItemHouseId())) {\r
+ logger.warn("{} has no Item HouseID specified in metadata.", mediaFile.getName());\r
+ return false;\r
+ }\r
+\r
+ if (StringUtils.isBlank(archiveItem.getItemTitle())) {\r
+ logger.warn("{} has no Item Title specified in metadata.", mediaFile.getName());\r
+ return false;\r
+ }\r
+\r
+ if (StringUtils.isBlank(archiveItem.getMediaHouseId())) {\r
+ logger.warn("{} has no Media HouseID specified in metadata.", mediaFile.getName());\r
+ return false;\r
+ }\r
+\r
+ if (StringUtils.isBlank(archiveItem.getMediaTitle())) {\r
+ logger.warn("{} has no Media Title specified in metadata.", mediaFile.getName());\r
+ return false;\r
+ }\r
+\r
+ //A tenyleges archivalast vesszuk elore\r
+ if (mediaFile.length() == 0)\r
+ archiveList.add(archiveItem);\r
+ else\r
+ archiveList.add(0, archiveItem);\r
+ //createCatchedFile(catchedFilePath);\r
+ return true;\r
+ }\r
+}\r
--- /dev/null
+package user.jobengine.server.steps;\r
+\r
+import java.io.File;\r
+import java.util.List;\r
+\r
+import org.apache.logging.log4j.LogManager;\r
+import org.apache.logging.log4j.Logger;\r
+import org.apache.logging.log4j.Marker;\r
+\r
+import user.commons.ListUtils;\r
+import user.jobengine.server.IJobEngine;\r
+import user.jobengine.server.IJobRuntime;\r
+\r
+public class ArchiveMaterialSubmitStep extends JobStep {\r
+ private static final Logger logger = LogManager.getLogger();\r
+ private static final String JOBTEMPLATE = "archive-material.xml";\r
+ private static final String KILL_DATE_DAYS = "killDateDays";\r
+ private static final String ARCHIVE = "Archiválás";\r
+ private static final String ARCHIVE_ITEM = "archiveItem";\r
+ private Marker marker;\r
+\r
+ @StepEntry\r
+ public Object[] execute(List<ArchiveItem> archiveList, int killDateDays, IJobEngine jobEngine, IJobRuntime jobRuntime) throws Exception {\r
+ marker = jobRuntime.getSessionMarker();\r
+ if (archiveList == null || archiveList.size() == 0)\r
+ return null;\r
+\r
+ if (jobRuntime.forkPrepare()) {\r
+ for (int i = 0; i < archiveList.size(); i++) {\r
+ ArchiveItem archiveItem = archiveList.get(i);\r
+ try {\r
+ IJobRuntime runtime = jobEngine.submit(jobRuntime, null, JOBTEMPLATE, ARCHIVE,\r
+ ListUtils.asMap(ARCHIVE_ITEM, archiveItem, KILL_DATE_DAYS, killDateDays));\r
+ int progress = (i + 1) * 100 / archiveList.size();\r
+ setProgress(progress);\r
+ //TODO kivezetni a submit hibaüzenetet\r
+ if (runtime == null)\r
+ throw new Exception("Submit returned null runtime");\r
+ } catch (Exception e) {\r
+ logger.catching(e);\r
+ String fileName = new File(archiveItem.getMediaFile()).getName();\r
+ logger.error(marker, "Az '{}' állomány archiválási kísérlete sikertelen. A rendszer üzenete: {}", fileName, e.getMessage());\r
+ if (!archiveItem.removeCatchedFile())\r
+ logger.error(marker, "Az '{}' állomány .catched jelző állománya nem törölhető.", fileName);\r
+ throw e;\r
+ }\r
+ }\r
+ }\r
+ jobRuntime.forkWaitComplete();\r
+ return null;\r
+ }\r
+\r
+}\r
--- /dev/null
+package user.jobengine.server.steps;\r
+\r
+import java.io.File;\r
+import java.io.IOException;\r
+import java.nio.file.FileVisitResult;\r
+import java.nio.file.FileVisitor;\r
+import java.nio.file.Files;\r
+import java.nio.file.Path;\r
+import java.nio.file.Paths;\r
+import java.nio.file.attribute.BasicFileAttributes;\r
+import java.text.SimpleDateFormat;\r
+import java.util.Arrays;\r
+import java.util.HashSet;\r
+import java.util.List;\r
+import java.util.Map;\r
+import java.util.Set;\r
+\r
+import org.apache.commons.io.FilenameUtils;\r
+import org.apache.commons.lang.StringUtils;\r
+import org.apache.logging.log4j.LogManager;\r
+import org.apache.logging.log4j.Logger;\r
+import org.apache.logging.log4j.message.Message;\r
+import org.apache.logging.log4j.message.ParameterizedMessage;\r
+\r
+import com.ibm.nosql.json.api.BasicDBObject;\r
+\r
+import user.commons.JobStatus;\r
+import user.commons.ListUtils;\r
+import user.commons.MediaCubeMarker;\r
+import user.commons.mediatool.MediaInfo;\r
+import user.jobengine.server.IJobEngine;\r
+import user.jobengine.server.IJobRuntime;\r
+import user.jobengine.server.steps.shared.EscortFiles;\r
+import user.mediacube.metadata.interfaces.IMetadata;\r
+import user.mediacube.metadata.interfaces.IMetadataProvider;\r
+import user.mediacube.metadata.interfaces.IMetadataProviderFactory;\r
+import user.mediacube.metadata.interfaces.MetadataProviderType;\r
+import user.mediacube.metadata.interfaces.MetadataType;\r
+import user.mediacube.metadata.interfaces.PlanAirMetadataListOptions;\r
+\r
+public class ArchiveRecursive extends JobStep implements FileVisitor<Path> {\r
+ private static final Logger logger = LogManager.getLogger();\r
+ private static final String JOBTEMPLATE = "archive-material.xml";\r
+ private static final String ITEM_TITLE = "itemTitle";\r
+ private static final String ITEM_HOUSEID = "itemHouseId";\r
+ private static final String MEDIA_HOUSEID = "mediaHouseId";\r
+ private static final String MEDIA_TITLE = "mediaTitle";\r
+ private static final String MEDIA_DESCRIPTION = "mediaDescription";\r
+ private static final String MEDIA_TYPE = "mediaType";\r
+ private static final String ARCHIVE = "Archiválás";\r
+ private static final String ARCHIVE_ITEM = "archiveItem";\r
+ private static final String KILL_DATE_DAYS = "killDateDays";\r
+ private SimpleDateFormat df = new SimpleDateFormat("yyyy.MM.dd");\r
+\r
+ private List<String> skipPathNames = Arrays.asList("!ARCHIVALAS_ALATT", EscortFiles.STATUSFOLDER,\r
+ EscortFiles.CONFLICTFOLDER);\r
+ private int limit;\r
+ private int submitted;\r
+ private int killDateDays;\r
+ private boolean disableProxy;\r
+\r
+ private boolean canReadMediaInfo(Path mediaFilePath) {\r
+ boolean result = false;\r
+ try {\r
+ MediaInfo mi = new MediaInfo(mediaFilePath);\r
+ mi.process();\r
+ result = true;\r
+ } catch (Exception e) {\r
+ logger.warn(getSessionMarker(), e.getMessage());\r
+ }\r
+ return result;\r
+ }\r
+\r
+ private void checkArchiveItem(ArchiveItem archiveItem) throws Exception {\r
+ if (archiveItem == null)\r
+ throw new Exception("No metadata specified.");\r
+\r
+ if (StringUtils.isBlank(archiveItem.getItemHouseId()))\r
+ throw new Exception("No Item HouseID specified in metadata.");\r
+\r
+ if (StringUtils.isBlank(archiveItem.getItemTitle()))\r
+ throw new Exception("No Item Title specified in metadata.");\r
+\r
+ if (StringUtils.isBlank(archiveItem.getMediaHouseId()))\r
+ throw new Exception("No Media HouseID specified in metadata.");\r
+\r
+ if (StringUtils.isBlank(archiveItem.getMediaTitle()))\r
+ throw new Exception("No Media Title specified in metadata.");\r
+ }\r
+\r
+ private String archiveItemJSON(ArchiveItem result) {\r
+ BasicDBObject obj = new BasicDBObject();\r
+ obj.put("itemHouseId", result.getItemHouseId());\r
+ obj.put("itemTitle", result.getItemTitle());\r
+ obj.put("mediaHouseId", result.getMediaHouseId());\r
+ obj.put("mediaTitle", result.getMediaTitle());\r
+ obj.put("mediaDescription", result.getMediaDescription());\r
+ obj.put("mediaType", result.getMediaType());\r
+ return obj.toPrettyString("");\r
+ }\r
+\r
+ private ArchiveItem createArchiveItem(Path filePath) throws Exception {\r
+ ArchiveItem result = null;\r
+\r
+ String fileName = filePath.getFileName().toString();\r
+ String mediaHouseId = FilenameUtils.removeExtension(fileName);\r
+ try {\r
+ result = getPlanAirMetadata(mediaHouseId);\r
+\r
+ if (result != null) {\r
+ result.setMediaFile(filePath.toString());\r
+ // 210617 proxy keszites tiltasa\r
+ result.setDisableProxy(disableProxy);\r
+ }\r
+\r
+ logger.info(getSessionMarker(), "PlanAir query done for {}", filePath);\r
+ } catch (Exception e) {\r
+ logger.error(getSessionMarker(), "PlanAir metadata error", e);\r
+ // nem latja a drivert pl.\r
+ throw e;\r
+ }\r
+\r
+//210616 Ha nincs metaadat, nem archivalunk\r
+ if (result == null) {\r
+ result = new ArchiveItem();\r
+ BasicFileAttributes attr = Files.readAttributes(filePath, BasicFileAttributes.class);\r
+ result.setItemHouseId(df.format(attr.lastModifiedTime().toMillis()));\r
+ result.setItemTitle(filePath.getParent().toString());\r
+ result.setMediaHouseId(mediaHouseId);\r
+ result.setMediaTitle(fileName);\r
+ result.setMediaDescription("/ARCHIVE-TEST");\r
+ result.setMediaType("Generic");\r
+ result.setMediaFile(filePath.toString());\r
+ }\r
+\r
+ return result;\r
+ }\r
+\r
+ @StepEntry\r
+ public Object[] execute(String sourcePath, int killDateDays, int limit, boolean disableProxy) throws Exception {\r
+ this.killDateDays = killDateDays;\r
+ this.limit = limit;\r
+ this.disableProxy = disableProxy;\r
+ logger.info(getSessionMarker(), "Starting in {}", sourcePath);\r
+ try {\r
+ if (getJobRuntime().forkPrepare()) {\r
+ Files.walkFileTree(Paths.get(sourcePath), this);\r
+ }\r
+ } catch (Exception e) {\r
+ logger.error(getSessionMarker(), "Az '{}' mappa elérése sikertelen. A rendszer hibaüzenete: {}", sourcePath,\r
+ e.getMessage());\r
+ } finally {\r
+ if (submitted > 0)\r
+ getJobRuntime().forkWaitComplete();\r
+ else\r
+ getJobRuntime().cancelForkPrepare();\r
+ }\r
+ return null;\r
+ }\r
+\r
+ private ArchiveItem getPlanAirMetadata(String mediaHouseId) throws Exception {\r
+ PlanAirMetadataListOptions opt = new PlanAirMetadataListOptions();\r
+ opt.setSearch(mediaHouseId);\r
+ opt.setType(MetadataType.Material);\r
+\r
+ BasicDBObject json = null;\r
+ List<IMetadata> data = null;\r
+\r
+ IMetadataProviderFactory factory = getService(IMetadataProviderFactory.class);\r
+ if (factory == null)\r
+ logger.info(getSessionMarker(), "IMetadataProviderFactory is null");\r
+\r
+ IMetadataProvider planairProvider = factory.getProvider(MetadataProviderType.PLANAIR);\r
+ if (planairProvider == null)\r
+ logger.info(getSessionMarker(), "IMetadataProvider is null");\r
+\r
+ ArchiveItem result = null;\r
+ data = planairProvider.list(opt);\r
+ if (data.size() != 0)\r
+ json = data.get(0).asJSON();\r
+ else {\r
+ opt.setType(MetadataType.Promo);\r
+ data = planairProvider.list(opt);\r
+ if (data.size() != 0)\r
+ json = data.get(0).asJSON();\r
+ else {\r
+ opt.setType(MetadataType.AD);\r
+ data = planairProvider.list(opt);\r
+ if (data.size() != 0)\r
+ json = data.get(0).asJSON();\r
+ }\r
+ }\r
+ if (json != null) {\r
+ result = new ArchiveItem();\r
+ result.setItemHouseId(json.getString(ITEM_HOUSEID));\r
+ result.setItemTitle(json.getString(ITEM_TITLE));\r
+ result.setMediaHouseId(json.getString(MEDIA_HOUSEID));\r
+ result.setMediaTitle(json.getString(MEDIA_TITLE));\r
+ result.setMediaDescription(json.getString(MEDIA_DESCRIPTION));\r
+ result.setMediaType(json.getString(MEDIA_TYPE));\r
+ }\r
+ return result;\r
+ }\r
+\r
+ private boolean handleArchiveConflict(Path mediaPath) throws Exception {\r
+ boolean result = false;\r
+ String sourceFileName = mediaPath.getFileName().toString();\r
+ if (getManager().isMediaFileExists(sourceFileName)) {\r
+ EscortFiles.createMediaCatch(mediaPath);\r
+ result = true;\r
+ }\r
+ return result;\r
+ }\r
+\r
+ @Override\r
+ public FileVisitResult postVisitDirectory(Path paramT, IOException paramIOException) throws IOException {\r
+ return FileVisitResult.CONTINUE;\r
+ }\r
+\r
+ @Override\r
+ public FileVisitResult preVisitDirectory(Path dir, BasicFileAttributes paramBasicFileAttributes)\r
+ throws IOException {\r
+ Path dirName = dir.getFileName();\r
+\r
+ if (skipPathNames.contains(dirName.toString())) {\r
+ logger.info(getSessionMarker(), "PreVisit skip {}", dir);\r
+ return FileVisitResult.SKIP_SUBTREE;\r
+ } else\r
+ logger.info(getSessionMarker(), "PreVisit {}", dir);\r
+\r
+ return FileVisitResult.CONTINUE;\r
+ }\r
+\r
+ private boolean processPathItem(Path mediaPath) throws Exception {\r
+ if (limit != 0 && submitted == limit) {\r
+ logger.info(getSessionMarker(), "Limit reached {}, canceling", limit);\r
+ return false;\r
+ }\r
+\r
+ String fileName = mediaPath.getFileName().toString();\r
+ if (fileName.startsWith(".") || fileName.endsWith(".nomd"))\r
+ return false;\r
+\r
+ logger.info(getSessionMarker(), "Processing {}", mediaPath);\r
+\r
+ File mediaFile = mediaPath.toFile(); \r
+ if (mediaFile.isDirectory()) {\r
+ //logger.info(getSessionMarker(), "Skipping directory {}", mediaPath);\r
+ return false;\r
+ }\r
+\r
+ if (EscortFiles.isMediaCatched(mediaPath)) {\r
+ //logger.info(getSessionMarker(), "Skipping already catched {}", mediaPath);\r
+ return false;\r
+ }\r
+\r
+ Path nomdFile = Paths.get(mediaPath.toString() + ".nomd");\r
+\r
+ if (nomdFile.toFile().exists()) {\r
+ //logger.info(getSessionMarker(), "Skipping nomd file exists {}", nomdFile);\r
+ return false;\r
+ }\r
+\r
+ if (handleArchiveConflict(mediaPath)) {\r
+ logger.info(getSessionMarker(), "Skipping archive db already contains {}", mediaPath);\r
+ return false;\r
+ }\r
+\r
+ if (!canReadMediaInfo(mediaPath)) {\r
+ logger.info(getSessionMarker(), "Skipping cant read mediainfo {}", mediaPath);\r
+ return false;\r
+ }\r
+\r
+ ArchiveItem archiveItem = createArchiveItem(mediaPath);\r
+\r
+ if (archiveItem == null) {\r
+ Message msg = new ParameterizedMessage("Nincs metaadat!");\r
+ logger.info(new MediaCubeMarker("vasary@elgekko.net,muszak@mediavivantis.hu",\r
+ "Értesítés problémás " + mediaPath.getFileName().toString() + " archiválásról"), msg);\r
+ Files.createFile(nomdFile);\r
+ return false;\r
+ }\r
+\r
+ try {\r
+ checkArchiveItem(archiveItem);\r
+\r
+ Map<String, Object> parameters = ListUtils.asMap(ARCHIVE_ITEM, archiveItem, KILL_DATE_DAYS, killDateDays);\r
+ IJobRuntime runtime = getEngine().submit(getJobRuntime(), e -> {\r
+ if (e.getStatus().equals(JobStatus.CANCELED) || e.getStatus().equals(JobStatus.SUSPENDED))\r
+ EscortFiles.removeMediaCatch(mediaPath);\r
+ }, JOBTEMPLATE, ARCHIVE, 1, IJobEngine.DEFAULT_OWNER, parameters);\r
+ if (runtime == null)\r
+ throw new Exception("Submit returned null runtime");\r
+ runtime.setRelated(mediaPath.toString());\r
+ EscortFiles.createMediaCatch(mediaPath);\r
+ String metadata = archiveItemJSON(archiveItem);\r
+ EscortFiles.createMetadata(mediaPath.getParent().toString(), mediaPath.getFileName().toString(), metadata);\r
+ submitted++;\r
+ } catch (Exception e) {\r
+ logger.error(getSessionMarker(),\r
+ "Az '{}' állomány archiválási kísérlete sikertelen. A rendszer üzenete: {}", mediaPath,\r
+ e.getMessage());\r
+ }\r
+\r
+ return true;\r
+ }\r
+\r
+ @Override\r
+ public FileVisitResult visitFile(Path filePath, BasicFileAttributes paramBasicFileAttributes) throws IOException {\r
+ logger.info(getSessionMarker(), "Will checked {}", filePath);\r
+ try {\r
+ processPathItem(filePath);\r
+ } catch (Exception e) {\r
+ logger.catching(e);\r
+ }\r
+ return FileVisitResult.CONTINUE;\r
+ }\r
+\r
+ @Override\r
+ public FileVisitResult visitFileFailed(Path filePath, IOException paramIOException) throws IOException {\r
+ logger.info("Error archive {}", filePath);\r
+ return FileVisitResult.CONTINUE;\r
+ }\r
+}\r
--- /dev/null
+package user.jobengine.server.steps;\r
+\r
+import java.util.List;\r
+import java.util.Map;\r
+\r
+import org.apache.logging.log4j.LogManager;\r
+import org.apache.logging.log4j.Logger;\r
+\r
+import user.commons.MediaCubeFinishMarker;\r
+import user.commons.MediaCubeMarker;\r
+import user.jobengine.db.ArchivedMedia;\r
+import user.jobengine.server.IJobEngine;\r
+import user.jobengine.server.IJobRuntime;\r
+import user.jobengine.server.scheduler.ScheduledJob;\r
+\r
+public class BatchRetrieveForkStep extends JobStep {\r
+ private static final String TARGET_PATH_TYPE = "targetPathType";\r
+ private static final Logger logger = LogManager.getLogger();\r
+ private static final String CHILD_TEMPLATE = "retrieve-ondemand.xml";\r
+ private static final String ARCHIVEDMEDIA = "archivedMedia";\r
+ private static final String RECIPIENT = "successRecipient";\r
+ private static final String HOUSEID = "houseId";\r
+ private MediaCubeMarker marker;\r
+\r
+ @StepEntry\r
+ public Object[] execute(List<ArchivedMedia> basket, String houseId, String recipient, String targetPathType, IJobEngine jobEngine, IJobRuntime jobRuntime)\r
+ throws Exception {\r
+ marker = (MediaCubeMarker) jobRuntime.getSessionMarker();\r
+\r
+ //session szinten csak a finishMarker cimzettje az erdekes, es ezt a cimet pluszban hasznalja a konfigban megadott cimmel\r
+ //a finishMarker orokli a cim bellitast a sessionMarkertol\r
+ marker.setTo(recipient);\r
+\r
+ ((MediaCubeMarker) jobRuntime.getFinishMarker()).setTo(recipient);\r
+\r
+ if (basket == null || basket.size() == 0)\r
+ return null;\r
+ setProgress(10);\r
+\r
+ MediaCubeMarker mailMarker = new MediaCubeMarker(recipient);\r
+ mailMarker.setSessionName("Archívum viszatöltés");\r
+ mailMarker.setSessionID(houseId);\r
+ logger.info(mailMarker, "A visszatöltések elindultak az alábbi állományokra:");\r
+\r
+ if (jobRuntime.forkPrepare()) {\r
+ for (ArchivedMedia archivedMedia : basket) {\r
+ logger.info(mailMarker, archivedMedia.getMedia().getMediaFilesName());\r
+ submit(archivedMedia, recipient, houseId, targetPathType, jobEngine, jobRuntime);\r
+ }\r
+ }\r
+ setProgress(50);\r
+ logger.info(new MediaCubeFinishMarker(mailMarker), "A visszatöltések végeztével megerősítő üzenetet küldünk.");\r
+ jobRuntime.forkWaitComplete();\r
+ setProgress(100);\r
+ return null;\r
+ }\r
+\r
+ public void submit(ArchivedMedia archivedMedia, String recipient, String houseId, String targetPathType, IJobEngine jobEngine, IJobRuntime jobRuntime)\r
+ throws Exception {\r
+ try {\r
+ ScheduledJob scheduledJob = jobEngine.getScheduledJob(CHILD_TEMPLATE);\r
+ Map<String, Object> parameters = scheduledJob.getJobParameters();\r
+ parameters.put(ARCHIVEDMEDIA, archivedMedia);\r
+ parameters.put(HOUSEID, houseId);\r
+ parameters.put(RECIPIENT, recipient);\r
+ parameters.put(TARGET_PATH_TYPE, targetPathType);\r
+ IJobRuntime child = jobEngine.submit(jobRuntime, null, CHILD_TEMPLATE, String.format("Visszatöltés %s részére", recipient), parameters);\r
+ ((MediaCubeMarker) child.getSessionMarker()).setTo(recipient);\r
+ } catch (Exception e) {\r
+ logger.catching(e);\r
+ logger.error(marker, "Hiba a kötegelt visszatöltésben. A rendszer üzenete: {}", e.getMessage());\r
+ }\r
+\r
+ }\r
+\r
+}\r
--- /dev/null
+package user.jobengine.server.steps;\r
+\r
+import java.io.File;\r
+import java.io.IOException;\r
+import java.nio.file.DirectoryStream;\r
+import java.nio.file.FileVisitResult;\r
+import java.nio.file.FileVisitor;\r
+import java.nio.file.Files;\r
+import java.nio.file.Path;\r
+import java.nio.file.Paths;\r
+import java.nio.file.SimpleFileVisitor;\r
+import java.nio.file.attribute.BasicFileAttributes;\r
+import java.text.ParseException;\r
+import java.text.SimpleDateFormat;\r
+import java.util.ArrayList;\r
+import java.util.Collections;\r
+import java.util.Date;\r
+import java.util.List;\r
+\r
+import org.apache.commons.io.FileUtils;\r
+import org.apache.commons.lang.StringUtils;\r
+import org.apache.logging.log4j.LogManager;\r
+import org.apache.logging.log4j.Logger;\r
+import org.apache.logging.log4j.Marker;\r
+\r
+import user.jobengine.server.steps.shared.ItemManagerExtensions;\r
+\r
+public class CleanupMountedLocationStep extends JobStep implements FileVisitor<Path> {\r
+\r
+ private static final Logger logger = LogManager.getLogger();\r
+ private static final String PROJECTFOLDER = "PROJECT";\r
+ private static final String DATEFORMAT = "yyyyMMdd";\r
+ private static final String DOT = ".";\r
+ private static final String STATUSFOLDER = ".STATUS";\r
+ private static final String EWC2EXT = ".ewc2";\r
+ private static final String XMPEXT = ".xmp";\r
+ private static final String CATCHEDEXT = ".catched";\r
+ private static final String KILLDATEEXT = ".killdate";\r
+ private static final String JSONEXT = ".json";\r
+\r
+ private static boolean isEmpty(final Path directory) throws IOException {\r
+ try (DirectoryStream<Path> dirStream = Files.newDirectoryStream(directory)) {\r
+ final int[] count = new int[] { 0 };\r
+ final int[] specialCount = new int[] { 0 };\r
+ dirStream.forEach(p -> {\r
+ count[0]++;\r
+ // if (p.getFileName().toString().toLowerCase().equals(PROJECTFOLDER.toLowerCase()))\r
+ // specialCount[0]++;\r
+ if (p.getFileName().toString().toLowerCase().equals(STATUSFOLDER.toLowerCase()))\r
+ specialCount[0]++;\r
+\r
+ });\r
+ if (specialCount[0] == count[0])\r
+ return true;\r
+ }\r
+ return false;\r
+ }\r
+\r
+ private Marker marker;\r
+\r
+ final int[] allCount = new int[] { 0 };\r
+ final int[] currentCount = new int[] { 0 };\r
+\r
+ private Path sourcePath;\r
+ private SimpleDateFormat dateFormat;\r
+ private boolean skipArchiveCheck;\r
+\r
+ private Date checkExpiration(List<Path> killDateFiles) {\r
+ Date killDate = null;\r
+ for (Path killDateFile : killDateFiles) {\r
+ Date currentKillDate = getKillDate(killDateFile);\r
+ if (currentKillDate == null)\r
+ continue;\r
+ if ((killDate != null && currentKillDate.after(killDate)) || killDate == null)\r
+ killDate = currentKillDate;\r
+ }\r
+ return new Date().after(killDate) ? killDate : null;\r
+ }\r
+\r
+ @StepEntry\r
+ public Object[] execute(String sourceFolder, boolean skipArchiveCheck) throws Exception {\r
+ this.skipArchiveCheck = skipArchiveCheck;\r
+ marker = getSessionMarker();\r
+ sourcePath = Paths.get(sourceFolder);\r
+ DirectoryStream<Path> directoryStream = null;\r
+ if (StringUtils.isBlank(sourcePath.toString())) {\r
+ logger.error(marker, "A folyamat 'sourcePath' bemeneti paramétere üres.");\r
+ throw new NullPointerException("System is not configured properly, 'sourceFolder' input parameter missing.");\r
+ }\r
+\r
+ if (!sourcePath.toFile().exists() || !sourcePath.toFile().isDirectory()) {\r
+ logger.error(marker, "A {} mappa nem létezik.", sourceFolder);\r
+ throw new NullPointerException(String.format("Directory %s not exists.", sourceFolder));\r
+ }\r
+\r
+ try {\r
+ setProgress(1);\r
+ dateFormat = new SimpleDateFormat(DATEFORMAT);\r
+\r
+ Files.walkFileTree(sourcePath, new SimpleFileVisitor<Path>() {\r
+ @Override\r
+ public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) throws IOException {\r
+ allCount[0]++;\r
+ return super.visitFile(file, attrs);\r
+ }\r
+ });\r
+ Files.walkFileTree(sourcePath, this);\r
+ } catch (Exception e) {\r
+ logger.catching(e);\r
+ logger.error(marker, "Hiba a '{}' mappa feldolgozásában. A rendszer hibaüzenete: {}", sourcePath, e.getMessage());\r
+ throw e;\r
+ } finally {\r
+ if (directoryStream != null) {\r
+ try {\r
+ directoryStream.close();\r
+ } catch (IOException e) {\r
+ }\r
+ }\r
+ }\r
+ return null;\r
+ }\r
+\r
+ private Date getKillDate(Path killDateFile) {\r
+ String fileName = killDateFile.getFileName().toString();\r
+ int end = fileName.lastIndexOf(DOT);\r
+ if (end < 1)\r
+ return null;\r
+ int start = fileName.lastIndexOf(DOT, end - 1);\r
+ if (start < 0)\r
+ return null;\r
+ String strKillDate = fileName.substring(start + 1, end);\r
+ Date result = null;\r
+ if (StringUtils.isNumeric(strKillDate)) {\r
+ try {\r
+ result = dateFormat.parse(strKillDate);\r
+ } catch (ParseException e) {\r
+ logger.error(marker, "A {} fájl 'killdate' állománya hibás formátumú, a {} karaktersorozat nem konvertálható dátummá.", fileName, strKillDate);\r
+ return null;\r
+ }\r
+ } else\r
+ logger.error(marker, "A {} fájl 'killdate' állománya hibás formátumú, az dátum helyett ez áll: '{}'.", fileName, strKillDate);\r
+ return result;\r
+ }\r
+\r
+ private List<Path> getKillDateFiles(Path filePath) {\r
+ String killDateFilePattern = String.format("%s.*%s", filePath.getFileName().toString(), KILLDATEEXT);\r
+ List<Path> result = new ArrayList<>();\r
+ Path statusPath = null;\r
+ try {\r
+ statusPath = Paths.get(filePath.getParent().toString(), STATUSFOLDER);\r
+ } catch (Exception e) {\r
+ logger.catching(e);\r
+ return null;\r
+ }\r
+ File statusPathFile = statusPath.toFile();\r
+ if (statusPathFile.exists() && statusPathFile.isDirectory()) {\r
+ try (DirectoryStream<Path> stream = Files.newDirectoryStream(statusPath, killDateFilePattern)) {\r
+ stream.forEach(p -> result.add(p));\r
+ } catch (Exception e) {\r
+ logger.catching(e);\r
+ }\r
+ }\r
+ Collections.sort(result);\r
+ return result;\r
+ }\r
+\r
+ @Override\r
+ public FileVisitResult postVisitDirectory(Path dir, IOException exc) throws IOException {\r
+ if (!dir.equals(sourcePath) && isEmpty(dir)) {\r
+ if (!removeExistingSpecialDirectory(dir, PROJECTFOLDER))\r
+ return FileVisitResult.CONTINUE;\r
+ if (!removeExistingSpecialDirectory(dir, STATUSFOLDER))\r
+ return FileVisitResult.CONTINUE;\r
+ if (removeFile(dir))\r
+ logger.info(marker, "A {} üres mappa törlése sikeres.", dir);\r
+\r
+ }\r
+ return FileVisitResult.CONTINUE;\r
+ }\r
+\r
+ @Override\r
+ public FileVisitResult preVisitDirectory(Path dir, BasicFileAttributes attrs) throws IOException {\r
+\r
+ //A .-al kezdodo mappakat kihagyjuk\r
+ if (dir.getFileName().toString().startsWith("."))\r
+ return FileVisitResult.SKIP_SUBTREE;\r
+\r
+ return FileVisitResult.CONTINUE;\r
+ }\r
+\r
+ private void processPathItem(Path filePath) {\r
+ currentCount[0]++;\r
+\r
+ if (filePath.getFileName().toString().startsWith("."))\r
+ return;\r
+\r
+ int progress = currentCount[0] * 100 / allCount[0];\r
+ setProgress(progress);\r
+\r
+ logger.info("Checking {}", filePath);\r
+ List<Path> killDateFiles = getKillDateFiles(filePath);\r
+ if (killDateFiles == null || killDateFiles.size() == 0) {\r
+ logger.warn(marker, "A {} fájlhoz nem található 'killdate' állomány.", filePath);\r
+ return;\r
+ }\r
+\r
+ if (killDateFiles.size() != 1)\r
+ logger.warn(marker, "A {} fájlhoz több 'killdate' állomány található, a legújabb dátum határozza meg a törlés időpontját.", filePath);\r
+\r
+ Date killDate = checkExpiration(killDateFiles);\r
+ if (killDate == null)\r
+ return;\r
+\r
+ if (!skipArchiveCheck && filePath.toFile().length() > 0) {\r
+ if (!ItemManagerExtensions.isArchived(getManager(), filePath)) {\r
+ logger.error(marker, "A(z) {} anyag törlésre van kijelölve, de nem található az archívumban.", filePath);\r
+ return;\r
+ }\r
+ }\r
+\r
+ if (removeFiles(filePath, killDateFiles))\r
+ logger.info(marker, "A {} fájl és kapcsolódó állományai a {} killdate bejegyzés alapján sikeresen törlődtek.", filePath.getFileName(),\r
+ dateFormat.format(killDate));\r
+ else\r
+ logger.warn(marker, "A {} fájl és kapcsolódó állományai a {} killdate bejegyzés alapján csak részlegesen vagy egyáltalán nem törlődtek.",\r
+ filePath.getFileName(), dateFormat.format(killDate));\r
+ }\r
+\r
+ private boolean removeExistingSpecialDirectory(Path dir, String folderName) throws IOException {\r
+ File projectPath = Paths.get(dir.toString(), folderName).toFile();\r
+ if (projectPath.exists() && projectPath.isDirectory()) {\r
+ FileUtils.deleteDirectory(projectPath);\r
+ if (projectPath.exists()) {\r
+ logger.warn(marker, "A {} alatti {} mappa törlése nem sikerült.", dir, folderName);\r
+ return false;\r
+ }\r
+ }\r
+ return true;\r
+ }\r
+\r
+ private boolean removeFile(Path filePath) {\r
+ boolean result = false;\r
+ try {\r
+ //logger.error("REMOVE {}", filePath);\r
+ File file = filePath.toFile();\r
+ if (file.exists())\r
+ result = file.delete();\r
+ } catch (Exception e) {\r
+ logger.error(marker, "A {} fájl nem törölhető. A rendszer hibaüzenete: {}", filePath, e.getMessage());\r
+ }\r
+ return result;\r
+ }\r
+\r
+ private boolean removeFiles(Path filePath, List<Path> killDateFiles) {\r
+ if (!removeFile(filePath))\r
+ return false;\r
+\r
+ removeFile(Paths.get(filePath.toString() + EWC2EXT));\r
+ removeFile(Paths.get(filePath.toString() + XMPEXT));\r
+ removeFile(Paths.get(filePath.getParent().toString(), STATUSFOLDER, filePath.getFileName().toString() + CATCHEDEXT));\r
+ removeFile(Paths.get(filePath.getParent().toString(), STATUSFOLDER, filePath.getFileName().toString() + JSONEXT));\r
+\r
+ boolean result = true;\r
+ for (Path killDateFile : killDateFiles) {\r
+ if (!removeFile(killDateFile))\r
+ result = false;\r
+ }\r
+\r
+ return result;\r
+ }\r
+\r
+ @Override\r
+ public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) throws IOException {\r
+ processPathItem(file);\r
+ return FileVisitResult.CONTINUE;\r
+ }\r
+\r
+ @Override\r
+ public FileVisitResult visitFileFailed(Path file, IOException e) throws IOException {\r
+ logger.error(marker, "A {} fájl nem érhető el. A rendszer hibaüzenete: {}", file.toString(), e.getMessage());\r
+ return FileVisitResult.CONTINUE;\r
+ }\r
+}\r
--- /dev/null
+package user.jobengine.server.steps;\r
+\r
+import java.nio.file.Paths;\r
+\r
+import org.apache.logging.log4j.LogManager;\r
+import org.apache.logging.log4j.Logger;\r
+\r
+import com.ibm.nosql.json.api.BasicDBObject;\r
+import com.ibm.nosql.json.api.DB;\r
+import com.ibm.nosql.json.api.DBCollection;\r
+\r
+import user.commons.nosql.NoSQLUtils;\r
+import user.jobengine.db.Media;\r
+import user.jobengine.db.Store;\r
+\r
+public class CreateArchiveItemStep extends JobStep {\r
+ private static final Logger logger = LogManager.getLogger();\r
+\r
+ @StepEntry\r
+ public Object[] execute(Media mediaCubeMedia, String localHiresPath) throws Exception {\r
+ DB db = NoSQLUtils.getNoSQLDB();\r
+ DBCollection collection = db.getCollection("missing_lowres");\r
+ Store highResStore = getManager().getSystemStore(false);\r
+\r
+ ArchiveItem archiveItem = null;\r
+ try {\r
+ if (mediaCubeMedia.getMediaFilesCount() != 1)\r
+ throw new Exception("Expected media count is 1, found " + mediaCubeMedia.getMediaFilesCount());\r
+ if (mediaCubeMedia.getMediaFiles().get(0).getStoreId() != highResStore.getId())\r
+ throw new Exception("Expected media store is a high-res store");\r
+\r
+ String name = mediaCubeMedia.getMediaFileRealName();\r
+ archiveItem = new ArchiveItem();\r
+ archiveItem.setMediaFile(Paths.get(localHiresPath, name).toString());\r
+ collection.save(new BasicDBObject("name", name));\r
+ } catch (Exception e) {\r
+ logger.catching(e);\r
+ logger.info(getMarker(), e.getMessage());\r
+ throw e;\r
+ } finally {\r
+ setProgress(100);\r
+ }\r
+ return new Object[] { archiveItem };\r
+ }\r
+}\r
--- /dev/null
+package user.jobengine.server.steps;\r
+\r
+import java.nio.file.Paths;\r
+\r
+import org.apache.logging.log4j.LogManager;\r
+import org.apache.logging.log4j.Logger;\r
+\r
+import com.ibm.nosql.json.api.BasicDBObject;\r
+import com.ibm.nosql.json.api.DB;\r
+import com.ibm.nosql.json.api.DBCollection;\r
+\r
+import user.commons.MediaCubeUndoMarker;\r
+import user.commons.nosql.NoSQLUtils;\r
+import user.jobengine.db.IItemManager;\r
+import user.jobengine.db.Media;\r
+import user.jobengine.db.MediaFile;\r
+import user.jobengine.server.IJobEngine;\r
+import user.jobengine.server.IJobRuntime;\r
+\r
+public class CreateMissingLowresStep extends JobStep {\r
+ private static final Logger logger = LogManager.getLogger();\r
+\r
+ @StepEntry\r
+ public Object[] execute(String localHiresPath, IJobEngine jobEngine, IJobRuntime jobRuntime) throws Exception {\r
+ Object[] result = new Object[] { null, null, "%s", null, 0, true };\r
+\r
+ DB db = NoSQLUtils.getNoSQLDB();\r
+ DBCollection collection = db.getCollection("missing_lowres");\r
+ IItemManager manager = jobEngine.getItemManager();\r
+ Media media = getFirstUntranscodedMedia(manager, collection);\r
+\r
+ try {\r
+ if (media == null) {\r
+ logger.info(new MediaCubeUndoMarker(getSessionMarker().getSessionID()), "Nincs feldolgozandó hiány.");\r
+ // throw new Exception("Nincs feldolgozandó hiány.");\r
+ cancel();\r
+ return null;\r
+ }\r
+\r
+ String name = media.getMediaFileRealName();\r
+ result[0] = media;\r
+ ArchiveItem archiveItem = new ArchiveItem();\r
+ archiveItem.setMediaFile(Paths.get(localHiresPath, name).toString());\r
+ result[1] = archiveItem;\r
+ collection.save(new BasicDBObject("name", name));\r
+ logger.info(getSessionMarker(), "Processing mediaID: {}", media.getId());\r
+\r
+ } catch (Exception e) {\r
+ logger.catching(e);\r
+ logger.error(getSessionMarker(), e.getMessage());\r
+ throw e;\r
+ } finally {\r
+ setProgress(100);\r
+ }\r
+ return result;\r
+ }\r
+\r
+ private Media getFirstUntranscodedMedia(IItemManager manager, DBCollection collection) {\r
+ Media[] result = new Media[] { null };\r
+ // MV\r
+ String query = "SELECT mediaid FROM VW_MISSING_PROXY_IDS WHERE HOUSEID like 'M%' or HOUSEID like 'P%' or HOUSEID like 'R%' ORDER BY modified DESC";\r
+\r
+ // HTV\r
+ // String query = "SELECT mediaid FROM VW_MISSING_PROXY_IDS";\r
+ manager.executeQuery(query, rs -> {\r
+ try {\r
+ long mediaId = rs.getLong(1);\r
+ Media media = manager.getMedia(mediaId);\r
+ // a nevgeneralas miatt az eredeti MediaFilesName nem jo, a pontos nev kell\r
+ // nekunk\r
+ String name = media.getMediaFileRealName();\r
+ //logger.info(getSessionMarker(), "Checking {}", name);\r
+ long existing = collection.find(new BasicDBObject("name", name)).count();\r
+ if (existing > 0) {\r
+ // logger.info(getSessionMarker(), "{} is on missing_lowres list", name);\r
+ return true;\r
+ }\r
+\r
+ // 210617 proxy keszites tiltasa\r
+ MediaFile mf = manager.getSystemMediaFile(media);\r
+ if (mf.isDisableProxy()) {\r
+ logger.info(getSessionMarker(), "Proxy disabled {}", name);\r
+ \r
+ return true;\r
+ }\r
+\r
+ result[0] = media;\r
+ } catch (Exception e) {\r
+ logger.error(e);\r
+ }\r
+ return false;\r
+ }, null);\r
+ return result[0];\r
+ }\r
+}\r
--- /dev/null
+package user.jobengine.server.steps;\r
+\r
+import org.apache.logging.log4j.LogManager;\r
+import org.apache.logging.log4j.Logger;\r
+\r
+import user.commons.RemoteFile;\r
+import user.commons.StoreUri;\r
+\r
+public class DeleteFile extends JobStep {\r
+ private static final Logger logger = LogManager.getLogger();\r
+\r
+ @StepEntry\r
+ public Object[] execute(String fileName, StoreUri tempStoreUri) throws Exception {\r
+ try {\r
+ RemoteFile remoteFile = tempStoreUri.getRemoteFile(fileName);\r
+ tempStoreUri.delete(remoteFile);\r
+ } catch (Exception e) {\r
+ logger.warn(getJobRuntime().getSessionMarker(), e.getMessage());\r
+ }\r
+ return null;\r
+ }\r
+}\r
--- /dev/null
+package user.jobengine.server.steps;\r
+\r
+import org.apache.logging.log4j.LogManager;\r
+import org.apache.logging.log4j.Logger;\r
+import org.apache.logging.log4j.Marker;\r
+\r
+import user.commons.StoreUri;\r
+import user.commons.remotestore.IProgressEventListener;\r
+import user.commons.remotestore.ProgressEvent;\r
+import user.commons.remotestore.RemoteStoreProtocol;\r
+import user.jobengine.db.IItemManager;\r
+import user.jobengine.server.IJobEngine;\r
+import user.jobengine.server.IJobRuntime;\r
+\r
+public class FileCopyStep extends JobStep {\r
+ private static final Logger logger = LogManager.getLogger();\r
+ private IItemManager manager;\r
+ private Marker marker;\r
+\r
+ private void check(String sourceProtocol, String sourcePath, String sourceFileName, String targetProtocol, String targetPath, String targetFileName,\r
+ IJobEngine jobEngine, IJobRuntime jobRuntime) {\r
+ if (jobEngine == null) {\r
+ logger.error(marker, "A folyamatkezelő réteg nem elérhető.");\r
+ throw new NullPointerException("Internal error, missing JobEngine reference.");\r
+ }\r
+ manager = jobEngine.getItemManager();\r
+ if (manager == null) {\r
+ logger.error(marker, "Az adatbáziskezelő réteg nem elérhető.");\r
+ throw new NullPointerException("Internal error, missing ItemManager reference.");\r
+ }\r
+ if (sourceProtocol == null) {\r
+ logger.error(marker, "A forrás protokol bemeneti paramétere üres.");\r
+ throw new NullPointerException("System is not configured properly, 'sourceProtocol' input parameter missing.");\r
+ }\r
+ if (sourcePath == null) {\r
+ logger.error(marker, "A forrás fájl elérés bemeneti paramétere üres.");\r
+ throw new NullPointerException("System is not configured properly, 'sourcePath' input parameter missing.");\r
+ }\r
+ if (sourceFileName == null) {\r
+ logger.error(marker, "A forrás fájlnév bemeneti paramétere üres.");\r
+ throw new NullPointerException("System is not configured properly, 'sourceFileName' input parameter missing.");\r
+ }\r
+ if (targetProtocol == null) {\r
+ logger.error(marker, "A cél protokol bemeneti paramétere üres.");\r
+ throw new NullPointerException("System is not configured properly, 'targetProtocol' input parameter missing.");\r
+ }\r
+ if (targetPath == null) {\r
+ logger.error(marker, "A cél fájl elérés bemeneti paramétere üres.");\r
+ throw new NullPointerException("System is not configured properly, 'targetPath' input parameter missing.");\r
+ }\r
+ if (targetFileName == null) {\r
+ logger.error(marker, "A cél fájlnév bemeneti paramétere üres.");\r
+ throw new NullPointerException("System is not configured properly, 'targetFileName' input parameter missing.");\r
+ }\r
+ }\r
+\r
+ @StepEntry\r
+ public Object[] execute(String sourceProtocol, String sourcePath, String sourceFileName, String targetProtocol, String targetPath, String targetFileName,\r
+ int killDateDays, IJobEngine jobEngine, IJobRuntime jobRuntime) throws Exception {\r
+ marker = jobRuntime.getSessionMarker();\r
+\r
+ check(sourceProtocol, sourcePath, sourceFileName, targetProtocol, targetPath, targetFileName, jobEngine, jobRuntime);\r
+\r
+ StoreUri source = null;\r
+ StoreUri target = null;\r
+\r
+ try {\r
+ source = manager.createStoreUri(Enum.valueOf(RemoteStoreProtocol.class, sourceProtocol), sourcePath);\r
+ target = manager.createStoreUri(Enum.valueOf(RemoteStoreProtocol.class, targetProtocol), targetPath);\r
+ source.addProgressListener(new IProgressEventListener() {\r
+ @Override\r
+ public void progressChanged(ProgressEvent evt) {\r
+ jobRuntime.incrementProgress(evt.getProgress());\r
+ }\r
+ });\r
+\r
+ source.transferFrom(target, sourceFileName, targetFileName);\r
+ if (killDateDays > -1)\r
+ EscortFiles.createUNCKillDate(targetPath, targetFileName, killDateDays, marker);\r
+\r
+ } catch (Exception e) {\r
+ logger.catching(e);\r
+ throw e;\r
+ } finally {\r
+ if (source != null) {\r
+ source.cleanUp();\r
+ }\r
+ if (target != null) {\r
+ target.cleanUp();\r
+ }\r
+ }\r
+\r
+ return null;\r
+ }\r
+}\r
--- /dev/null
+package user.jobengine.server.steps;\r
+\r
+import java.io.BufferedInputStream;\r
+import java.io.BufferedOutputStream;\r
+import java.io.File;\r
+import java.io.FileInputStream;\r
+import java.io.FileOutputStream;\r
+import java.io.IOException;\r
+import java.io.InputStream;\r
+import java.io.OutputStream;\r
+import java.nio.file.Files;\r
+import java.nio.file.Path;\r
+import java.nio.file.Paths;\r
+import java.nio.file.attribute.BasicFileAttributes;\r
+import java.text.SimpleDateFormat;\r
+import java.util.Arrays;\r
+import java.util.LinkedHashMap;\r
+import java.util.List;\r
+import java.util.Map;\r
+\r
+import org.apache.commons.io.FilenameUtils;\r
+import org.apache.logging.log4j.LogManager;\r
+import org.apache.logging.log4j.Logger;\r
+import org.apache.logging.log4j.Marker;\r
+\r
+import com.ibm.nosql.json.api.BasicDBObject;\r
+import com.ibm.nosql.json.api.DB;\r
+import com.ibm.nosql.json.api.DBCollection;\r
+import com.ibm.nosql.json.api.QueryBuilder;\r
+\r
+import user.commons.IEntityBase;\r
+import user.commons.MediaCubeMarker;\r
+import user.commons.nosql.NoSQLUtils;\r
+import user.jobengine.db.IItemManager;\r
+import user.jobengine.db.MediaFile;\r
+import user.jobengine.db.MediaFileDAO;\r
+import user.mediacube.metadata.interfaces.IMetadata;\r
+import user.mediacube.metadata.interfaces.IMetadataProvider;\r
+import user.mediacube.metadata.interfaces.IMetadataProviderFactory;\r
+import user.mediacube.metadata.interfaces.MetadataProviderType;\r
+import user.mediacube.metadata.interfaces.MetadataType;\r
+import user.mediacube.metadata.interfaces.PlanAirMetadataListOptions;\r
+\r
+public class HSMMigrateStep extends JobStep {\r
+ private static final Logger logger = LogManager.getLogger();\r
+ private static final String MXFEXT = ".mxf";\r
+ private static final String MOVEXT = ".mov";\r
+ private Marker marker = null;\r
+ private IMetadataProvider hsmProvider;\r
+ private Map<String, BasicDBObject> tapeContents = new LinkedHashMap<>();\r
+ private DBCollection excludes;\r
+ private DBCollection fileHistory;\r
+ private DBCollection volumeHistory;\r
+ private DB db;\r
+ private IMetadataProvider planairProvider;\r
+ private SimpleDateFormat df = new SimpleDateFormat("yyyy.MM.dd");\r
+\r
+ private void cleanupHistory() {\r
+ if (db == null)\r
+ db = NoSQLUtils.getNoSQLDB();\r
+ if (fileHistory == null)\r
+ fileHistory = db.getCollection("hsm_migrate_file_history");\r
+ if (volumeHistory == null)\r
+ volumeHistory = db.getCollection("hsm_migrate_volume_history");\r
+ fileHistory.drop();\r
+ volumeHistory.drop();\r
+ }\r
+\r
+ private BasicDBObject createMetadata(String volumeName, String fileName) throws Exception {\r
+\r
+ Path filePath = Paths.get(fileName);\r
+ String mediaHouseId = FilenameUtils.removeExtension(filePath.getFileName().toString());\r
+ BasicDBObject result = null;\r
+ try {\r
+ result = getPlanAirMetadata(mediaHouseId);\r
+ } catch (Exception e) {\r
+\r
+ logger.error("PlanAir metadata error", e);\r
+ //nem latja a drivert pl.\r
+ //throw e;\r
+ }\r
+\r
+ if (result == null) {\r
+ result = new BasicDBObject();\r
+ BasicFileAttributes attr = Files.readAttributes(filePath, BasicFileAttributes.class);\r
+ result.put("itemHouseId", df.format(attr.lastModifiedTime().toMillis()));\r
+ result.put("itemTitle", filePath.getParent().toString());\r
+ result.put("mediaHouseId", mediaHouseId);\r
+ result.put("mediaTitle", fileName);\r
+ result.put("mediaDescription", volumeName);\r
+ result.put("mediaType", "Generic");\r
+ }\r
+ result.put("userName", "mediacube");\r
+ return result;\r
+ }\r
+\r
+ @SuppressWarnings("serial")\r
+ @StepEntry\r
+ public Object[] execute(String sourceLocation, String targetLocation) throws Exception {\r
+ marker = getJobRuntime().getSessionMarker();\r
+ //remove from prod\r
+ //cleanupHistory();\r
+ hsmProvider = getMetadataProvider(MetadataProviderType.HSM);\r
+ if (hsmProvider == null)\r
+ throw new NullPointerException("No HSM metadata provider available");\r
+ planairProvider = getMetadataProvider(MetadataProviderType.PLANAIR);\r
+ if (planairProvider == null)\r
+ throw new NullPointerException("No PLANAIR metadata provider available");\r
+ Path targetPath = Paths.get(targetLocation);\r
+ try {\r
+ List<IMetadata> volumes = hsmProvider.list(new BasicDBObject());\r
+ for (IMetadata volume : volumes) {\r
+ String volumeName = volume.getTitle();\r
+\r
+ BasicDBObject historyResult = queryVolumeHistory(volumeName);\r
+ if (historyResult != null) {\r
+ logger.info(marker, "A kazetta már feldolgozásra került: {}", volumeName);\r
+ continue;\r
+ }\r
+ logger.info(marker, "A kazetta feldolgozása elindul: {}", volumeName);\r
+ List<IMetadata> contents = getContents(volumeName);\r
+ int p = 0;\r
+ boolean oneSuccess = false;\r
+ boolean hasError = false;\r
+ for (IMetadata content : contents) {\r
+ BasicDBObject c = content.asJSON();\r
+ String hsmFileName = c.getString("fileName");\r
+ long contentFileSize = NoSQLUtils.asLong(c, "fileSize");\r
+ if (!tapeContents.containsKey(hsmFileName)) {\r
+ tapeContents.put(hsmFileName, c);\r
+ Path sourceFilePath = Paths.get(sourceLocation, hsmFileName);\r
+ if (contentFileSize < Files.getFileStore(targetPath).getUsableSpace()) {\r
+ Path targetFilePath = Paths.get(targetLocation, sourceFilePath.getFileName().toString());\r
+\r
+ try {\r
+ if (processHSMFile(volumeName, hsmFileName, sourceFilePath, targetFilePath)) {\r
+ oneSuccess = true;\r
+ }\r
+ } catch (Exception e) {\r
+ hasError = true;\r
+ }\r
+ }\r
+ }\r
+ setProgress(p++ * 100 / contents.size());\r
+\r
+ if (getJobRuntime().isWaitingCancel()) {\r
+ logger.info("Job canceled by user");\r
+ //ne mentsuk a szalagot meg\r
+ return null;\r
+ }\r
+\r
+ }\r
+\r
+ if (oneSuccess && !hasError) {\r
+ saveVolumeHistory(volumeName);\r
+ String subject = "A kazetta eltávolítható a HSM rendszerből: " + volumeName;\r
+ logger.info(new MediaCubeMarker() {\r
+ {\r
+ setSubject(subject);\r
+ }\r
+ }, subject);\r
+ }\r
+\r
+ //ha mar sikerult valamit archivalni kilepunk\r
+ if (oneSuccess)\r
+ break;\r
+ }\r
+ } catch (Exception e) {\r
+ logger.error(marker, "Hiba a migráció során. A rendszer hibaüzenete: {}", e.getMessage());\r
+ throw e;\r
+ }\r
+\r
+ return null;\r
+ }\r
+\r
+ private List<IMetadata> getContents(String volumeName) throws Exception {\r
+ List<IMetadata> contents = null;\r
+ contents = hsmProvider.list(new BasicDBObject("volumeName", volumeName));\r
+ return contents;\r
+ }\r
+\r
+ protected IMetadataProvider getMetadataProvider(MetadataProviderType type) {\r
+ IMetadataProviderFactory factory = getService(IMetadataProviderFactory.class);\r
+ if (factory == null)\r
+ return null;\r
+ return factory.getProvider(type);\r
+ }\r
+\r
+ private BasicDBObject getPlanAirMetadata(String mediaHouseId) throws Exception {\r
+ PlanAirMetadataListOptions opt = new PlanAirMetadataListOptions();\r
+ opt.setSearch(mediaHouseId);\r
+\r
+ List<IMetadata> result = null;\r
+ opt.setType(MetadataType.Material);\r
+ result = planairProvider.list(opt);\r
+ if (result.size() != 0)\r
+ return result.get(0).asJSON();\r
+\r
+ opt.setType(MetadataType.Promo);\r
+ result = planairProvider.list(opt);\r
+ if (result.size() != 0)\r
+ return result.get(0).asJSON();\r
+\r
+ opt.setType(MetadataType.AD);\r
+ result = planairProvider.list(opt);\r
+ if (result.size() != 0)\r
+ return result.get(0).asJSON();\r
+\r
+ return null;\r
+ }\r
+\r
+ //true if need copy\r
+ public boolean prepareCopy(String hsmFileName, Path source, Path target) throws IOException {\r
+ boolean result = true;\r
+ BasicDBObject excludeResult = queryExclude(hsmFileName);\r
+ if (excludeResult != null) {\r
+ logger.warn(marker, "Kivételként megjelölt: {}", hsmFileName);\r
+ return false;\r
+ }\r
+\r
+ File sourceFile = source.toFile();\r
+ File targetFile = target.toFile();\r
+\r
+ if (!sourceFile.exists()) {\r
+ logger.warn(marker, "A forrás nem elérhető: {}", source);\r
+ return false;\r
+ }\r
+\r
+ // BasicDBObject historyResult = queryFileHistory(contentFileName);\r
+ // if (historyResult != null)\r
+ // return false;\r
+\r
+ // if (!mediaFile.getName().toLowerCase().endsWith(MOVEXT.toLowerCase()) && !mediaFile.getName().toLowerCase().endsWith(MXFEXT.toLowerCase()))\r
+ // return;\r
+ // logger.info("Start copy from {} to {}", sourceFilePath, targetFilePath);\r
+\r
+ boolean targetExists = targetFile.exists();\r
+\r
+ long targetLength = targetFile.length();\r
+ long sourceLength = sourceFile.length();\r
+\r
+ if (targetLength == sourceLength) {\r
+ logger.warn(marker, "A fájl már fel van dolgozva: {}, {} -> {}", source, sourceLength, targetLength);\r
+ return false;\r
+ }\r
+\r
+ if (targetLength > sourceLength) {\r
+ logger.warn(marker, "A célfájl nagyobb, törlöm: {}", target);\r
+ Files.delete(target);\r
+ targetLength = 0;\r
+ targetExists = false;\r
+ }\r
+\r
+ if (targetExists) {\r
+ logger.warn(marker, "A fájl már létezik, a másolás folytatódik: {}, {} -> {}", target, sourceLength, targetLength);\r
+ } else\r
+ logger.warn(marker, "Migrálás: {}, {} -> {}", source, sourceLength, targetLength);\r
+\r
+ return result;\r
+ }\r
+\r
+ private boolean processHSMFile(String volumeName, String hsmFileName, Path sourceFilePath, Path targetFilePath) throws Exception {\r
+ int repeat = 4;\r
+ boolean successCopy = false;\r
+\r
+ IItemManager manager = getManager();\r
+ MediaFileDAO mfDAO = (MediaFileDAO) manager.getBaseDAO(MediaFile.class);\r
+ List<IEntityBase> mediaFiles = mfDAO.getByHouseId(sourceFilePath.getFileName().toString());\r
+ if (mediaFiles != null && mediaFiles.size() > 0) {\r
+ logger.warn(marker, "Már archivált: {}", hsmFileName);\r
+ return false;\r
+ }\r
+\r
+ if (prepareCopy(hsmFileName, sourceFilePath, targetFilePath)) {\r
+ while (repeat > 0) {\r
+ try {\r
+ resumeableCopy(sourceFilePath, targetFilePath);\r
+ repeat = 0;\r
+ successCopy = true;\r
+ } catch (Exception e) {\r
+ if (Files.exists(targetFilePath) && targetFilePath.toFile().length() == 0)\r
+ Files.delete(targetFilePath);\r
+ //logger.warn(marker, "Hiba a másolás során: {} ({})", sourceFilePath, e.getMessage());\r
+ repeat--;\r
+ }\r
+ }\r
+ }\r
+ String metadataFileName = sourceFilePath.getFileName() + EscortFiles.DOT_JSON;\r
+ Path metadataPath = Paths.get(targetFilePath.getParent().toString(), EscortFiles.STATUSFOLDER, metadataFileName);\r
+ boolean createMetadata = Files.exists(targetFilePath) && !Files.exists(metadataPath);\r
+\r
+ if (successCopy || createMetadata) {\r
+ String metadata = null;\r
+ try {\r
+ metadata = createMetadata(volumeName, hsmFileName).toPrettyString("");\r
+ EscortFiles.createMetadata(targetFilePath.getParent().toString(), targetFilePath.getFileName().toString(), metadata);\r
+ //saveFileHistory(contentFileName);\r
+\r
+ } catch (Exception e) {\r
+ logger.error(marker, "Metadata error", e);\r
+ return false;\r
+ }\r
+ return true;\r
+ }\r
+\r
+ logger.error(marker, "A fájl másolása nem lehetséges: {}", sourceFilePath);\r
+ return false;\r
+ }\r
+\r
+ public BasicDBObject queryExclude(String fileName) {\r
+ if (db == null)\r
+ db = NoSQLUtils.getNoSQLDB();\r
+ if (excludes == null)\r
+ excludes = db.getCollection("hsm_migrate_exclude");\r
+ Path filePath = Paths.get(fileName);\r
+ String pureFileName = FilenameUtils.removeExtension(filePath.getFileName().toString());\r
+ QueryBuilder qb = QueryBuilder.start("name").in(Arrays.asList(fileName, pureFileName));\r
+ BasicDBObject exceptionResult = NoSQLUtils.asSingle(excludes.find(qb.get()));\r
+ return exceptionResult;\r
+ }\r
+\r
+ public BasicDBObject queryFileHistory(String fileName) {\r
+ if (db == null)\r
+ db = NoSQLUtils.getNoSQLDB();\r
+ if (fileHistory == null)\r
+ fileHistory = db.getCollection("hsm_migrate_file_history");\r
+ QueryBuilder qb = QueryBuilder.start("name").in(Arrays.asList(fileName));\r
+ BasicDBObject historyResult = NoSQLUtils.asSingle(fileHistory.find(qb.get()));\r
+ return historyResult;\r
+ }\r
+\r
+ public BasicDBObject queryVolumeHistory(String volumeName) {\r
+ if (db == null)\r
+ db = NoSQLUtils.getNoSQLDB();\r
+ if (volumeHistory == null)\r
+ volumeHistory = db.getCollection("hsm_migrate_volume_history");\r
+ QueryBuilder qb = QueryBuilder.start("name").in(Arrays.asList(volumeName));\r
+ BasicDBObject historyResult = NoSQLUtils.asSingle(volumeHistory.find(qb.get()));\r
+ return historyResult;\r
+ }\r
+\r
+ public void resumeableCopy(Path source, Path target) throws Exception {\r
+ File sourceFile = source.toFile();\r
+ File targetFile = target.toFile();\r
+ boolean targetExists = targetFile.exists();\r
+\r
+ long targetLength = targetFile.length();\r
+ long sourceLength = sourceFile.length();\r
+\r
+ try (InputStream in = new BufferedInputStream(new FileInputStream(sourceFile));\r
+ OutputStream out = new BufferedOutputStream(new FileOutputStream(targetFile, targetExists))) {\r
+\r
+ byte[] buffer = new byte[128 * 1024];\r
+ int lengthRead;\r
+\r
+ if (targetExists)\r
+ in.skip(targetLength);\r
+\r
+ while ((lengthRead = in.read(buffer)) > 0) {\r
+ out.write(buffer, 0, lengthRead);\r
+ out.flush();\r
+ targetLength = targetFile.length();\r
+ if (targetLength > sourceLength) {\r
+ throw new Exception("Hiba! A fájl túl nagy lett.");\r
+ }\r
+\r
+ if (getJobRuntime().isWaitingCancel()) {\r
+ break;\r
+ }\r
+ }\r
+\r
+ targetLength = targetFile.length();\r
+ sourceLength = sourceFile.length();\r
+ if (targetLength != sourceLength) {\r
+ throw new Exception("Hiba! A fájl mérete nem egyezik.");\r
+ }\r
+ }\r
+ }\r
+\r
+ private void saveFileHistory(String fileName) {\r
+ if (db == null)\r
+ db = NoSQLUtils.getNoSQLDB();\r
+ if (fileHistory == null)\r
+ fileHistory = db.getCollection("hsm_migrate_file_history");\r
+ BasicDBObject item = new BasicDBObject();\r
+ item.put("name", fileName);\r
+ fileHistory.save(item);\r
+\r
+ }\r
+\r
+ private void saveVolumeHistory(String volumeName) {\r
+ if (db == null)\r
+ db = NoSQLUtils.getNoSQLDB();\r
+ if (volumeHistory == null)\r
+ volumeHistory = db.getCollection("hsm_migrate_volume_history");\r
+ BasicDBObject item = new BasicDBObject();\r
+ item.put("name", volumeName);\r
+ volumeHistory.save(item);\r
+\r
+ }\r
+}\r
--- /dev/null
+package user.jobengine.server.steps;\r
+\r
+import java.nio.file.Files;\r
+import java.nio.file.Path;\r
+import java.nio.file.Paths;\r
+import java.util.concurrent.CountDownLatch;\r
+\r
+import org.apache.logging.log4j.LogManager;\r
+import org.apache.logging.log4j.Logger;\r
+\r
+import user.commons.ListUtils;\r
+import user.jobengine.db.ArchivedMedia;\r
+import user.jobengine.db.Item;\r
+import user.jobengine.db.Media;\r
+import user.jobengine.server.steps.shared.ItemManagerExtensions;\r
+\r
+public class IntegrationTestStep extends JobStep {\r
+ private static final Logger logger = LogManager.getLogger();\r
+ private ArchivedMedia archivedMedia;\r
+\r
+ private void _00_test_cancelable() throws Exception {\r
+ CountDownLatch finishLatch = new CountDownLatch(1);\r
+ getEngine().submit(null, e -> {\r
+ if (e.isRuntimeTerminated())\r
+ finishLatch.countDown();\r
+ }, "cancelable.xml", "Test cancelable", ListUtils.asMap("param", 1));\r
+ finishLatch.await();\r
+ logger.info("_00_test_cancelable SUCCESS");\r
+ }\r
+\r
+ private void _01_test_retrieve_ondemand() throws Exception {\r
+ archivedMedia = new ArchivedMedia();\r
+ Media media = getManager().getMedia(15285);\r
+ Item item = getManager().getItem(media.getItemId());\r
+ archivedMedia.setItem(item);\r
+ archivedMedia.setMedia(media);\r
+\r
+ CountDownLatch finishLatch = new CountDownLatch(1);\r
+ getEngine().submit(null, e -> {\r
+ if (e.isRuntimeTerminated())\r
+ finishLatch.countDown();\r
+\r
+ }, "retrieve-ondemand.xml", "Test retrieve-ondemand",\r
+ ListUtils.asMap("globalRetrievePath", "file://10.11.1.100", "localRetrievePath", "/mediacube/data", "materialOutputFolder", "/",\r
+ "promoOutputFolder", "/", "advertisementOutputFolder", "/", "octopusOutputFolder", "/", "genericOutputFolder", "/",\r
+ "onlineOutputFolder", "/", "killDateDays", -1, "nexioAgency", "ARCHIVE_RESTORE", "nexioPort", 2098, "nexioUserName", "administrator",\r
+ "nexioPassword", "system", "archivedMedia", archivedMedia, "successRecipient", "vasary@elgekko.net", "houseId",\r
+ archivedMedia.getMedia().getHouseId(), "targetPathType", "0"));\r
+\r
+ finishLatch.await();\r
+ Path output = Paths.get("/mediacube/data", archivedMedia.getMedia().getHouseId(),\r
+ archivedMedia.getMedia().getHouseId() + "-ARCH-" + archivedMedia.getMedia().getMediaFileRealName());\r
+ if (!output.toFile().exists())\r
+ throw new Exception("File not exists: " + output);\r
+\r
+ logger.info("_01_test_retrieve_ondemand SUCCESS");\r
+ }\r
+\r
+ private void _02_test_archive_material() throws Exception {\r
+ Path input = Paths.get("/mediacube/data", archivedMedia.getMedia().getHouseId(),\r
+ archivedMedia.getMedia().getHouseId() + "-ARCH-" + archivedMedia.getMedia().getMediaFileRealName());\r
+\r
+ String outputName = "IntegrationTest_" + System.currentTimeMillis() + ".mxf";\r
+ Path ouptput = Paths.get(input.getParent().toString(), outputName);\r
+ Files.copy(input, ouptput);\r
+ if (!ouptput.toFile().exists())\r
+ throw new Exception("File not exists: " + ouptput);\r
+\r
+ ArchiveItem archiveItem = new ArchiveItem();\r
+ archiveItem.setMediaFile(ouptput.toString());\r
+ archiveItem.setItemHouseId(outputName);\r
+ archiveItem.setItemTitle(outputName);\r
+ archiveItem.setMediaHouseId(outputName);\r
+ archiveItem.setMediaTitle(outputName);\r
+ archiveItem.setMediaType("Generic");\r
+\r
+ CountDownLatch finishLatch = new CountDownLatch(1);\r
+ getEngine().submit(null, e -> {\r
+ if (e.isRuntimeTerminated())\r
+ finishLatch.countDown();\r
+ }, "archive-material.xml", "Test archive-material", ListUtils.asMap("archiveItem", archiveItem, "killDateDays", 0));\r
+ finishLatch.await();\r
+\r
+ if (!ItemManagerExtensions.isArchived(getManager(), ouptput))\r
+ throw new Exception("File not archived: " + ouptput);\r
+\r
+ Files.delete(ouptput);\r
+\r
+ logger.info("_02_test_archive_material SUCCESS");\r
+ }\r
+\r
+ private void _03_test_delete_materials() throws Exception {\r
+ Path output = Paths.get("/mediacube/data", archivedMedia.getMedia().getHouseId(),\r
+ archivedMedia.getMedia().getHouseId() + "-ARCH-" + archivedMedia.getMedia().getMediaFileRealName());\r
+ String source = output.getParent().toString();\r
+\r
+ CountDownLatch finishLatch = new CountDownLatch(1);\r
+ getEngine().submit(null, e -> {\r
+ if (e.isRuntimeTerminated())\r
+ finishLatch.countDown();\r
+ }, "delete-materials.xml", "Test delete-materials", ListUtils.asMap("sourcePath", source, "skipArchiveCheck", true));\r
+ finishLatch.await();\r
+\r
+ if (output.toFile().exists())\r
+ throw new Exception("File exists: " + output);\r
+ }\r
+\r
+ @StepEntry\r
+ public Object[] execute() throws Exception {\r
+ getJobRuntime().setDescription("_00_test_cancelable");\r
+ _00_test_cancelable();\r
+ setProgress(25);\r
+\r
+ getJobRuntime().setDescription("_01_test_retrieve_ondemand");\r
+ _01_test_retrieve_ondemand();\r
+ setProgress(50);\r
+\r
+ getJobRuntime().setDescription("_02_test_archive_material");\r
+ _02_test_archive_material();\r
+ setProgress(75);\r
+\r
+ getJobRuntime().setDescription("_03_test_delete_materials");\r
+ _03_test_delete_materials();\r
+ setProgress(100);\r
+ return null;\r
+ }\r
+\r
+}\r
--- /dev/null
+package user.jobengine.server.steps;\r
+\r
+import org.apache.commons.lang.StringUtils;\r
+import org.apache.logging.log4j.LogManager;\r
+import org.apache.logging.log4j.Logger;\r
+import org.apache.logging.log4j.Marker;\r
+\r
+import user.commons.RemoteFile;\r
+import user.commons.StoreUri;\r
+import user.commons.remotestore.IProgressEventListener;\r
+import user.commons.remotestore.ProgressEvent;\r
+import user.commons.remotestore.RemoteStoreProtocol;\r
+//import user.jobengine.db.Media;\r
+import user.jobengine.db.ArchivedMedia;\r
+import user.jobengine.db.IItemManager;\r
+import user.jobengine.db.Store;\r
+import user.jobengine.server.IJobEngine;\r
+import user.jobengine.server.IJobRuntime;\r
+\r
+public class MXFCutterStep extends JobStep {\r
+ private static final String TARGETNAMEPATTERN = "-ARCH-%s";\r
+ private static final Logger logger = LogManager.getLogger();\r
+ private IItemManager manager;\r
+ private StoreUri tempTargetUri;\r
+ private StoreUri tempSourceUri;\r
+ private String sourceFileName;\r
+ private Marker marker;\r
+ private int nexioPort;\r
+ private String nexioUserName, nexioPassword;\r
+ private String nexioHost;\r
+\r
+ protected void checkTargetPath(String targetPath) {\r
+ if (StringUtils.isBlank(targetPath)) {\r
+ logger.error(marker, "A folyamat 'targetPath' bemeneti paramétere üres.");\r
+ throw new NullPointerException("System is not configured properly, 'targetPath' input parameter missing.");\r
+ }\r
+ }\r
+\r
+ protected StoreUri createTargetUri(IItemManager manager, String targetPath) throws NullPointerException {\r
+ StoreUri result = null;\r
+ result = manager.createStoreUri(RemoteStoreProtocol.FTP, nexioHost);\r
+ result.setPortNumber(nexioPort);\r
+ result.setUserName(nexioUserName);\r
+ result.setPassword(nexioPassword);\r
+\r
+ return result;\r
+ }\r
+\r
+ @StepEntry\r
+ public Object[] execute(ArchivedMedia archivedMedia, String targetPath, String houseId, String successRecipient, int killDateDays, boolean useNexioTarget,\r
+ String nexioAgency, int nexioPort, String nexioUserName, String nexioPassword, IJobEngine jobEngine, IJobRuntime jobRuntime) throws Exception {\r
+ this.nexioPort = nexioPort;\r
+ this.nexioUserName = nexioUserName;\r
+ this.nexioPassword = nexioPassword;\r
+ nexioHost = System.getProperty("nexio.host");\r
+ marker = jobRuntime.getSessionMarker();\r
+\r
+ if (useNexioTarget && archivedMedia.getTcIn() != null && archivedMedia.getTcOut() != null) {\r
+ setAndCheck(archivedMedia, houseId, targetPath, useNexioTarget, jobEngine);\r
+\r
+ final IJobRuntime runtime = jobRuntime;\r
+ sourceFileName = houseId + TARGETNAMEPATTERN;\r
+ tempSourceUri.addProgressListener(new IProgressEventListener() {\r
+ @Override\r
+ public void progressChanged(ProgressEvent evt) {\r
+ runtime.incrementProgress(evt.getProgress());\r
+ }\r
+ });\r
+\r
+ RemoteFile result = tempSourceUri.transferFrom(tempTargetUri, sourceFileName, sourceFileName);\r
+\r
+ EscortFiles.setNEXIOKillDate(killDateDays, houseId, nexioAgency, tempTargetUri);\r
+\r
+ logger.info("A {} videó kivágva {}s - {}s", sourceFileName, archivedMedia.getTcIn(), archivedMedia.getTcOut());\r
+ }\r
+\r
+ return null;\r
+ }\r
+\r
+ // private String getSourceFileName(ArchivedMedia archivedMedia, Store\r
+ // store) {\r
+ // List<MediaFile> mediaFiles = archivedMedia.getMedia().getMediaFiles();\r
+ // if (mediaFiles == null)\r
+ // return null;\r
+ // for (MediaFile mediaFile : mediaFiles) {\r
+ // if (mediaFile.getStore().getId() == store.getId())\r
+ // return mediaFile.getRelativePath();\r
+ // }\r
+ // return null;\r
+ // }\r
+\r
+ private void setAndCheck(ArchivedMedia archivedMedia, String houseId, String targetPath, boolean useNexioTarget, IJobEngine jobEngine) {\r
+ if (jobEngine == null) {\r
+ logger.error(marker, "A folyamatkezelő réteg nem elérhető.");\r
+ throw new NullPointerException("Internal error, missing JobEngine reference.");\r
+ }\r
+ manager = jobEngine.getItemManager();\r
+ if (manager == null) {\r
+ logger.error(marker, "Az adatbáziskezelő réteg nem elérhető.");\r
+ throw new NullPointerException("Internal error, missing ItemManager reference.");\r
+ }\r
+ if (archivedMedia == null) {\r
+ logger.error(marker, "A folyamat 'mediaCubeMedia' bemeneti paramétere üres.");\r
+ throw new NullPointerException("System is not configured properly, 'mediaCubeMedia' input parameter missing.");\r
+ }\r
+ checkTargetPath(targetPath);\r
+ if (StringUtils.isBlank(houseId)) {\r
+ logger.error(marker, "A folyamat 'houseId' bemeneti paramétere üres.");\r
+ throw new NullPointerException("System is not configured properly, 'houseId' input parameter missing.");\r
+ }\r
+ Store tsmStore = manager.getSystemStore(false);\r
+ if (tsmStore == null) {\r
+ logger.error(marker, "A TSM rendszer beállítás nem elérhető.");\r
+ throw new NullPointerException("System is not configured properly, missing TSM Store.");\r
+ }\r
+\r
+ tempSourceUri = manager.createStoreUri(RemoteStoreProtocol.LOCAL, targetPath);\r
+ if (tempSourceUri == null) {\r
+ logger.error(marker, "A TSM rendszer beállítás paraméterei nem elérhetőek.");\r
+ throw new NullPointerException("System is not configured properly, missing TSM StoreUri.");\r
+ }\r
+ tempTargetUri = createTargetUri(manager, targetPath);\r
+ // sourceFileName = getSourceFileName(archivedMedia, tsmStore);\r
+ if (sourceFileName == null) {\r
+ logger.error(marker, "Adatbázis bejegyzés hiba, a visszatöltendő fájl neve nem található.");\r
+ throw new NullPointerException("Database error, missing MediaFile 'relativePath'.");\r
+ }\r
+ }\r
+}\r
--- /dev/null
+package user.jobengine.server.steps;\r
+\r
+import java.nio.file.Path;\r
+import java.nio.file.Paths;\r
+\r
+import org.apache.logging.log4j.LogManager;\r
+import org.apache.logging.log4j.Logger;\r
+\r
+import user.commons.mediaarea.MediaArea;\r
+import user.jobengine.db.Media;\r
+\r
+public class MediaToolStep extends JobStep {\r
+ private static final Logger logger = LogManager.getLogger();\r
+\r
+ @StepEntry\r
+ public Object[] execute(ArchiveItem archiveItem, Media mediaCubeMedia) throws Exception {\r
+ Path filePath = Paths.get(archiveItem.getMediaFile());\r
+ MediaArea ma = new MediaArea(filePath);\r
+ ma.process();\r
+ long frames = ma.getFrameCount();\r
+ if (frames > 0) {\r
+ logger.info("Media {} length is {}", filePath, frames);\r
+ mediaCubeMedia.setLength(frames);\r
+ getManager().modify(mediaCubeMedia);\r
+ }\r
+ return null;\r
+ }\r
+\r
+}\r
--- /dev/null
+package user.jobengine.server.steps;\r
+\r
+import java.io.File;\r
+import java.nio.file.Files;\r
+import java.nio.file.Path;\r
+import java.nio.file.Paths;\r
+import java.nio.file.attribute.FileAttribute;\r
+import java.nio.file.attribute.PosixFilePermission;\r
+import java.nio.file.attribute.PosixFilePermissions;\r
+import java.util.Date;\r
+import java.util.Set;\r
+\r
+import org.apache.logging.log4j.LogManager;\r
+import org.apache.logging.log4j.Logger;\r
+import org.apache.logging.log4j.Marker;\r
+\r
+import com.ibm.nosql.json.api.BasicDBList;\r
+\r
+import user.jobengine.db.Item;\r
+import user.jobengine.db.ItemManager;\r
+import user.jobengine.db.ItemType;\r
+import user.jobengine.db.Media;\r
+import user.jobengine.server.IJobEngine;\r
+import user.jobengine.server.IJobRuntime;\r
+\r
+/**\r
+ * Itemek es mediak krealasa a ArchiveItem objektum alapjan.\r
+ *\r
+ * @author robi\r
+ */\r
+public class MetadataTransformStep extends JobStep {\r
+ private static final String CONFLICT = ".CONFLICT";\r
+ private static final Logger logger = LogManager.getLogger();\r
+ private static final String ITEM_MANAGER_IS_NULL = "ItemManager is null";\r
+ public static final String DEFAULT_MEDIATYPE = "Generic";\r
+ private ItemManager itemManager;\r
+\r
+ private Marker marker;;\r
+\r
+ private void addTags(ArchiveItem archiveItem, Media mediaCubeMedia) {\r
+ BasicDBList tags = archiveItem.getTags();\r
+ if (tags != null) {\r
+ for (Object tag : tags) {\r
+\r
+ try {\r
+ String tagText = String.valueOf(tag);\r
+ itemManager.addMediaTag(tagText, mediaCubeMedia.getId());\r
+ System.out.println();\r
+\r
+ } catch (Exception e) {\r
+ logger.catching(e);\r
+ }\r
+ }\r
+ }\r
+ }\r
+\r
+ private void checkDuplicates(ArchiveItem archiveItem, String sourceFileName) throws Exception {\r
+ if (itemManager.isMediaFileExists(sourceFileName)) {\r
+ try {\r
+ Path sourcePath = Paths.get(archiveItem.getMediaFile());\r
+ Path parent = sourcePath.getParent();\r
+ Path conflictPath = Paths.get(parent.toString(), CONFLICT);\r
+ File folder = conflictPath.toFile();\r
+ if (!folder.exists() || !folder.isDirectory()) {\r
+ Set<PosixFilePermission> perms = PosixFilePermissions.fromString("rwxrwxrwx");\r
+ FileAttribute<Set<PosixFilePermission>> attr = PosixFilePermissions.asFileAttribute(perms);\r
+ try {\r
+ Files.createDirectories(conflictPath, attr);\r
+ } catch (Exception e) {\r
+ try {\r
+ Files.createDirectory(conflictPath);\r
+ } catch (Exception e1) {\r
+ logger.catching(e);\r
+ throw e;\r
+ }\r
+ }\r
+ }\r
+\r
+ Files.move(sourcePath, Paths.get(conflictPath.toString(), sourceFileName + (new Date()).getTime()));\r
+ } catch (Exception e1) {\r
+ logger.catching(e1);\r
+ logger.error(marker, "Hiba az '{}' állomány mappába másolásakor. A rendszer üzenete: {}", CONFLICT, e1.getMessage());\r
+ }\r
+ throw new Exception("Az '" + sourceFileName + "' állomány már megtalálható az archívumban, archiválása nem lehetséges.");\r
+ }\r
+ }\r
+\r
+ private Item createItem(ArchiveItem archiveItem) {\r
+ Item mediaCubeItem = getExistingItem(archiveItem.getItemHouseId(), archiveItem.getItemTitle());\r
+ if (mediaCubeItem == null)\r
+ mediaCubeItem = itemManager.createItem(DEFAULT_MEDIATYPE, archiveItem.getItemTitle(), archiveItem.getItemDescription(),\r
+ archiveItem.getItemHouseId());\r
+ return mediaCubeItem;\r
+ }\r
+\r
+ private Media createMedia(ArchiveItem archiveItem, Item mediaCubeItem, String mediaType) {\r
+ Media mediaCubeMedia;\r
+ mediaCubeMedia = itemManager.createMedia(mediaType, archiveItem.getMediaTitle(), archiveItem.getMediaDescription(), archiveItem.getMediaHouseId());\r
+ mediaCubeMedia.setLength(archiveItem.getDuration());\r
+ mediaCubeItem.appendMedia(mediaCubeMedia);\r
+\r
+ return mediaCubeMedia;\r
+ }\r
+\r
+ @StepEntry\r
+ public Object[] execute(ArchiveItem archiveItem, IJobEngine jobEngine, IJobRuntime jobRuntime) throws Exception {\r
+ marker = jobRuntime.getSessionMarker();\r
+ Media mediaCubeMedia = null;\r
+ itemManager = (ItemManager) jobEngine.getItemManager();\r
+ if (itemManager == null)\r
+ throw new NullPointerException(ITEM_MANAGER_IS_NULL);\r
+ try {\r
+ File sourceMediaFile = new File(archiveItem.getMediaFile());\r
+ String sourceFileName = sourceMediaFile.getName();\r
+ checkDuplicates(archiveItem, sourceFileName);\r
+ Item mediaCubeItem = createItem(archiveItem);\r
+ jobRuntime.incrementProgress(50);\r
+ String mediaType = getCreateType(archiveItem);\r
+ mediaCubeMedia = createMedia(archiveItem, mediaCubeItem, mediaType);\r
+ //ha itemid 0 akkor merge, egyebkent media insert\r
+\r
+ if (mediaCubeItem.getId() == 0)\r
+ itemManager.mergeItemStructure(mediaCubeItem);\r
+ else {\r
+ mediaCubeMedia.setItemId(mediaCubeItem.getId());\r
+ mediaCubeMedia.add();\r
+ }\r
+\r
+ addTags(archiveItem, mediaCubeMedia);\r
+\r
+ } catch (Exception e) {\r
+ logger.catching(e);\r
+ String fileName = new File(archiveItem.getMediaFile()).getName();\r
+ logger.error(marker, "Az '{}' állomány nem archiválható, mert a metaadat transzformáció sikertelen. A rendszer üzenete: {}", fileName,\r
+ e.getMessage());\r
+ if (!archiveItem.removeCatchedFile())\r
+ logger.error(marker, "Az '{}' állomány .catched jelző állománya nem törölhető.", fileName);\r
+ throw e;\r
+ } finally {\r
+ jobRuntime.incrementProgress(100);\r
+ }\r
+ return new Object[] { mediaCubeMedia };\r
+ }\r
+\r
+ private String getCreateType(ArchiveItem archiveItem) {\r
+ String mediaType = archiveItem.getMediaType();\r
+ if (mediaType == null || mediaType.length() == 0)\r
+ mediaType = DEFAULT_MEDIATYPE;\r
+ else {\r
+ ItemType mediaItemType = itemManager.getItemType(mediaType);\r
+ if (mediaItemType == null)\r
+ itemManager.createItemType(mediaType, mediaType).add();\r
+ }\r
+ return mediaType;\r
+ }\r
+\r
+ private Item getExistingItem(String itemHouseId, String itemTitle) {\r
+ Item[] result = new Item[] { null };\r
+ String sql = String.format("select id from item where houseid='%s' and title='%s'", itemHouseId, itemTitle);\r
+ itemManager.executeQuery(sql, rs -> {\r
+ long id = rs.getLong("id");\r
+ result[0] = itemManager.getItem(id);\r
+ return true;\r
+ }, null);\r
+ return result[0];\r
+ }\r
+\r
+}\r
--- /dev/null
+package user.jobengine.server.steps;\r
+\r
+import java.io.IOException;\r
+import java.nio.file.Path;\r
+import java.nio.file.Paths;\r
+import java.sql.ResultSet;\r
+import java.text.SimpleDateFormat;\r
+import java.util.Date;\r
+import java.util.LinkedHashSet;\r
+import java.util.List;\r
+import java.util.Set;\r
+\r
+import org.apache.commons.io.FileUtils;\r
+import org.apache.ibatis.jdbc.SQL;\r
+import org.apache.logging.log4j.LogManager;\r
+import org.apache.logging.log4j.Logger;\r
+import org.apache.logging.log4j.Marker;\r
+import org.apache.logging.log4j.MarkerManager;\r
+\r
+import com.ibm.nosql.json.api.BasicDBObject;\r
+\r
+import user.jobengine.db.IItemManager;\r
+import user.jobengine.server.steps.shared.MetadataTypeDetector;\r
+import user.mediacube.metadata.interfaces.IMetadata;\r
+import user.mediacube.metadata.interfaces.IMetadataProvider;\r
+import user.mediacube.metadata.interfaces.IMetadataProviderFactory;\r
+import user.mediacube.metadata.interfaces.MetadataProviderType;\r
+import user.mediacube.metadata.interfaces.MetadataType;\r
+import user.mediacube.metadata.interfaces.PlanAirMetadataListOptions;\r
+\r
+public class MetadataUpdater extends JobStep {\r
+ private static final String EMPTY = "";\r
+ private static final String DOT = ".";\r
+ private static final Logger logger = LogManager.getLogger(MetadataUpdater.class);\r
+ private Marker csvMarker = MarkerManager.getMarker("METADATA-UPDATER-CSV");\r
+\r
+ private static final String ITEM_TITLE = "itemTitle";\r
+ private static final String ITEM_HOUSEID = "itemHouseId";\r
+ private static final String MEDIA_HOUSEID = "mediaHouseId";\r
+ private static final String MEDIA_TITLE = "mediaTitle";\r
+ private static final String MEDIA_DESCRIPTION = "mediaDescription";\r
+ private static final String MEDIA_TYPE = "mediaType";\r
+ private static final String MEDIAFILE_HOUSEID = "mediaFileHouseId";\r
+\r
+ private SimpleDateFormat df = new SimpleDateFormat("yyyy.MM.dd HH:mm:ss");\r
+ private Set<String> includeList;\r
+\r
+ public String createQuery() {\r
+ SQL isql = new SQL();\r
+ isql.SELECT("id");\r
+ isql.FROM("STORE");\r
+ isql.WHERE("name = 'TSM'");\r
+\r
+ SQL sql = new SQL();\r
+ sql.SELECT("i.id AS itemId");\r
+ sql.SELECT("i.houseid AS itemHouseId");\r
+ sql.SELECT("i.title AS itemTitle");\r
+ //sql.SELECT("i.description AS itemdescription");\r
+ sql.SELECT("m.id AS mediaId");\r
+ sql.SELECT("m.houseid AS mediaHouseId");\r
+ sql.SELECT("m.title AS mediaTitle");\r
+ sql.SELECT("m.description AS mediaDescription");\r
+ sql.SELECT("mf.houseid AS mediaFileHouseId");\r
+ sql.FROM("MEDIAFILE mf");\r
+ sql.LEFT_OUTER_JOIN("MEDIA m ON (m.id = mf.mediaid)");\r
+ sql.LEFT_OUTER_JOIN("ITEM i ON (i.id = m.itemid)");\r
+ sql.WHERE(String.format("mf.storeid = (%s)", isql.toString()));\r
+ return sql.toString();\r
+ }\r
+\r
+ public String createCountQuery() {\r
+ SQL sql = new SQL();\r
+ sql.SELECT("COUNT(*) as count");\r
+ sql.FROM("MEDIA");\r
+ return sql.toString();\r
+ }\r
+\r
+ @StepEntry\r
+ public Object[] execute() throws Exception {\r
+ try {\r
+\r
+ String location = "/opt/test-mediacube/file_list_original.txt";\r
+ includeList = loadIncludeList(location);\r
+\r
+ IItemManager manager = getManager();\r
+ String sql = createCountQuery();\r
+ long[] count = new long[1];\r
+ count[0] = 0;\r
+ manager.executeQuery(sql, rs -> {\r
+ try {\r
+ count[0] = rs.getLong("count");\r
+ logger.info(getSessionMarker(), "Processing rs");\r
+ } catch (Exception e) {\r
+ logger.error(getMarker(), e.getMessage());\r
+ }\r
+ return true;\r
+ }, null);\r
+\r
+ logger.info(getSessionMarker(), "Count {}", count[0]);\r
+\r
+ long[] current = new long[1];\r
+ current[0] = 0;\r
+\r
+ logger.info(csvMarker,\r
+ "Date;Name;isProgramById;includeContains;isMetadataEquals;itemHouseId;P itemHouseId;itemHouseIdEquals;itemTitle;P itemTitle;itemTitleEquals;mediaHouseId;"\r
+ + "P mediaHouseId;mediaHouseIdEquals;mediaTitle;P mediaTitle;mediaTitleEquals;"\r
+ + "mediaDescription;P mediaDescription;mediaDescriptionEquals;");\r
+\r
+ sql = createQuery();\r
+ manager.executeQuery(sql, rs -> {\r
+\r
+ if (getJobRuntime().isWaitingCancel())\r
+ return false;\r
+\r
+ current[0]++;\r
+ processRecord(rs);\r
+ int p = ((int) current[0] * 100) / ((int) count[0]);\r
+ setProgress(p);\r
+ return true;\r
+ }, null);\r
+\r
+ } catch (Exception e) {\r
+ logger.error(getSessionMarker(), e.getMessage());\r
+ }\r
+ return null;\r
+ }\r
+\r
+ private String set(String value) {\r
+ return value == null ? EMPTY : value;\r
+ }\r
+\r
+ private void processRecord(ResultSet rs) {\r
+ try {\r
+ String itemHouseId = set(rs.getString(ITEM_HOUSEID));\r
+ String itemTitle = set(rs.getString(ITEM_TITLE));\r
+ String mediaHouseId = set(rs.getString(MEDIA_HOUSEID));\r
+ String mediaTitle = set(rs.getString(MEDIA_TITLE));\r
+ String mediaDescription = set(rs.getString(MEDIA_DESCRIPTION));\r
+ String pitemHouseId = EMPTY;\r
+ String pitemTitle = EMPTY;\r
+ String pmediaHouseId = EMPTY;\r
+ String pmediaTitle = EMPTY;\r
+ String pmediaDescription = EMPTY;\r
+\r
+ String fileName = rs.getString(MEDIAFILE_HOUSEID);\r
+ String mediaFileHouseId = fileName;\r
+ if (mediaFileHouseId.contains(DOT))\r
+ mediaFileHouseId = mediaFileHouseId.substring(0, mediaFileHouseId.lastIndexOf(DOT));\r
+\r
+ user.jobengine.server.steps.shared.MetadataType metadataType = MetadataTypeDetector\r
+ .GuessMetadataType(mediaFileHouseId);\r
+\r
+ boolean isProgramById = false;\r
+ boolean includeContains = includeList.contains(fileName);\r
+ boolean itemHouseIdEquals = false;\r
+ boolean itemTitleEquals = false;\r
+ boolean mediaHouseIdEquals = false;\r
+ boolean mediaTitleEquals = false;\r
+ boolean mediaDescriptionEquals = false;\r
+\r
+ logger.info(getMarker(), "{} {}", mediaFileHouseId, metadataType);\r
+ ArchiveItem archiveItem = getPlanAirMetadata(mediaFileHouseId);\r
+ if (archiveItem != null) {\r
+ isProgramById = true;\r
+\r
+ if (isProgramById) {\r
+\r
+ pitemHouseId = set(archiveItem.getItemHouseId());\r
+ itemHouseIdEquals = pitemHouseId.equals(itemHouseId);\r
+\r
+ pitemTitle = set(archiveItem.getItemTitle());\r
+ itemTitleEquals = pitemTitle.equals(itemTitle);\r
+\r
+ pmediaHouseId = set(archiveItem.getMediaHouseId());\r
+ mediaHouseIdEquals = pmediaHouseId.equals(mediaHouseId);\r
+\r
+ pmediaTitle = set(archiveItem.getMediaTitle());\r
+ mediaTitleEquals = pmediaTitle.equals(mediaTitle);\r
+\r
+ pmediaDescription = set(archiveItem.getMediaDescription());\r
+ mediaDescriptionEquals = pmediaDescription.equals(mediaDescription);\r
+\r
+ boolean isMetadataEquals = itemHouseIdEquals && itemTitleEquals && mediaHouseIdEquals\r
+ && mediaTitleEquals && mediaDescriptionEquals;\r
+\r
+ Date now = new Date(System.currentTimeMillis());\r
+ logger.info(csvMarker, "{};{};{};{};{};{};{};{};{};{};{};{};{};{};{};{};{};{};{};{};", D(now),\r
+ fileName, YN(isProgramById), YN(includeContains), YN(isMetadataEquals), itemHouseId,\r
+ pitemHouseId, itemHouseIdEquals, itemTitle, pitemTitle, itemTitleEquals, mediaHouseId,\r
+ pmediaHouseId, mediaHouseIdEquals, mediaTitle, pmediaTitle, mediaTitleEquals,\r
+ mediaDescription, pmediaDescription, mediaDescriptionEquals);\r
+\r
+ logger.info(getSessionMarker(), "Processed {} {}{}{}", fileName, YN(isProgramById),\r
+ YN(includeContains), YN(isMetadataEquals));\r
+ }\r
+ }\r
+\r
+ } catch (\r
+\r
+ Exception e) {\r
+ logger.error(getSessionMarker(), e.getMessage());\r
+ }\r
+\r
+ }\r
+\r
+ private ArchiveItem getPlanAirMetadata(String mediaHouseId) throws Exception {\r
+ PlanAirMetadataListOptions opt = new PlanAirMetadataListOptions();\r
+ opt.setSearch(mediaHouseId);\r
+ opt.setType(MetadataType.Material);\r
+\r
+ BasicDBObject json = null;\r
+ List<IMetadata> data = null;\r
+\r
+ IMetadataProviderFactory factory = getService(IMetadataProviderFactory.class);\r
+ if (factory == null)\r
+ logger.info(getSessionMarker(), "IMetadataProviderFactory is null");\r
+\r
+ IMetadataProvider planairProvider = factory.getProvider(MetadataProviderType.PLANAIR);\r
+ if (planairProvider == null)\r
+ logger.info(getSessionMarker(), "IMetadataProvider is null");\r
+\r
+ ArchiveItem result = null;\r
+ data = planairProvider.list(opt);\r
+ if (data.size() != 0)\r
+ json = data.get(0).asJSON();\r
+ else {\r
+ opt.setType(MetadataType.Promo);\r
+ data = planairProvider.list(opt);\r
+ if (data.size() != 0)\r
+ json = data.get(0).asJSON();\r
+ else {\r
+ opt.setType(MetadataType.AD);\r
+ data = planairProvider.list(opt);\r
+ if (data.size() != 0)\r
+ json = data.get(0).asJSON();\r
+ }\r
+ }\r
+ if (json != null) {\r
+ result = new ArchiveItem();\r
+ result.setItemHouseId(json.getString(ITEM_HOUSEID));\r
+ result.setItemTitle(json.getString(ITEM_TITLE));\r
+ result.setMediaHouseId(json.getString(MEDIA_HOUSEID));\r
+ result.setMediaTitle(json.getString(MEDIA_TITLE));\r
+ result.setMediaDescription(json.getString(MEDIA_DESCRIPTION));\r
+ result.setMediaType(json.getString(MEDIA_TYPE));\r
+ }\r
+ return result;\r
+ }\r
+\r
+ private char YN(boolean value) {\r
+ return value ? 'Y' : 'N';\r
+ }\r
+\r
+ private String D(Date value) {\r
+ return value == null ? EMPTY : df.format(value);\r
+ }\r
+\r
+ private Set<String> loadIncludeList(String location) throws IOException {\r
+ logger.info(getSessionMarker(), "Loading include list {}", location);\r
+ Set<String> result = new LinkedHashSet<>();\r
+\r
+ Path path = Paths.get(location);\r
+ List<String> lines = FileUtils.readLines(path.toFile());\r
+\r
+ String lastDir = null;\r
+ for (String line : lines) {\r
+ line = line.trim();\r
+ if (line.startsWith("Directory of")) {\r
+ lastDir = line;\r
+ lastDir = lastDir.replace("Directory of", EMPTY);\r
+ lastDir = lastDir.replace("X:", EMPTY);\r
+ lastDir = lastDir.replace("\\", "/");\r
+ lastDir = lastDir.trim();\r
+ }\r
+\r
+ if (lastDir != null && lastDir.endsWith(".STATUS"))\r
+ continue;\r
+\r
+ if (line.startsWith("2") && line.length() > 39) {\r
+ String file = line.substring(39).trim();\r
+\r
+ if (file.equals(DOT) || file.equals(".."))\r
+ continue;\r
+\r
+ String len = line.substring(21, 39).trim();\r
+ if (len.length() != 0) {\r
+ String fullpath = "/mnt/POLC/FINISHED_SHOWS" + lastDir + "/" + file;\r
+ result.add(Paths.get(fullpath).getFileName().toString());\r
+ }\r
+ }\r
+\r
+ }\r
+\r
+ return result;\r
+ }\r
+\r
+}\r
--- /dev/null
+package user.jobengine.server.steps;\r
+\r
+import java.io.IOException;\r
+import java.nio.file.FileVisitResult;\r
+import java.nio.file.FileVisitor;\r
+import java.nio.file.Files;\r
+import java.nio.file.Path;\r
+import java.nio.file.Paths;\r
+import java.nio.file.SimpleFileVisitor;\r
+import java.nio.file.attribute.BasicFileAttributes;\r
+\r
+import org.apache.commons.lang.StringUtils;\r
+import org.apache.logging.log4j.LogManager;\r
+import org.apache.logging.log4j.Logger;\r
+import org.apache.logging.log4j.Marker;\r
+\r
+import user.jobengine.db.ArchivedMedia;\r
+import user.jobengine.server.IJobEngine;\r
+import user.jobengine.server.IJobRuntime;\r
+import user.jobengine.server.steps.shared.EscortFiles;\r
+import user.jobengine.server.steps.shared.MetadataType;\r
+import user.jobengine.server.steps.shared.MetadataTypeDetector;\r
+\r
+public class OutputPathAndNameSelectorStep extends JobStep {\r
+\r
+ private static final String TARGETNAMEPATTERN = "-ARCH-%s";\r
+\r
+ private static final Logger logger = LogManager.getLogger();\r
+\r
+ private Marker marker;\r
+\r
+ private void check(String localRetrievePath, String materialOutputFolder, String promoOutputFolder, String advertisementOutputFolder,\r
+ String octopusOutputFolder, String genericOutputFolder, String houseId, String targetPathType) {\r
+ if (StringUtils.isBlank(localRetrievePath)) {\r
+ logger.error(marker, "A folyamat 'localRetrievePath' bemeneti paramétere üres.");\r
+ throw new NullPointerException("System is not configured properly, 'localRetrievePath' input parameter missing.");\r
+ }\r
+ if (StringUtils.isBlank(materialOutputFolder)) {\r
+ logger.error(marker, "A folyamat 'materialOutputFolder' bemeneti paramétere üres.");\r
+ throw new NullPointerException("System is not configured properly, 'materialOutputFolder' input parameter missing.");\r
+ }\r
+ if (StringUtils.isBlank(promoOutputFolder)) {\r
+ logger.error(marker, "A folyamat 'promoOutputFolder' bemeneti paramétere üres.");\r
+ throw new NullPointerException("System is not configured properly, 'promoOutputFolder' input parameter missing.");\r
+ }\r
+ if (StringUtils.isBlank(advertisementOutputFolder)) {\r
+ logger.error(marker, "A folyamat 'advertisementOutputFolder' bemeneti paramétere üres.");\r
+ throw new NullPointerException("System is not configured properly, 'advertisementOutputFolder' input parameter missing.");\r
+ }\r
+ if (StringUtils.isBlank(octopusOutputFolder)) {\r
+ logger.error(marker, "A folyamat 'octopusOutputFolder' bemeneti paramétere üres.");\r
+ throw new NullPointerException("System is not configured properly, 'octopusOutputFolder' input parameter missing.");\r
+ }\r
+ if (StringUtils.isBlank(genericOutputFolder)) {\r
+ logger.error(marker, "A folyamat 'genericOutputFolder' bemeneti paramétere üres.");\r
+ throw new NullPointerException("System is not configured properly, 'genericOutputFolder' input parameter missing.");\r
+ }\r
+ if (StringUtils.isBlank(houseId)) {\r
+ logger.error(marker, "A folyamat 'houseId' bemeneti paramétere üres.");\r
+ throw new NullPointerException("System is not configured properly, 'houseId' input parameter missing.");\r
+ }\r
+ if (StringUtils.isBlank(targetPathType)) {\r
+ logger.error(marker, "A folyamat 'targetPathType' bemeneti paramétere üres.");\r
+ throw new NullPointerException("System is not configured properly, 'targetPathType' input parameter missing.");\r
+ }\r
+ }\r
+\r
+ @StepEntry\r
+ public Object[] execute(String localRetrievePath, String materialOutputFolder, String promoOutputFolder, String advertisementOutputFolder,\r
+ String octopusOutputFolder, String genericOutputFolder, String onlineOutputFolder, String houseId, String targetPathType,\r
+ ArchivedMedia archivedMedia, IJobEngine jobEngine, IJobRuntime jobRuntime) throws Exception {\r
+ marker = jobRuntime.getSessionMarker();\r
+ check(localRetrievePath, materialOutputFolder, promoOutputFolder, advertisementOutputFolder, octopusOutputFolder, genericOutputFolder, houseId,\r
+ targetPathType);\r
+ Object[] result = null;\r
+ switch (Integer.parseInt(targetPathType)) {\r
+ case 0:\r
+ String outputFolder = getFolderById(materialOutputFolder, promoOutputFolder, advertisementOutputFolder, octopusOutputFolder, genericOutputFolder,\r
+ houseId, archivedMedia);\r
+ result = localTargetInit(localRetrievePath, outputFolder, houseId, jobRuntime);\r
+ break;\r
+ case 1:\r
+ result = localTargetInit(localRetrievePath, onlineOutputFolder, houseId, jobRuntime);\r
+ break;\r
+ case 2:\r
+\r
+ if (archivedMedia.getTcIn() != null && archivedMedia.getTcOut() != null)\r
+ result = new Object[] { genericOutputFolder, houseId, true };\r
+ else\r
+ result = new Object[] { null, houseId, true };\r
+ break;\r
+ }\r
+ return result;\r
+ }\r
+\r
+ private String getFolderById(String materialOutputFolder, String promoOutputFolder, String advertisementOutputFolder, String octopusOutputFolder,\r
+ String genericOutputFolder, String houseId, ArchivedMedia archivedMedia) throws Exception {\r
+ String id = houseId.toUpperCase();\r
+ MetadataType mdType = MetadataTypeDetector.GuessMetadataType(id);\r
+ String result = null;\r
+\r
+ //a groovy nem latja enumnak, hanem az objektum tulajdonsaganak\r
+ switch (mdType.toString()) {\r
+ case "OctopusPlaceholder":\r
+ case "OctopusStory":\r
+ result = octopusOutputFolder;\r
+ break;\r
+ case "TrafficMaterial":\r
+ result = materialOutputFolder;\r
+ break;\r
+ case "TrafficPromo":\r
+ result = promoOutputFolder;\r
+ break;\r
+ case "TrafficAD":\r
+ result = advertisementOutputFolder;\r
+ break;\r
+ case "Generic":\r
+ result = genericOutputFolder;\r
+ break;\r
+ }\r
+ return result;\r
+ }\r
+\r
+ private String getPossiblePath(String id, Path targetPath) throws IOException {\r
+ String[] result = new String[] { targetPath.toString() };\r
+ FileVisitor<Path> matcherVisitor = new SimpleFileVisitor<Path>() {\r
+ @Override\r
+ public FileVisitResult preVisitDirectory(Path dir, BasicFileAttributes attrs) throws IOException {\r
+ String dirName = dir.getFileName().toString();\r
+ if (dirName.startsWith(id + "-") || dirName.equals(id)) {\r
+ result[0] = dir.toString();\r
+ return FileVisitResult.TERMINATE;\r
+ }\r
+ return FileVisitResult.CONTINUE;\r
+ }\r
+ };\r
+ Files.walkFileTree(targetPath.getParent(), matcherVisitor);\r
+ return result[0];\r
+ }\r
+\r
+ private Object[] localTargetInit(String localRetrievePath, String outputFolder, String houseId, IJobRuntime jobRuntime) throws IOException {\r
+ String id = houseId.toUpperCase();\r
+ String targetPath = getPossiblePath(id, Paths.get(localRetrievePath, outputFolder, id)).toString();\r
+ String targetNamePattern = houseId + TARGETNAMEPATTERN;\r
+ try {\r
+ EscortFiles.ensureUNCFolder(Paths.get(targetPath));\r
+ } catch (Exception e) {\r
+ logger.error(jobRuntime.getSessionMarker(), "A cél mappa '{}' nem létezik és nem hozható létre. A rendszer hibaüzenete: {}", targetPath,\r
+ e.getMessage());\r
+ throw e;\r
+ }\r
+ return new Object[] { targetPath, targetNamePattern, false };\r
+ }\r
+}\r
--- /dev/null
+package user.jobengine.server.steps;\r
+\r
+import java.util.List;\r
+\r
+import org.apache.logging.log4j.LogManager;\r
+import org.apache.logging.log4j.Logger;\r
+\r
+import user.commons.StoreUri;\r
+import user.commons.remotestore.RemoteStoreProtocol;\r
+import user.jobengine.db.Media;\r
+import user.jobengine.db.MediaFile;\r
+import user.jobengine.db.Store;\r
+\r
+public class PrepareMediaRestoreStep extends JobStep {\r
+ private static final Logger logger = LogManager.getLogger();\r
+\r
+ @StepEntry\r
+ public Object[] execute(Media media, String tempStoreName, String tempStoreProtocol) throws Exception {\r
+ StoreUri mediaStoreUri = null;\r
+ List<MediaFile> mediaFiles = media.getMediaFiles();\r
+ if (mediaFiles.size() > 1)\r
+ throw new Exception("Media " + media.getId() + " already has proxy");\r
+ for (MediaFile mediaFile : mediaFiles) {\r
+ Store store = mediaFile.getStore();\r
+ if (store.isSystem() && !store.isLowres()) {\r
+ mediaStoreUri = store.getStoreUri(RemoteStoreProtocol.TSM);\r
+ if (mediaStoreUri != null)\r
+ break;\r
+ }\r
+ }\r
+\r
+ StoreUri tempStoreUri = getManager().getStoreUri(tempStoreName, Enum.valueOf(RemoteStoreProtocol.class, tempStoreProtocol));\r
+\r
+ return new Object[] { mediaStoreUri, tempStoreUri, media.getMediaFileRealName() };\r
+ }\r
+\r
+}\r
--- /dev/null
+package user.jobengine.server.steps;\r
+\r
+import java.nio.file.Path;\r
+import java.nio.file.Paths;\r
+\r
+import org.apache.logging.log4j.LogManager;\r
+import org.apache.logging.log4j.Logger;\r
+\r
+import com.ibm.nosql.json.api.BasicDBObject;\r
+\r
+import user.commons.configuration.SystemConfiguration;\r
+\r
+public class PrepareRemoteTranscodeStep extends JobStep {\r
+ static private final Logger logger = LogManager.getLogger();\r
+ private static boolean RANDOMIZE_ARCHIVES = SystemConfiguration.getInstance().value("tsm.randomize-archives",\r
+ false);\r
+\r
+ @StepEntry\r
+ public Object[] execute(String profileName, String fileName) throws Exception {\r
+ String hiResRoot = "m:/";\r
+ String lowResRoot = "m:/lowres";\r
+\r
+ Path inputPath = Paths.get(hiResRoot, fileName);\r
+\r
+ String realFileName = fileName;\r
+\r
+ if (RANDOMIZE_ARCHIVES)\r
+ realFileName = realFileName.substring(9);\r
+\r
+ String outFileName = realFileName.substring(0, realFileName.lastIndexOf(".")) + ".mp4";\r
+\r
+ Path relativeOutputPath = null;\r
+ if (realFileName.length() > 2)\r
+ relativeOutputPath = Paths.get(realFileName.substring(0, 1), realFileName.substring(1, 2),\r
+ realFileName.substring(2, 3), outFileName);\r
+ else\r
+ relativeOutputPath = Paths.get("0", outFileName);\r
+\r
+ BasicDBObject parameters = new BasicDBObject();\r
+ parameters.put("profile", profileName);\r
+ parameters.put("input", inputPath.toString());\r
+ parameters.put("output", Paths.get(lowResRoot, relativeOutputPath.toString()).toString());\r
+\r
+ logger.info("Prepared for remote transode {}", parameters);\r
+ return new Object[] { parameters, relativeOutputPath.toString() };\r
+ }\r
+\r
+}\r
--- /dev/null
+package user.jobengine.server.steps;\r
+\r
+import org.apache.logging.log4j.LogManager;\r
+import org.apache.logging.log4j.Logger;\r
+\r
+import com.ibm.nosql.json.api.BasicDBObject;\r
+import com.ibm.nosql.json.api.DB;\r
+import com.ibm.nosql.json.api.DBCollection;\r
+\r
+import user.commons.MediaCubeUndoMarker;\r
+import user.commons.nosql.NoSQLUtils;\r
+import user.jobengine.db.IItemManager;\r
+import user.jobengine.db.Media;\r
+import user.jobengine.db.MediaFile;\r
+\r
+public class QueryMissingProxyMediaStep extends JobStep {\r
+ private static final Logger logger = LogManager.getLogger();\r
+\r
+ @StepEntry\r
+ public Object[] execute() throws Exception {\r
+ setDescription("Looking for missing proxy");\r
+ DB db = NoSQLUtils.getNoSQLDB();\r
+ DBCollection collection = db.getCollection("missing_lowres");\r
+ Media media = getFirstUntranscodedMedia(collection);\r
+\r
+ if (media == null) {\r
+ logger.info(new MediaCubeUndoMarker(getSessionMarker().getSessionID()), "Nincs feldolgozandó hiány.");\r
+ cancel();\r
+ return null;\r
+ }\r
+\r
+ String name = media.getMediaFileRealName();\r
+ collection.save(new BasicDBObject("name", name));\r
+ logger.info(getSessionMarker(), "Processing mediaID: {}", media.getId());\r
+ return new Object[] { media };\r
+ }\r
+\r
+ private Media getFirstUntranscodedMedia(DBCollection collection) {\r
+ Media[] result = new Media[] { null };\r
+ //MV\r
+ String query = "SELECT mediaid FROM VW_MISSING_PROXY_IDS WHERE HOUSEID like 'M%' or HOUSEID like 'P%' or HOUSEID like 'R%' ORDER BY modified DESC";\r
+\r
+ //HTV\r
+ //String query = "SELECT mediaid FROM VW_MISSING_PROXY_IDS";\r
+ IItemManager manager = getManager();\r
+ manager.executeQuery(query, rs -> {\r
+ try {\r
+ long mediaId = rs.getLong(1);\r
+ Media media = manager.getMedia(mediaId);\r
+ //a nevgeneralas miatt az eredeti MediaFilesName nem jo, a pontos nev kell nekunk\r
+ String name = media.getMediaFileRealName();\r
+ //logger.info(getSessionMarker(), "Checking {}", name);\r
+ long existing = collection.find(new BasicDBObject("name", name)).count();\r
+ if (existing > 0) {\r
+ //logger.info(getSessionMarker(), "{} is on missing_lowres list", name);\r
+ return true;\r
+ }\r
+ // 210617 proxy keszites tiltasa\r
+ MediaFile mf = manager.getSystemMediaFile(media);\r
+ if (mf.isDisableProxy()) {\r
+ //logger.info(getSessionMarker(), "Proxy disabled {}", name);\r
+ \r
+ return true;\r
+ }\r
+\r
+ result[0] = media;\r
+ } catch (Exception e) {\r
+ logger.error(e);\r
+ }\r
+ return false;\r
+ }, null);\r
+ return result[0];\r
+ }\r
+}\r
--- /dev/null
+package user.jobengine.server.steps;\r
+\r
+import org.apache.logging.log4j.LogManager;\r
+import org.apache.logging.log4j.Logger;\r
+\r
+import com.ibm.nosql.json.api.BasicDBObject;\r
+\r
+import user.jobengine.server.steps.shared.MediaCubeClient;\r
+\r
+public class RemoteJobStep extends JobStep {\r
+ private static final Logger logger = LogManager.getLogger();\r
+\r
+ @StepEntry\r
+ public Object[] execute(String remoteServer, String template, String name, BasicDBObject remoteJobParameters) throws Exception {\r
+ setDescription("Executing on {}, template {}, profile {}", remoteServer, template, remoteJobParameters.getString("profile"));\r
+ MediaCubeClient mc = new MediaCubeClient(remoteServer);\r
+ long jobId = mc.startjob(template, name, remoteJobParameters);\r
+ logger.info(getMarker(), "Started {} on server {}", jobId, remoteServer);\r
+ while (true) {\r
+ BasicDBObject status = mc.getStatus(jobId);\r
+ if (status != null)\r
+ setProgress(status.getInt("progress"));\r
+\r
+ Thread.sleep(2000);\r
+\r
+ String jobStatus = status.getString("status");\r
+ if ("SUSPENDED".equals(jobStatus))\r
+ throw new Exception(status.getString("description"));\r
+\r
+ if ("FINISHED".equals(jobStatus))\r
+ break;\r
+ }\r
+\r
+ return null;\r
+ }\r
+}\r
--- /dev/null
+package user.jobengine.server.steps;\r
+\r
+import java.io.File;\r
+import java.io.IOException;\r
+import java.nio.file.DirectoryStream;\r
+import java.nio.file.FileVisitResult;\r
+import java.nio.file.FileVisitor;\r
+import java.nio.file.Files;\r
+import java.nio.file.Path;\r
+import java.nio.file.Paths;\r
+import java.nio.file.attribute.BasicFileAttributes;\r
+import java.text.SimpleDateFormat;\r
+import java.util.ArrayList;\r
+import java.util.Arrays;\r
+import java.util.Collections;\r
+import java.util.Date;\r
+import java.util.LinkedHashSet;\r
+import java.util.List;\r
+import java.util.Set;\r
+\r
+import org.apache.commons.io.FileUtils;\r
+import org.apache.commons.io.FilenameUtils;\r
+import org.apache.logging.log4j.LogManager;\r
+import org.apache.logging.log4j.Logger;\r
+\r
+import com.ibm.nosql.json.api.BasicDBObject;\r
+\r
+import user.commons.IEntityBase;\r
+import user.commons.RemoteFile;\r
+import user.commons.StoreUri;\r
+import user.commons.mediatool.MediaInfo;\r
+import user.commons.remotestore.RemoteStoreProtocol;\r
+import user.jobengine.db.Item;\r
+import user.jobengine.db.Media;\r
+import user.jobengine.db.MediaFile;\r
+import user.jobengine.db.MediaFileDAO;\r
+import user.jobengine.db.Store;\r
+import user.jobengine.server.steps.shared.EscortFiles;\r
+import user.mediacube.metadata.interfaces.IMetadata;\r
+import user.mediacube.metadata.interfaces.IMetadataListOptions;\r
+import user.mediacube.metadata.interfaces.IMetadataProvider;\r
+import user.mediacube.metadata.interfaces.IMetadataProviderFactory;\r
+import user.mediacube.metadata.interfaces.MetadataProviderType;\r
+import user.mediacube.metadata.interfaces.MetadataType;\r
+import user.mediacube.metadata.interfaces.PlanAirMetadataListOptions;\r
+\r
+public class SafeDeleteRecursive extends JobStep implements FileVisitor<Path> {\r
+ private static final Logger logger = LogManager.getLogger();\r
+ private static final String ITEM_TITLE = "itemTitle";\r
+ private static final String ITEM_HOUSEID = "itemHouseId";\r
+ private static final String MEDIA_HOUSEID = "mediaHouseId";\r
+ private static final String MEDIA_TITLE = "mediaTitle";\r
+ private static final String MEDIA_DESCRIPTION = "mediaDescription";\r
+ private static final String MEDIA_TYPE = "mediaType";\r
+ private static final String KILLDATEEXT = ".killdate";\r
+ private SimpleDateFormat df = new SimpleDateFormat("yyyy.MM.dd HH:mm:ss");\r
+\r
+ private List<String> skipPathNames = Arrays.asList("!ARCHIVALAS_ALATT", EscortFiles.STATUSFOLDER,\r
+ EscortFiles.CONFLICTFOLDER);\r
+ private Set<String> includeList;\r
+\r
+ private boolean canReadMediaInfo(Path mediaFilePath) {\r
+ boolean result = false;\r
+ try {\r
+ MediaInfo mi = new MediaInfo(mediaFilePath);\r
+ mi.process();\r
+ result = true;\r
+ } catch (Exception e) {\r
+ logger.warn(getSessionMarker(), e.getMessage());\r
+ }\r
+ return result;\r
+ }\r
+\r
+ private boolean checkArchiveItem(ArchiveItem archiveItem, Item item, Media media) {\r
+ if (archiveItem == null)\r
+ return false;\r
+\r
+ String itemHouseId = archiveItem.getItemHouseId() == null ? "" : archiveItem.getItemHouseId();\r
+ if (!itemHouseId.equals(item.getHouseId())) {\r
+ logger.error(getSessionMarker(), "ItemHouseId");\r
+ return false;\r
+ }\r
+\r
+ String itemTitle = archiveItem.getItemTitle() == null ? "" : archiveItem.getItemTitle();\r
+ if (!itemTitle.equals(item.getTitle())) {\r
+ logger.error(getSessionMarker(), "ItemTitle");\r
+ return false;\r
+ }\r
+ String mediaHouseId = archiveItem.getMediaHouseId() == null ? "" : archiveItem.getMediaHouseId();\r
+ if (!mediaHouseId.equals(media.getHouseId())) {\r
+ logger.error(getSessionMarker(), "MediaHouseId");\r
+ return false;\r
+ }\r
+ String mediaTitle = archiveItem.getMediaTitle() == null ? "" : archiveItem.getMediaTitle();\r
+ if (!mediaTitle.equals(media.getTitle())) {\r
+ logger.error(getSessionMarker(), "MediaTitle");\r
+ return false;\r
+ }\r
+\r
+ String mediaDescription = archiveItem.getMediaDescription() == null ? "" : archiveItem.getMediaDescription();\r
+ String storedMediaDescription = media.getDescription() == null ? "" : media.getDescription();\r
+ if (!mediaDescription.equals(storedMediaDescription)) {\r
+ logger.error(getSessionMarker(), "MediaDescription");\r
+ return false;\r
+ }\r
+\r
+ return true;\r
+ }\r
+\r
+ private ArchiveItem createArchiveItem(Path filePath) throws Exception {\r
+ ArchiveItem result = null;\r
+\r
+ String fileName = filePath.getFileName().toString();\r
+ String mediaHouseId = FilenameUtils.removeExtension(fileName);\r
+ try {\r
+ result = getPlanAirMetadata(mediaHouseId);\r
+\r
+ if (result != null)\r
+ result.setMediaFile(filePath.toString());\r
+\r
+ } catch (Exception e) {\r
+ logger.error(getSessionMarker(), "PlanAir metadata error", e);\r
+ }\r
+\r
+ return result;\r
+ }\r
+\r
+ @StepEntry\r
+ public Object[] execute(String sourcePath) throws Exception {\r
+ logger.info(getSessionMarker(), "Starting in {}", sourcePath);\r
+ logger.info(getSessionMarker(), "{};{};{};{};{};{};{};{};{};{};{};{};{};{};{};{};{};{};", "Napló időbélyeg",\r
+ "Neve", "Elérése", "Mérete", "Létrehozva", "Módosítva", "Archiválva (MC)", "Archiválva (TSM)",\r
+ "Méret (TSM)", ".catched", ".killdate", ".nomd", "Archivált (MC)", "Archivált (TSM)",\r
+ "Metaadat egyezés", "Méret egyezés", "Törölhető", "Kezelendő");\r
+\r
+ try {\r
+ String location = "/opt/test-mediacube/file_list_original.txt";\r
+ includeList = loadIncludeList(location);\r
+ Files.walkFileTree(Paths.get(sourcePath), this);\r
+ } catch (Exception e) {\r
+ logger.error(getSessionMarker(), "Az '{}' mappa elérése sikertelen. A rendszer hibaüzenete: {}", sourcePath,\r
+ e.getMessage());\r
+ }\r
+ return null;\r
+ }\r
+\r
+ private ArchiveItem getPlanAirMetadata(String mediaHouseId) throws Exception {\r
+ PlanAirMetadataListOptions opt = new PlanAirMetadataListOptions();\r
+ opt.setSearch(mediaHouseId);\r
+ opt.setType(MetadataType.Material);\r
+\r
+ BasicDBObject json = null;\r
+ List<IMetadata> data = null;\r
+\r
+ IMetadataProviderFactory factory = getService(IMetadataProviderFactory.class);\r
+ if (factory == null)\r
+ logger.info(getSessionMarker(), "IMetadataProviderFactory is null");\r
+\r
+ IMetadataProvider planairProvider = factory.getProvider(MetadataProviderType.PLANAIR);\r
+ if (planairProvider == null)\r
+ logger.info(getSessionMarker(), "IMetadataProvider is null");\r
+\r
+ ArchiveItem result = null;\r
+ data = planairProvider.list(opt);\r
+ if (data.size() != 0)\r
+ json = data.get(0).asJSON();\r
+ else {\r
+ opt.setType(MetadataType.Promo);\r
+ data = planairProvider.list(opt);\r
+ if (data.size() != 0)\r
+ json = data.get(0).asJSON();\r
+ else {\r
+ opt.setType(MetadataType.AD);\r
+ data = planairProvider.list(opt);\r
+ if (data.size() != 0)\r
+ json = data.get(0).asJSON();\r
+ }\r
+ }\r
+ if (json != null) {\r
+ result = new ArchiveItem();\r
+ result.setItemHouseId(json.getString(ITEM_HOUSEID));\r
+ result.setItemTitle(json.getString(ITEM_TITLE));\r
+ result.setMediaHouseId(json.getString(MEDIA_HOUSEID));\r
+ result.setMediaTitle(json.getString(MEDIA_TITLE));\r
+ result.setMediaDescription(json.getString(MEDIA_DESCRIPTION));\r
+ result.setMediaType(json.getString(MEDIA_TYPE));\r
+ }\r
+ return result;\r
+ }\r
+\r
+ @Override\r
+ public FileVisitResult postVisitDirectory(Path paramT, IOException paramIOException) throws IOException {\r
+ return FileVisitResult.CONTINUE;\r
+ }\r
+\r
+ @Override\r
+ public FileVisitResult preVisitDirectory(Path dir, BasicFileAttributes paramBasicFileAttributes)\r
+ throws IOException {\r
+ Path dirName = dir.getFileName();\r
+\r
+ if (skipPathNames.contains(dirName.toString())) {\r
+ //logger.info(getSessionMarker(), "PreVisit skip {}", dir);\r
+ return FileVisitResult.SKIP_SUBTREE;\r
+ } //else\r
+ //logger.info(getSessionMarker(), "PreVisit {}", dir);\r
+\r
+ return FileVisitResult.CONTINUE;\r
+ }\r
+\r
+ private boolean processPathItem(Path mediaPath) throws Exception {\r
+ if (getJobRuntime().isWaitingCancel()) {\r
+ cancel();\r
+ return false;\r
+ }\r
+\r
+ BasicFileAttributes attr = Files.readAttributes(mediaPath, BasicFileAttributes.class);\r
+ File mediaFSFile = mediaPath.toFile();\r
+ String fileName = mediaPath.getFileName().toString();\r
+\r
+ if (mediaFSFile.isDirectory())\r
+ return true;\r
+\r
+ if (fileName.startsWith(".") || fileName.endsWith(".nomd"))\r
+ return true;\r
+\r
+ Path nomdFile = Paths.get(mediaPath.toString() + ".nomd");\r
+ ArchiveItem archiveItem = createArchiveItem(mediaPath);\r
+ List<Path> killDateFiles = getKillDateFiles(mediaPath);\r
+ MediaFile mediaFile = getMediaFile(fileName);\r
+\r
+ long size = mediaFSFile.length();\r
+ Date lastModifiedDate = new Date(attr.lastModifiedTime().toMillis());\r
+ //Date lastAccesDate = new Date(attr.lastAccessTime().toMillis());\r
+ Date createDate = new Date(attr.creationTime().toMillis());\r
+ boolean catchedExists = EscortFiles.isMediaCatched(mediaPath);\r
+ boolean noMDExists = nomdFile.toFile().exists();\r
+ boolean killdateExists = killDateFiles.size() > 0;\r
+ //boolean mediaInfoAvailable = canReadMediaInfo(mediaPath);\r
+ boolean mcArchived = mediaFile != null;\r
+ boolean metadataEquals = false;\r
+ boolean tsmArchived = false;\r
+ boolean sizeEquals = false;\r
+ Date mcArchivedDate = null;\r
+ long tsmSize = 0;\r
+ Date tsmBackupDate = null;\r
+ boolean includeContains = includeList.contains(mediaPath.toString());\r
+\r
+ if (mcArchived) {\r
+ //metadata\r
+ Media media = getManager().getMedia(mediaFile.getMediaId());\r
+ mcArchivedDate = media.getCreated();\r
+ Item item = getManager().getItem(media.getItemId());\r
+ metadataEquals = checkArchiveItem(archiveItem, item, media);\r
+\r
+ //tsm\r
+ String tsmFileName = mediaFile.getRelativePath();\r
+ RemoteFile tsmFile = getTSMFile(tsmFileName);\r
+ tsmArchived = tsmFile != null;\r
+ if (tsmArchived) {\r
+ tsmSize = tsmFile.getSize();\r
+ tsmBackupDate = getTSMBackupDate(tsmFileName);\r
+ sizeEquals = tsmSize == size;\r
+ }\r
+ }\r
+\r
+ boolean canDelete = !noMDExists && catchedExists && killdateExists && mcArchived && tsmArchived\r
+ && metadataEquals;\r
+ Date now = new Date(System.currentTimeMillis());\r
+ logger.info(getSessionMarker(), "{};{};{};{};{};{};{};{};{};{};{};{};{};{};{};{};{};{};", D(now), fileName,\r
+ mediaPath.getParent(), size, D(createDate), D(lastModifiedDate), D(mcArchivedDate), D(tsmBackupDate),\r
+ tsmSize, YN(catchedExists), YN(killdateExists), YN(noMDExists), YN(mcArchived), YN(tsmArchived),\r
+ YN(metadataEquals), YN(sizeEquals), YN(canDelete), YN(includeContains));\r
+\r
+ return true;\r
+ }\r
+\r
+ private Date getTSMBackupDate(String tsmFileName) throws Exception {\r
+ Date result = null;\r
+ IMetadataProviderFactory factory = getService(IMetadataProviderFactory.class);\r
+ if (factory == null)\r
+ return null;\r
+ IMetadataProvider provider = factory.getProvider(MetadataProviderType.TSM);\r
+ if (provider == null)\r
+ return null;\r
+\r
+ IMetadataListOptions opt = provider.createOptions(new BasicDBObject("fileName", tsmFileName));\r
+\r
+ List<IMetadata> tsmContents = provider.list(opt);\r
+ if (tsmContents != null && tsmContents.size() > 0) {\r
+\r
+ for (IMetadata md : tsmContents) {\r
+ Date backupDate = md.asJSON().getDate("backupDate");\r
+ if (result == null)\r
+ result = backupDate;\r
+ else {\r
+ if (backupDate.after(result))\r
+ result = backupDate;\r
+ }\r
+ }\r
+ }\r
+ return result;\r
+ }\r
+\r
+ private char YN(boolean value) {\r
+ return value ? 'Y' : 'N';\r
+ }\r
+\r
+ private String D(Date value) {\r
+ return value == null ? "" : df.format(value);\r
+ }\r
+\r
+ @Override\r
+ public FileVisitResult visitFile(Path filePath, BasicFileAttributes paramBasicFileAttributes) throws IOException {\r
+ //logger.info(getSessionMarker(), "Will checked {}", filePath);\r
+ try {\r
+ if (!processPathItem(filePath))\r
+ return FileVisitResult.TERMINATE;\r
+ } catch (Exception e) {\r
+ logger.catching(e);\r
+ }\r
+ return FileVisitResult.CONTINUE;\r
+ }\r
+\r
+ @Override\r
+ public FileVisitResult visitFileFailed(Path filePath, IOException paramIOException) throws IOException {\r
+ logger.info("Error archive {}", filePath);\r
+ return FileVisitResult.CONTINUE;\r
+ }\r
+\r
+ private List<Path> getKillDateFiles(Path filePath) {\r
+ String killDateFilePattern = String.format("%s.*%s", filePath.getFileName().toString(), KILLDATEEXT);\r
+ List<Path> result = new ArrayList<>();\r
+ Path statusPath = null;\r
+ try {\r
+ statusPath = Paths.get(filePath.getParent().toString(), EscortFiles.STATUSFOLDER);\r
+ } catch (Exception e) {\r
+ logger.catching(e);\r
+ return null;\r
+ }\r
+ File statusPathFile = statusPath.toFile();\r
+ if (statusPathFile.exists() && statusPathFile.isDirectory()) {\r
+ try (DirectoryStream<Path> stream = Files.newDirectoryStream(statusPath, killDateFilePattern)) {\r
+ stream.forEach(p -> result.add(p));\r
+ } catch (Exception e) {\r
+ logger.catching(e);\r
+ }\r
+ }\r
+ Collections.sort(result);\r
+ return result;\r
+ }\r
+\r
+ MediaFile getMediaFile(String fileName) {\r
+ MediaFile result = null;\r
+ MediaFileDAO mfDAO = (MediaFileDAO) getManager().getBaseDAO(MediaFile.class);\r
+ List<IEntityBase> mfList = mfDAO.getByHouseId(fileName);\r
+ if (mfList != null && mfList.size() == 1)\r
+ result = (MediaFile) mfList.get(0);\r
+ return result;\r
+ }\r
+\r
+ RemoteFile getTSMFile(String mcFileName) {\r
+ RemoteFile result = null;\r
+ Store tsmStore = getManager().getSystemStore(false);\r
+ StoreUri tsmStoreUri = tsmStore.getSourceStoreUri(RemoteStoreProtocol.TSM);\r
+ try {\r
+ result = tsmStoreUri.getRemoteFile(mcFileName);\r
+ } catch (Exception e) {\r
+ logger.error(e);\r
+ } finally {\r
+ tsmStoreUri.cleanUp();\r
+ }\r
+ return result;\r
+ }\r
+\r
+ private Set<String> loadIncludeList(String location) throws IOException {\r
+ Set<String> result = new LinkedHashSet<>();\r
+\r
+ Path path = Paths.get(location);\r
+ List<String> lines = FileUtils.readLines(path.toFile());\r
+\r
+ String lastDir = null;\r
+ for (String line : lines) {\r
+ line = line.trim();\r
+ if (line.startsWith("Directory of")) {\r
+ lastDir = line;\r
+ lastDir = lastDir.replace("Directory of", "");\r
+ lastDir = lastDir.replace("X:", "");\r
+ lastDir = lastDir.replace("\\", "/");\r
+ lastDir = lastDir.trim();\r
+ // if (!lastDir.endsWith(".STATUS"))\r
+ // System.out.println(">> " + lastDir);\r
+ }\r
+\r
+ if (lastDir != null && lastDir.endsWith(".STATUS"))\r
+ continue;\r
+\r
+ if (line.startsWith("2") && line.length() > 39) {\r
+ String file = line.substring(39).trim();\r
+\r
+ if (file.equals(".") || file.equals(".."))\r
+ continue;\r
+\r
+ String len = line.substring(21, 39).trim();\r
+ if (len.length() != 0) {\r
+ String fullpath = "/mnt/POLC/FINISHED_SHOWS" + lastDir + "/" + file;\r
+ //System.out.println("/mnt/POLC/FINISHED_SHOWS" + lastDir + "/" + file + " : " + len);\r
+ result.add(fullpath);\r
+ }\r
+ }\r
+\r
+ }\r
+\r
+ return result;\r
+ }\r
+\r
+}\r
--- /dev/null
+package user.jobengine.server.steps;\r
+\r
+import user.jobengine.db.IItemManager;\r
+import user.jobengine.db.Media;\r
+import user.jobengine.db.MediaFile;\r
+import user.jobengine.db.Store;\r
+\r
+public class SaveMediaProxy extends JobStep {\r
+\r
+ @StepEntry\r
+ public Object[] execute(Media media, String proxyRelativePath) throws Exception {\r
+ IItemManager manager = getManager();\r
+ Store store = manager.getCurrentLowresStore();\r
+ MediaFile mediaFile = new MediaFile();\r
+ mediaFile.setMedia(media);\r
+ mediaFile.setStore(store);\r
+ mediaFile.setFileType(manager.getFileType("Low-res"));\r
+ mediaFile.setRelativePath(proxyRelativePath);\r
+ manager.add(mediaFile);\r
+ return null;\r
+ }\r
+}\r
--- /dev/null
+package user.jobengine.server.steps;\r
+\r
+import java.io.File;\r
+import java.io.IOException;\r
+import java.sql.Timestamp;\r
+import java.util.Date;\r
+import java.util.List;\r
+\r
+import org.apache.commons.lang.RandomStringUtils;\r
+import org.apache.logging.log4j.LogManager;\r
+import org.apache.logging.log4j.Logger;\r
+import org.apache.logging.log4j.Marker;\r
+import org.apache.logging.log4j.message.Message;\r
+import org.apache.logging.log4j.message.ParameterizedMessage;\r
+\r
+import user.commons.RemoteFile;\r
+import user.commons.StoreUri;\r
+import user.commons.configuration.SystemConfiguration;\r
+import user.commons.mediatool.Timecode;\r
+import user.commons.mediatool.Timecode.Type;\r
+import user.commons.remotestore.IProgressEventListener;\r
+import user.commons.remotestore.IStatusEventListener;\r
+import user.commons.remotestore.ProgressEvent;\r
+import user.commons.remotestore.RemoteStoreProtocol;\r
+import user.commons.remotestore.StatusEvent;\r
+import user.jobengine.db.FileType;\r
+import user.jobengine.db.IItemManager;\r
+import user.jobengine.db.Media;\r
+import user.jobengine.db.MediaFile;\r
+import user.jobengine.db.Store;\r
+import user.jobengine.server.IJobEngine;\r
+import user.jobengine.server.IJobRuntime;\r
+import user.jobengine.server.JobEngineException;\r
+import user.jobengine.server.steps.shared.EscortFiles;\r
+import user.jobengine.server.steps.shared.ItemManagerExtensions;\r
+\r
+public class TSMBackupStep extends JobStep {\r
+ private static final String MXFEXT = ".MXF";\r
+ private static final Logger logger = LogManager.getLogger();\r
+ private static boolean RANDOMIZE_ARCHIVES = SystemConfiguration.getInstance().value("tsm.randomize-archives");\r
+ private IItemManager manager;\r
+ private File sourceMediaFile;\r
+ private Store tsmStore;\r
+ private StoreUri targetUri;\r
+ private FileType fileType;\r
+ private Marker marker;\r
+\r
+ @StepEntry\r
+ public Object[] execute(ArchiveItem archiveItem, Media mediaCubeMedia, int killDateDays) throws Exception {\r
+ marker = getSessionMarker();\r
+\r
+ File sourceMediaFile = new File(archiveItem.getMediaFile());\r
+ String sourceFileName = sourceMediaFile.getName();\r
+ long fileSize = sourceMediaFile.length();\r
+\r
+ try {\r
+ Timecode timecode = new Timecode(mediaCubeMedia.getLength(), Type.PAL);\r
+ String details = String.format("%s (%s, %d bytes)", sourceFileName, timecode.toString(), fileSize);\r
+ logger.info(marker, details);\r
+ getJobRuntime().setDescription(details);\r
+ } catch (Exception e) {\r
+ String details = String.format("%s (%d bytes)", sourceFileName, fileSize);\r
+ getJobRuntime().setDescription(details);\r
+ }\r
+ Timecode timecode = new Timecode(mediaCubeMedia.getLength(), Type.PAL);\r
+ String details = String.format("%s (%s, %d bytes)", sourceFileName, timecode.toString(), fileSize);\r
+ getJobRuntime().setDescription(details);\r
+ try {\r
+\r
+ setAndCheck(archiveItem, mediaCubeMedia, getEngine());\r
+\r
+ // TODO mxf helyett az osszes kiterjesztest!!!!!\r
+ // A dupla ellenorzes a napon beluli ismetlesek miatt kell\r
+ long existingMediaId = archiveItem.getExistingMediaId();\r
+\r
+ if (fileSize == 0 && existingMediaId == 0) {\r
+ existingMediaId = ItemManagerExtensions.getExistingRundownMedia(manager,\r
+ sourceFileName.replace(MXFEXT, ""));\r
+ if (existingMediaId == 0)\r
+ existingMediaId = -1;\r
+ }\r
+\r
+ if (existingMediaId == 0)\r
+ existingMediaId = ItemManagerExtensions.getExistingRundownMedia(manager,\r
+ sourceFileName.replace(MXFEXT, ""));\r
+ String targetFileName;\r
+ if (RANDOMIZE_ARCHIVES) {\r
+ // a-z, A-Z, 0-9. For example: WRMcpIk7, s57JwCVA\r
+ // veletlenszeru neveket adunk!\r
+ targetFileName = String.format("%s-%s", RandomStringUtils.randomAlphanumeric(8), sourceFileName);\r
+ } else\r
+ targetFileName = sourceFileName;\r
+\r
+ if (existingMediaId == 0) {\r
+ StoreUri sourceUri = manager.createStoreUri(RemoteStoreProtocol.LOCAL,\r
+ sourceMediaFile.getParent().toString());\r
+\r
+ final IJobRuntime runtime = getJobRuntime();\r
+ sourceUri.addProgressListener(new IProgressEventListener() {\r
+ @Override\r
+ public void progressChanged(ProgressEvent evt) {\r
+ runtime.incrementProgress(evt.getProgress());\r
+ }\r
+ });\r
+ sourceUri.addStatusListener(new IStatusEventListener() {\r
+ @Override\r
+ public void statusChanged(StatusEvent evt) {\r
+ evt.setCancel(!canContinue());\r
+ }\r
+ });\r
+\r
+ RemoteFile remoteFile = sourceUri.transferFrom(targetUri, sourceFileName, targetFileName);\r
+ }\r
+\r
+ if (existingMediaId > 0)\r
+ logger.info(marker, "Az '{}' TSM mentése nem szükséges, mert már megtalálható az archívumban.",\r
+ sourceFileName);\r
+\r
+ // Fel kell szabadítani, hogy a kovetkezo archivalaskor is nekifusson\r
+ if (existingMediaId == -1) {\r
+ logger.info(marker,\r
+ "Az '{}' mentése jelenleg nem lehetséges, mert a szükséges metaadat még nem található meg az archívumban.",\r
+ sourceFileName);\r
+ if (!archiveItem.removeCatchedFile())\r
+ logger.error(marker,\r
+ "Az '{}' állomány .catched jelző állománya nem törölhető. Az újabb archiválási kísérlethez annak kézi eltávolítása szükséges!",\r
+ sourceMediaFile.getName());\r
+ mediaCubeMedia.remove();\r
+ } else {\r
+ saveMetadata(mediaCubeMedia, sourceMediaFile, targetFileName, existingMediaId, fileSize,\r
+ archiveItem.isDisableProxy());\r
+ logger.info(marker, "Az '{}' archiválása sikeres.", sourceFileName);\r
+ if (killDateDays != 0)\r
+ EscortFiles.createUNCKillDate(sourceMediaFile.getParent(), sourceFileName, killDateDays, marker);\r
+ }\r
+\r
+ } catch (Exception e) {\r
+ logger.catching(e);\r
+ Message m = new ParameterizedMessage("Az '{}' állomány archiválása sikertelen. A rendszer hibaüzenete: {}",\r
+ details, e.getMessage());\r
+ logger.error(marker, m);\r
+ if (!archiveItem.removeCatchedFile())\r
+ logger.error(marker,\r
+ "Az '{}' állomány .catched jelző állománya nem törölhető. Az újabb archiválási kísérlethez annak kézi eltávolítása szükséges!",\r
+ sourceMediaFile.getName());\r
+ throw new Exception(m.getFormattedMessage());\r
+ }\r
+ return null;\r
+ }\r
+\r
+ private void saveMetadata(Media mediaCubeMedia, File sourceFile, String targetFileName, long existingMediaId,\r
+ long fileSize, boolean disableProxy) {\r
+\r
+ if (existingMediaId == 0) {\r
+ MediaFile mf = manager.createMediaFile(targetFileName, fileType, tsmStore, mediaCubeMedia);\r
+ mf.setHouseId(sourceFile.getName());\r
+ mf.setFileSize(fileSize);\r
+ // 210617 proxy keszites tiltasa\r
+ mf.setDisableProxy(disableProxy);\r
+ mf.add();\r
+ } else {\r
+ Media existingMedia = manager.getMedia(existingMediaId);\r
+ List<MediaFile> mediaFiles = existingMedia.getMediaFiles();\r
+ if (mediaFiles != null) {\r
+ for (MediaFile mf : mediaFiles) {\r
+ mf.setPersister(manager);\r
+ mf.setId(0);\r
+ mf.setMedia(mediaCubeMedia);\r
+ // mivel itt masolat keszul, nem allitunk at semmit\r
+// mf.setFileSize(fileSize);\r
+// mf.setDisableProxy(disableProxy);\r
+ mf.add();\r
+ }\r
+ }\r
+ }\r
+ mediaCubeMedia.setPersister(manager);\r
+\r
+ // 210614 megis maradjon az aktualis idopont\r
+ mediaCubeMedia.setArchived(new Timestamp(new Date().getTime()));\r
+\r
+ /*\r
+ try {\r
+ BasicFileAttributes attr = Files.readAttributes(sourceFile.toPath(), BasicFileAttributes.class);\r
+ mediaCubeMedia.setArchived(new Timestamp(attr.creationTime().toMillis()));\r
+ } catch (IOException e) {\r
+ logger.catching(e);\r
+ }\r
+ */\r
+ mediaCubeMedia.modify();\r
+ }\r
+\r
+ private void setAndCheck(ArchiveItem archiveItem, Media mediaCubeMedia, IJobEngine jobEngine)\r
+ throws JobEngineException, IOException {\r
+ if (jobEngine == null) {\r
+ logger.error(marker, "Az folyamatkezelő réteg nem elérhető.");\r
+ throw new NullPointerException("Internal error, missing JobEngine reference.");\r
+ }\r
+ manager = jobEngine.getItemManager();\r
+ if (manager == null) {\r
+ logger.error(marker, "Az adatbáziskezelő réteg nem elérhető.");\r
+ throw new NullPointerException("Internal error, missing ItemManager reference.");\r
+ }\r
+ if (archiveItem == null) {\r
+ logger.error(marker, "A folyamat 'archiveItem' bemeneti paramétere üres.");\r
+ throw new NullPointerException("System is not configured properly, missing 'archiveItem' input parameter.");\r
+ }\r
+ sourceMediaFile = new File(archiveItem.getMediaFile());\r
+ if (sourceMediaFile == null) {\r
+ logger.error(marker, "A folyamat 'archiveItem' bemeneti paraméter 'mediaFile' értéke üres.");\r
+ throw new NullPointerException(\r
+ "System is not configured properly, missing 'mediaFile' value in 'archiveItem' input parameter.");\r
+ }\r
+ if (!sourceMediaFile.exists()) {\r
+ logger.error(marker, "Az {} állomány nem létezik vagy nem érhető el.", sourceMediaFile.getName());\r
+ throw new IOException(String.format("Input file {} not exists or unreachable.", sourceMediaFile.getName()));\r
+ }\r
+ tsmStore = manager.getSystemStore(false);\r
+ if (tsmStore == null) {\r
+ logger.error(marker, "A TSM rendszer beállítás nem elérhető.");\r
+ throw new NullPointerException("System is not configured properly, missing TSM Store.");\r
+ }\r
+ targetUri = tsmStore.getSourceStoreUri(RemoteStoreProtocol.TSM);\r
+ if (targetUri == null) {\r
+ logger.error(marker, "A TSM rendszer beállítás paraméterei nem elérhetőek.");\r
+ throw new NullPointerException("System is not configured properly, missing TSM StoreUri.");\r
+ }\r
+ fileType = manager.getFileType("High-res");\r
+ if (fileType == null) {\r
+ logger.error(marker, "Adatbázis bejegyzés hiba, a 'High-res' FileType nem található.");\r
+ throw new NullPointerException("System is not configured properly, missing 'High-res' FileType.");\r
+ }\r
+ if (mediaCubeMedia == null) {\r
+ logger.error(marker, "A folyamat 'mediaCubeMedia' bemeneti paramétere üres.");\r
+ throw new NullPointerException(\r
+ "System is not configured properly, 'mediaCubeMedia' input parameter missing.");\r
+ }\r
+\r
+ }\r
+}\r
--- /dev/null
+package user.jobengine.server.steps;\r
+\r
+import org.apache.logging.log4j.LogManager;\r
+import org.apache.logging.log4j.Logger;\r
+\r
+import user.commons.StoreUri;\r
+import user.commons.remotestore.RemoteStoreProtocol;\r
+import user.jobengine.db.ArchivedMedia;\r
+import user.jobengine.db.IItemManager;\r
+import user.jobengine.server.IJobEngine;\r
+import user.jobengine.server.IJobRuntime;\r
+\r
+/*\r
+import user.jobengine.server.steps.EscortFiles;\r
+*/\r
+\r
+public class TSMExtendedRetrieveStep extends TSMRestoreStep {\r
+ private static String NEXIO_HOST = System.getProperty("nexio.host");\r
+ // private static String NEXIO_HOST = SystemConfiguration.getInstance().value("services.nexio.host");\r
+ private static final Logger logger = LogManager.getLogger();\r
+\r
+ private boolean useNexioTarget;\r
+ private int nexioPort;\r
+ private String nexioUserName, nexioPassword;\r
+ private String nexioAgency;\r
+\r
+ @Override\r
+ protected void afterRestore(StoreUri targetUri, String targetPath, int killDateDays, String targetFileName) throws Exception {\r
+ if (useNexioTarget) {\r
+ EscortFiles.setNEXIOKillDate(killDateDays, targetFileName, nexioAgency, targetUri);\r
+ } else {\r
+ super.afterRestore(targetUri, targetPath, killDateDays, targetFileName);\r
+ }\r
+ }\r
+\r
+ @Override\r
+ protected void beforeRestore(StoreUri targetURI, String targetName) throws Exception {\r
+ String newTargetName = targetName;\r
+ if (targetName.contains("."))\r
+ newTargetName = targetName.substring(0, targetName.lastIndexOf('.'));\r
+ if (useNexioTarget)\r
+ if (targetURI.fileExists(newTargetName + ".mxf"))\r
+ throw new Exception(String.format("%s-The newly created file name is existed.", getClass().getSimpleName()));\r
+ }\r
+\r
+ @Override\r
+ protected void checkTargetPath(String targetPath) {\r
+ if (!useNexioTarget)\r
+ super.checkTargetPath(targetPath);\r
+ }\r
+\r
+ @Override\r
+ protected StoreUri createTargetUri(IItemManager manager, String targetPath) throws NullPointerException {\r
+ StoreUri result = null;\r
+ logger.info(getSessionMarker(), "Create target uri {}", targetPath);\r
+ if (useNexioTarget) {\r
+ if (NEXIO_HOST == null) {\r
+ throw new NullPointerException("Missing system property on 'nexio.host' name");\r
+ }\r
+ result = manager.createStoreUri(RemoteStoreProtocol.FTP, NEXIO_HOST);\r
+ result.setPortNumber(nexioPort);\r
+ result.setUserName(nexioUserName);\r
+ result.setPassword(nexioPassword);\r
+ } else\r
+ result = super.createTargetUri(manager, targetPath);\r
+ return result;\r
+ }\r
+\r
+ @StepEntry\r
+ public Object[] execute(ArchivedMedia archivedMedia, String targetPath, String targetNamePattern, String successRecipient, int killDateDays,\r
+ String localRetrievePath, String globalRetrievePath, boolean useNexioTarget, String nexioAgency, int nexioPort, String nexioUserName,\r
+ String nexioPassword, IJobEngine jobEngine, IJobRuntime jobRuntime) throws Exception {\r
+ this.useNexioTarget = useNexioTarget;\r
+ this.nexioAgency = nexioAgency;\r
+ this.nexioPort = nexioPort;\r
+ this.nexioUserName = nexioUserName;\r
+ this.nexioPassword = nexioPassword;\r
+ if (nexioPort == 0) {\r
+ throw new NullPointerException("System is not configured properly, 'nexioPort' input parameter missing.");\r
+ }\r
+ if (nexioUserName == null) {\r
+ throw new NullPointerException("System is not configured properly, 'nexioUserName' input parameter missing.");\r
+ }\r
+ if (nexioPassword == null) {\r
+ throw new NullPointerException("System is not configured properly, 'nexioPassword' input parameter missing.");\r
+ }\r
+ if (nexioAgency == null) {\r
+ throw new NullPointerException("System is not configured properly, 'nexioAgency' input parameter missing.");\r
+ }\r
+\r
+ return super.execute(archivedMedia.getMedia(), targetPath, targetNamePattern, successRecipient, killDateDays, localRetrievePath, globalRetrievePath,\r
+ jobEngine, jobRuntime);\r
+ }\r
+\r
+}\r
--- /dev/null
+package user.jobengine.server.steps;\r
+\r
+import java.io.IOException;\r
+import java.nio.file.Paths;\r
+import java.text.Normalizer;\r
+import java.util.List;\r
+import java.util.regex.Pattern;\r
+\r
+import org.apache.commons.lang.StringUtils;\r
+import org.apache.logging.log4j.LogManager;\r
+import org.apache.logging.log4j.Logger;\r
+import org.apache.logging.log4j.Marker;\r
+import org.apache.logging.log4j.message.Message;\r
+\r
+import user.commons.LogUtils;\r
+import user.commons.RemoteFile;\r
+import user.commons.StoreUri;\r
+import user.commons.mediatool.Timecode;\r
+import user.commons.mediatool.Timecode.Type;\r
+import user.commons.remotestore.IProgressEventListener;\r
+import user.commons.remotestore.IStatusEventListener;\r
+import user.commons.remotestore.ProgressEvent;\r
+import user.commons.remotestore.RemoteStoreProtocol;\r
+import user.commons.remotestore.StatusEvent;\r
+import user.jobengine.db.IItemManager;\r
+import user.jobengine.db.Media;\r
+import user.jobengine.db.MediaFile;\r
+import user.jobengine.db.Store;\r
+import user.jobengine.server.IJobEngine;\r
+import user.jobengine.server.IJobRuntime;\r
+import user.jobengine.server.steps.shared.EscortFiles;\r
+\r
+public class TSMRestoreStep extends JobStep {\r
+ private static final String DOT = ".";\r
+ public static final Pattern DIACRITICS_AND_FRIENDS = Pattern.compile("[\\p{InCombiningDiacriticalMarks}\\p{IsLm}\\p{IsSk}]+");\r
+ private static final Logger logger = LogManager.getLogger();\r
+ private IItemManager manager;\r
+ private StoreUri targetUri;\r
+ private StoreUri sourceUri;\r
+ private String sourceFileName;\r
+ private Marker marker;\r
+\r
+ protected void afterRestore(StoreUri targetUri, String targetPath, int killDateDays, String targetFileName) throws IOException, Exception {\r
+ if (killDateDays != 0)\r
+ EscortFiles.createUNCKillDate(targetPath, targetFileName, killDateDays, marker);\r
+ }\r
+\r
+ protected void beforeRestore(StoreUri targetURI, String targetFileName) throws Exception {\r
+ }\r
+\r
+ protected void checkTargetPath(String targetPath) {\r
+ if (StringUtils.isBlank(targetPath)) {\r
+ logger.error(marker, "A folyamat 'targetPath' bemeneti paramétere üres.");\r
+ throw new NullPointerException("System is not configured properly, 'targetPath' input parameter missing.");\r
+ }\r
+ }\r
+\r
+ protected StoreUri createTargetUri(IItemManager manager, String targetPath) {\r
+ return manager.createStoreUri(RemoteStoreProtocol.LOCAL, targetPath);\r
+ }\r
+\r
+ @StepEntry\r
+ public Object[] execute(Media mediaCubeMedia, String targetPath, String targetNamePattern, String successRecipient, int killDateDays,\r
+ String localRetrievePath, String globalRetrievePath, IJobEngine jobEngine, IJobRuntime jobRuntime) throws Exception {\r
+ marker = jobRuntime.getSessionMarker();\r
+ setAndCheck(mediaCubeMedia, targetPath, targetNamePattern, localRetrievePath, globalRetrievePath, jobEngine);\r
+ String targetFileName = String.format(targetNamePattern, sourceFileName);\r
+ //20210129\r
+ //targetFileName = getMaximizedFileName(mediaCubeMedia, targetFileName, 120);\r
+\r
+ Timecode timecode = new Timecode(mediaCubeMedia.getLength(), Type.PAL);\r
+ try {\r
+ String details = String.format("%s (%s)", sourceFileName, timecode.toString());\r
+ jobRuntime.setDescription(details);\r
+ beforeRestore(targetUri, targetNamePattern);\r
+ final IJobRuntime runtime = jobRuntime;\r
+ sourceUri.addProgressListener(new IProgressEventListener() {\r
+ @Override\r
+ public void progressChanged(ProgressEvent evt) {\r
+ runtime.incrementProgress(evt.getProgress());\r
+ }\r
+ });\r
+ sourceUri.addStatusListener(new IStatusEventListener() {\r
+ @Override\r
+ public void statusChanged(StatusEvent evt) {\r
+ evt.setCancel(!canContinue());\r
+ }\r
+ });\r
+ RemoteFile result = sourceUri.transferFrom(targetUri, sourceFileName, targetFileName);\r
+\r
+ String globalTargetPath = Paths.get(targetPath, targetFileName)\r
+ .getParent()\r
+ .toString()\r
+ .replace(Paths.get(localRetrievePath).toString(), globalRetrievePath);\r
+\r
+ logger.info(marker,\r
+ "Az '{}' állomány visszatöltése sikeres volt '{}' néven. A célmappa a <a href='{}' target='_blank'>ide kattintva</a> nyitható meg.",\r
+ sourceFileName, targetFileName, globalTargetPath);\r
+ afterRestore(targetUri, targetPath, killDateDays, targetFileName);\r
+\r
+ } catch (Exception e) {\r
+ Message msg = LogUtils.format("Az '{}' állomány visszatöltése sikertelen. A rendszer hibaüzenete: {}", sourceFileName, e.getMessage());\r
+ logger.error(marker, msg);\r
+ // logger.error(jobRuntime.marker, msg);\r
+ logger.catching(e);\r
+ throw e;\r
+ }\r
+\r
+ return null;\r
+ }\r
+\r
+ private String getMaximizedFileName(Media mediaCubeMedia, String targetFileName, int limit) {\r
+ String name = targetFileName;\r
+ String extension = "";\r
+ if (name.contains(DOT)) {\r
+ extension = DOT + name.substring(name.lastIndexOf(DOT) + 1);\r
+ name = name.substring(0, name.lastIndexOf(DOT));\r
+ }\r
+ String typeName = Normalizer.normalize(mediaCubeMedia.getItemType().getName(), Normalizer.Form.NFD);\r
+ typeName = DIACRITICS_AND_FRIENDS.matcher(typeName).replaceAll("");\r
+ typeName = typeName.replace(" ", "_");\r
+\r
+ int allowedSize = limit - typeName.length() - 1 - extension.length();\r
+ if (name.length() > allowedSize)\r
+ name = name.substring(0, allowedSize);\r
+\r
+ return String.format("%s_%s%s", name, typeName, extension);\r
+ }\r
+\r
+ private String getSourceFileName(Media mediaCubeMedia, Store store) {\r
+ List<MediaFile> mediaFiles = mediaCubeMedia.getMediaFiles();\r
+ if (mediaFiles == null)\r
+ return null;\r
+ for (MediaFile mediaFile : mediaFiles) {\r
+ if (mediaFile.getStore().getId() == store.getId())\r
+ return mediaFile.getRelativePath();\r
+ }\r
+ return null;\r
+ }\r
+\r
+ private void setAndCheck(Media mediaCubeMedia, String targetPath, String targetNamePattern, String localRetrievePath, String globalRetrievePath,\r
+ IJobEngine jobEngine) {\r
+ if (jobEngine == null) {\r
+ logger.error(marker, "Az folyamatkezelő réteg nem elérhető.");\r
+ throw new NullPointerException("Internal error, missing JobEngine reference.");\r
+ }\r
+ manager = jobEngine.getItemManager();\r
+ if (manager == null) {\r
+ logger.error(marker, "Az adatbáziskezelő réteg nem elérhető.");\r
+ throw new NullPointerException("Internal error, missing ItemManager reference.");\r
+ }\r
+ if (mediaCubeMedia == null) {\r
+ logger.error(marker, "A folyamat 'mediaCubeMedia' bemeneti paramétere üres.");\r
+ throw new NullPointerException("System is not configured properly, 'mediaCubeMedia' input parameter missing.");\r
+ }\r
+ checkTargetPath(targetPath);\r
+ if (StringUtils.isBlank(targetNamePattern)) {\r
+ logger.error(marker, "A folyamat 'targetNamePattern' bemeneti paramétere üres.");\r
+ throw new NullPointerException("System is not configured properly, 'targetNamePattern' input parameter missing.");\r
+ }\r
+ Store tsmStore = manager.getSystemStore(false);\r
+ if (tsmStore == null) {\r
+ logger.error(marker, "A TSM rendszer beállítás nem elérhető.");\r
+ throw new NullPointerException("System is not configured properly, missing TSM Store.");\r
+ }\r
+ sourceUri = tsmStore.getSourceStoreUri(RemoteStoreProtocol.TSM);\r
+ if (sourceUri == null) {\r
+ logger.error(marker, "A TSM rendszer beállítás paraméterei nem elérhetőek.");\r
+ throw new NullPointerException("System is not configured properly, missing TSM StoreUri.");\r
+ }\r
+ targetUri = createTargetUri(manager, targetPath);\r
+ sourceFileName = getSourceFileName(mediaCubeMedia, tsmStore);\r
+ if (sourceFileName == null) {\r
+ logger.error(marker, "Adatbázis bejegyzés hiba, a visszatöltendő fájl neve nem található.");\r
+ throw new NullPointerException("Database error, missing MediaFile 'relativePath'.");\r
+ }\r
+\r
+ if (StringUtils.isBlank(localRetrievePath)) {\r
+ logger.error(marker, "A folyamat 'localRetrievePath' bemeneti paramétere üres.");\r
+ throw new NullPointerException("System is not configured properly, 'localRetrievePath' input parameter missing.");\r
+ }\r
+ if (StringUtils.isBlank(globalRetrievePath)) {\r
+ logger.error(marker, "A folyamat 'globalRetrievePath' bemeneti paramétere üres.");\r
+ throw new NullPointerException("System is not configured properly, 'globalRetrievePath' input parameter missing.");\r
+ }\r
+ }\r
+}\r
--- /dev/null
+package user.jobengine.server.steps;\r
+\r
+import java.nio.file.Files;\r
+import java.nio.file.Paths;\r
+\r
+import org.apache.logging.log4j.LogManager;\r
+import org.apache.logging.log4j.Logger;\r
+\r
+import user.commons.StoreUri;\r
+import user.commons.mediatool.Timecode;\r
+import user.commons.mediatool.Timecode.Type;\r
+import user.commons.remotestore.IProgressEventListener;\r
+import user.commons.remotestore.IStatusEventListener;\r
+import user.commons.remotestore.ProgressEvent;\r
+import user.commons.remotestore.RemoteStoreProtocol;\r
+import user.commons.remotestore.StatusEvent;\r
+import user.jobengine.db.Media;\r
+import user.jobengine.db.MediaFile;\r
+import user.jobengine.db.Store;\r
+\r
+public class TSMSimpleRestoreStep extends JobStep {\r
+ private static final Logger logger = LogManager.getLogger();\r
+\r
+ @StepEntry\r
+ public Object[] execute(long mediaId, String targetPath) throws Exception {\r
+ String fileName = null;\r
+ try {\r
+ Media media = getManager().getMedia(mediaId);\r
+ MediaFile mediaFile = getManager().getSystemMediaFile(media);\r
+ fileName = mediaFile.getRelativePath();\r
+ Timecode timecode = new Timecode(media.getLength(), Type.PAL);\r
+ getJobRuntime().setDescription(String.format("%s (%s)", fileName, timecode.toString()));\r
+ Store tsmStore = getManager().getSystemStore(false);\r
+ StoreUri sourceStoreUri = getManager().getStoreUri(tsmStore.getName(), RemoteStoreProtocol.TSM);\r
+ sourceStoreUri.addProgressListener(new IProgressEventListener() {\r
+ @Override\r
+ public void progressChanged(ProgressEvent evt) {\r
+ setProgress(evt.getProgress());\r
+ }\r
+ });\r
+ sourceStoreUri.addStatusListener(new IStatusEventListener() {\r
+ @Override\r
+ public void statusChanged(StatusEvent evt) {\r
+ evt.setCancel(!canContinue());\r
+ }\r
+ });\r
+ StoreUri targetStoreUri = getManager().createStoreUri(RemoteStoreProtocol.LOCAL, targetPath);\r
+ sourceStoreUri.transferFrom(targetStoreUri, fileName, fileName + ".part");\r
+ Files.move(Paths.get(targetPath, fileName + ".part"), Paths.get(targetPath, fileName));\r
+ } catch (Exception e) {\r
+ logger.error("Az '{}' állomány visszatöltése sikertelen. A rendszer üzenete: {}", fileName, e.getMessage());\r
+ throw e;\r
+ }\r
+\r
+ return null;\r
+ }\r
+}\r
--- /dev/null
+package user.jobengine.server.steps;\r
+\r
+import org.apache.logging.log4j.LogManager;\r
+import org.apache.logging.log4j.Logger;\r
+\r
+import user.commons.IJob;\r
+import user.commons.JobStatus;\r
+import user.commons.ListUtils;\r
+import user.jobengine.server.IJobEngine;\r
+import user.jobengine.server.IJobRuntime;\r
+\r
+public class TestForkCancelableStep extends JobStep {\r
+ private static final String CHILD_TITLE = "Párhuzamosított alfolyamat";\r
+ // private static final String CHILD_TEMPLATE = "fake-concurrent.xml";\r
+ private static final String CHILD_TEMPLATE = "cancelable.xml";\r
+ private static final Logger logger = LogManager.getLogger();\r
+ int count = 3;\r
+\r
+ @StepEntry\r
+ public Object[] execute(IJobEngine jobEngine, IJobRuntime jobRuntime) throws Exception {\r
+ for (int i = 0; i < count; i++) {\r
+ IJobRuntime rt = getEngine().submit(null, e -> {\r
+ if (e.getStatus().equals(JobStatus.SUSPENDED)) {\r
+ IJobRuntime rt = (IJobRuntime) e.getSource();\r
+ logger.info("Cleanup on SUSPEND {}, {}", rt.getId(), rt.isDisableRetry());\r
+ }\r
+ }, CHILD_TEMPLATE, "JOB " + i, 0, IJobEngine.DEFAULT_OWNER, ListUtils.asMap("param", i));\r
+ \r
+ rt.setRelated("TEST" + rt.getId());\r
+ }\r
+\r
+ logger.info("Done");\r
+ return null;\r
+ }\r
+}\r
--- /dev/null
+package user.jobengine.server.steps;\r
+\r
+import java.io.File;\r
+import java.io.IOException;\r
+import java.nio.file.Files;\r
+import java.nio.file.Path;\r
+import java.nio.file.Paths;\r
+\r
+import org.apache.commons.io.FilenameUtils;\r
+import org.apache.logging.log4j.LogManager;\r
+import org.apache.logging.log4j.Logger;\r
+import org.apache.logging.log4j.Marker;\r
+import org.apache.logging.log4j.message.Message;\r
+import org.apache.logging.log4j.message.ParameterizedMessage;\r
+\r
+import user.commons.FFAStransAPI;\r
+import user.commons.IFFAStransAPI;\r
+import user.commons.StoreUri;\r
+import user.commons.mediatool.Timecode;\r
+import user.commons.mediatool.Timecode.Type;\r
+import user.commons.remotestore.RemoteStoreProtocol;\r
+import user.jobengine.db.FileType;\r
+import user.jobengine.db.IItemManager;\r
+import user.jobengine.db.Media;\r
+import user.jobengine.db.Store;\r
+import user.jobengine.server.IJobEngine;\r
+import user.jobengine.server.IJobRuntime;\r
+\r
+public class TranscodeFFAStranStep extends JobStep {\r
+ private static final int POLL_INTERVALL = 3000;\r
+ private static final String MP4EXT = ".MP4";\r
+ private static final String MXFEXT = ".MXF";\r
+ private static final String LOWRES_FILETYPE = "Low-res";\r
+ private static final Logger logger = LogManager.getLogger("TranscodeFFAStranStep");\r
+ private IItemManager manager;\r
+ private Store store;\r
+ private FileType fileType;\r
+ private Media mediaCubeMedia;\r
+ private Marker marker;\r
+\r
+ @StepEntry\r
+ public Object[] execute(ArchiveItem archiveItem, Media mediaCubeMedia, String transcoderAddress, String transcoderTemplateName,\r
+ String globalHiresSourcePath, String localLowresTargetPath, boolean deleteSource, IJobEngine jobEngine, IJobRuntime jobRuntime) throws Exception {\r
+\r
+ this.marker = jobRuntime.getSessionMarker();\r
+ this.manager = jobEngine.getItemManager();\r
+ this.store = check(manager.getCurrentLowresStore(), "lowres Store");\r
+ this.fileType = check(manager.getFileType(LOWRES_FILETYPE), "lowres FileType");\r
+ this.mediaCubeMedia = check(mediaCubeMedia, "mediaCubeMedia");\r
+ check(archiveItem, "archiveItem");\r
+ check(transcoderAddress, "transcoderAddress");\r
+ check(transcoderTemplateName, "transcoderTemplateName");\r
+ check(globalHiresSourcePath, "globalHiresSourcePath");\r
+ check(localLowresTargetPath, "localLowresTargetPath");\r
+\r
+ File sourceMediaFile = new File(archiveItem.getMediaFile());\r
+ logger.info("Transcoding {}", archiveItem.getMediaFile());\r
+ String sourceFileName = sourceMediaFile.getName();\r
+ Timecode timecode = new Timecode(mediaCubeMedia.getLength(), Type.PAL);\r
+\r
+ String details = String.format("%s (%s, %d bytes)", sourceFileName, timecode.toString(), sourceMediaFile.length());\r
+\r
+ StoreUri storeUri = store.getTargetStoreUri(RemoteStoreProtocol.LOCAL);\r
+ if (storeUri == null)\r
+ throw new Exception("Can not detect proxy folder.");\r
+\r
+ String webPath = storeUri.toString(true);\r
+\r
+ Path targetPath = null;\r
+ try {\r
+ String targetFileName = FilenameUtils.removeExtension(sourceFileName) + MP4EXT;\r
+ targetPath = Paths.get(localLowresTargetPath, targetFileName);\r
+ if (!targetPath.toFile().exists()) {\r
+ // jobRuntime.setDescription(String.format("%s: %s", jobRuntime.getDescription(), details));\r
+ jobRuntime.setDescription(String.format("%s transzkódolása", details));\r
+ String sourceFile = Paths.get(globalHiresSourcePath, sourceFileName).toString();\r
+ IFFAStransAPI api = new FFAStransAPI(transcoderAddress, p -> {\r
+ if (p <= 100)\r
+ jobRuntime.incrementProgress(p);\r
+ });\r
+\r
+ api.submit(transcoderTemplateName, sourceFile);\r
+ api.monitor(POLL_INTERVALL);\r
+ }\r
+\r
+ //a sikeres transzkod utan nem mindig van ott egybol a fajl\r
+ long started = System.currentTimeMillis();\r
+ while (!targetPath.toFile().exists()) {\r
+ long current = System.currentTimeMillis();\r
+ //max 5 perc varakozas\r
+ if (current - started > 5 * 60 * 1000)\r
+ throw new Exception("Transcode job target file access timed out");\r
+ Thread.sleep(POLL_INTERVALL);\r
+ }\r
+\r
+ postprocess(targetPath, webPath);\r
+\r
+ } catch (Exception e) {\r
+ logger.catching(e);\r
+ Message m = new ParameterizedMessage("{} átkódolás hiba: {}", sourceFileName, e.getMessage());\r
+ logger.error(marker, m);\r
+ throw new Exception(m.getFormattedMessage());\r
+ } finally {\r
+ try {\r
+ if (deleteSource && sourceMediaFile != null && sourceMediaFile.exists())\r
+ sourceMediaFile.delete();\r
+ } catch (Exception e) {\r
+ logger.catching(e);\r
+ }\r
+ try {\r
+ if (deleteSource && targetPath != null && targetPath.toFile().exists())\r
+ Files.delete(targetPath);\r
+ } catch (Exception e) {\r
+ logger.catching(e);\r
+ }\r
+ }\r
+ return null;\r
+ }\r
+\r
+ private void postprocess(Path transcodedFilePath, String webPath) throws IOException {\r
+ Path lowresPath = null;\r
+ try {\r
+ String transcodedFileName = transcodedFilePath.getFileName().toString();\r
+ String targetPath = null;\r
+ if (transcodedFileName.indexOf(".") > 2) {\r
+ Path subdir = Paths.get(transcodedFileName.substring(0, 1), transcodedFileName.substring(1, 2), transcodedFileName.substring(2, 3));\r
+ EscortFiles.ensureUNCFolder(webPath, subdir.toString());\r
+ targetPath = Paths.get(subdir.toString(), transcodedFileName).toString();\r
+ } else {\r
+ targetPath = transcodedFileName;\r
+ }\r
+ lowresPath = Paths.get(webPath, targetPath);\r
+ int version = 1;\r
+ while (lowresPath.toFile().exists()) {\r
+ String fileName = transcodedFileName + version + MP4EXT;\r
+ lowresPath = Paths.get(lowresPath.toString().replace(transcodedFileName, fileName));\r
+ targetPath = targetPath.replace(transcodedFileName, fileName);\r
+ transcodedFileName = fileName;\r
+ version++;\r
+ }\r
+\r
+ Files.move(transcodedFilePath, lowresPath);\r
+ manager.createMediaFile(targetPath, fileType, store, mediaCubeMedia).add();\r
+ } catch (IOException e) {\r
+ logger.catching(e);\r
+ logger.error(marker, "A '{}' állomány mozgatása a '{}' helyre nem sikerült.", transcodedFilePath, lowresPath);\r
+ throw e;\r
+ }\r
+ }\r
+\r
+}\r
--- /dev/null
+package user.jobengine.server.steps;\r
+\r
+import java.nio.file.Path;\r
+import java.nio.file.Paths;\r
+\r
+import org.apache.commons.net.ftp.FTPClient;\r
+import org.apache.logging.log4j.LogManager;\r
+import org.apache.logging.log4j.Logger;\r
+\r
+import user.commons.RemoteFile;\r
+import user.commons.StoreUri;\r
+import user.commons.remotestore.FtpDirectoryLister;\r
+import user.commons.remotestore.RemoteStoreProtocol;\r
+import user.jobengine.db.Store;\r
+\r
+public class TransferStep extends JobStep {\r
+ private static final String DOT_PART = ".part";\r
+ private static final Logger logger = LogManager.getLogger();\r
+\r
+ private void copy(StoreUri sourceStoreUri, String sourceFileName, StoreUri targetStoreUri, String targetFileName) throws Exception {\r
+ String currentTargetFileName = targetFileName;\r
+\r
+ boolean renameAfterCopy = false;\r
+ boolean renameAfterFTP = false;\r
+ if (getTmpExtension() != null) {\r
+ if (RemoteStoreProtocol.LOCAL.equals(targetStoreUri.getProtocol())) {\r
+ currentTargetFileName += getTmpExtension();\r
+ renameAfterCopy = true;\r
+ }\r
+ Store targetStore = getManager().getStore(targetStoreUri.getStoreId());\r
+\r
+ if (RemoteStoreProtocol.FTP.equals(targetStoreUri.getProtocol()) && !"NEXIO1".equals(targetStore.getName())\r
+ && !"NEXIO2".equals(targetStore.getName())) {\r
+ currentTargetFileName += getTmpExtension();\r
+ renameAfterFTP = true;\r
+ }\r
+ }\r
+\r
+ sourceStoreUri.transferFrom(targetStoreUri, sourceFileName, currentTargetFileName);\r
+\r
+ logger.info(getMarker(), "Transfer of {} completed from {} to {}", sourceFileName, sourceStoreUri, currentTargetFileName);\r
+\r
+ if (renameAfterCopy) {\r
+ Path tmpTargetFile = Paths.get(targetStoreUri.toString(true), currentTargetFileName);\r
+ Path targetFile = Paths.get(targetStoreUri.toString(true), targetFileName);\r
+ try {\r
+ logger.info(getMarker(), "Renaming LOCAL file from {} to {} on {}", currentTargetFileName, targetFileName, sourceStoreUri);\r
+ tmpTargetFile.toFile().renameTo(targetFile.toFile());\r
+ } catch (Exception e) {\r
+ logger.error(getMarker(), e.getMessage());\r
+ }\r
+ }\r
+ if (renameAfterFTP) {\r
+ try {\r
+ FtpDirectoryLister lister = (FtpDirectoryLister) targetStoreUri.getLister();\r
+ FTPClient client = lister.connect();\r
+ logger.info(getMarker(), "Renaming FTP file from {} to {} on {}", currentTargetFileName, targetFileName, sourceStoreUri);\r
+ client.rename(currentTargetFileName, targetFileName);\r
+ } catch (Exception e) {\r
+ logger.error(getMarker(), e.getMessage());\r
+ } finally {\r
+ targetStoreUri.cleanUp();\r
+ }\r
+ }\r
+ }\r
+\r
+ @StepEntry\r
+ public Object[] execute(StoreUri sourceStoreUri, String sourceFileName, StoreUri targetStoreUri, String targetFileName) throws Exception {\r
+ try {\r
+ getJobRuntime().setCancelable(false);\r
+\r
+ Store sourceStore = getManager().getStore(sourceStoreUri.getStoreId());\r
+ Store targetStore = getManager().getStore(targetStoreUri.getStoreId());\r
+\r
+ setDescription("{} -> {} : {}", sourceStore.getName(), targetStore.getName(), sourceFileName);\r
+\r
+ sourceStoreUri.addProgressListener(e -> setProgress(e.getProgress()));\r
+\r
+ StoreUri currentTargetStoreUri = getTargetStoreUri(targetStoreUri);\r
+\r
+ try {\r
+ RemoteFile remoteFile = currentTargetStoreUri.getRemoteFile(targetFileName);\r
+ if (remoteFile != null) {\r
+ logger.info("File {} already exists on target {}, skipping transfer", targetFileName, targetStore.getName());\r
+ }\r
+ } catch (Exception e) {\r
+ }\r
+\r
+ copy(sourceStoreUri, sourceFileName, currentTargetStoreUri, targetFileName);\r
+ } catch (Exception e) {\r
+ logger.error(getMarker(), "Error in transfer of {} when copying from {} to {}.", sourceFileName, sourceStoreUri, targetStoreUri);\r
+ throw e;\r
+ } finally {\r
+ getJobRuntime().setDescription(null);\r
+ if (sourceStoreUri != null)\r
+ sourceStoreUri.cleanUp();\r
+ if (targetStoreUri != null)\r
+ targetStoreUri.cleanUp();\r
+ }\r
+ return null;\r
+ }\r
+\r
+ protected StoreUri getTargetStoreUri(StoreUri targetStoreUri) {\r
+ return targetStoreUri;\r
+ }\r
+\r
+ protected String getTmpExtension() {\r
+ return DOT_PART;\r
+ }\r
+}\r
--- /dev/null
+package user.jobengine.server.steps;\r
+\r
+import java.util.List;\r
+\r
+import org.apache.logging.log4j.LogManager;\r
+import org.apache.logging.log4j.Logger;\r
+import org.apache.logging.log4j.Marker;\r
+\r
+import user.commons.remotestore.RemoteStoreProtocol;\r
+import user.jobengine.db.IItemManager;\r
+import user.jobengine.db.Media;\r
+import user.jobengine.db.MediaFile;\r
+import user.jobengine.db.Store;\r
+import user.jobengine.server.IJobEngine;\r
+import user.jobengine.server.IJobRuntime;\r
+import user.jobengine.server.steps.shared.MetadataType;\r
+import user.jobengine.server.steps.shared.MetadataTypeDetector;\r
+\r
+public class UpdateGhostMediaDataStep extends JobStep {\r
+ private static final Logger logger = LogManager.getLogger();\r
+ private Marker marker;\r
+\r
+ @StepEntry\r
+ public Object[] execute(Media mediaCubeMedia, IJobEngine jobEngine, IJobRuntime jobRuntime) throws Exception {\r
+ marker = jobRuntime.getSessionMarker();\r
+\r
+ IItemManager manager = jobEngine.getItemManager();\r
+ //Refresh from db\r
+ List<MediaFile> mediaFiles = manager.getMedia(mediaCubeMedia.getId()).getMediaFiles();\r
+ if (mediaFiles != null && mediaFiles.size() == 2) {\r
+ MediaFile lowres = null;\r
+ MediaFile highres = null;\r
+\r
+ for (MediaFile mf : mediaFiles) {\r
+ if (mf.getStore().getSourceStoreUri(RemoteStoreProtocol.HTTP) != null)\r
+ lowres = mf;\r
+ else\r
+ highres = mf;\r
+ }\r
+\r
+ if (highres == null) {\r
+ logger.info(marker, "Nincs highres mediaId: {}", mediaCubeMedia.getId());\r
+ return null;\r
+ }\r
+ if (lowres == null) {\r
+ logger.info(marker, "Nincs lowres mediaId: {}", mediaCubeMedia.getId());\r
+ return null;\r
+ }\r
+\r
+ String id = MetadataTypeDetector.truncateExtension(highres.getRelativePath());\r
+ id = MetadataTypeDetector.truncateVersion(id);\r
+ boolean detect = MetadataTypeDetector.GuessMetadataType(id) == MetadataType.OctopusPlaceholder\r
+ || MetadataTypeDetector.GuessMetadataType(id) == MetadataType.OctopusStory;\r
+ if (!detect) {\r
+ logger.info(marker, "Nem bejátszó mediaId: {}, file: {}", mediaCubeMedia.getId(), highres.getRelativePath());\r
+ return null;\r
+ }\r
+\r
+ Store highresStore = manager.getSystemStore(false);\r
+ final long sourceMediaId = lowres.getId();\r
+ final long highresMediaFileId = highres.getId();\r
+ final String highresRealtivePath = highres.getRelativePath();\r
+\r
+ manager.executeQuery("SELECT mediaid FROM mediafile WHERE relativepath=? and storeid=? and id!=?", rs -> {\r
+ long mediaId = rs.getLong(1);\r
+ Media media = manager.getMedia(mediaId);\r
+ if (media.getMediaFilesCount() == 1) {\r
+ logger.info(marker, "Hiányzó szellem lowres hozzáadása {} alapján", media.getId());\r
+\r
+ MediaFile mf = (MediaFile) manager.get(MediaFile.class, sourceMediaId);\r
+ mf.setMedia(media);\r
+ mf.setId(0);\r
+ manager.add(mf);\r
+ media.setLength(mediaCubeMedia.getLength());\r
+ manager.modify(media);\r
+ }\r
+ return true;\r
+ }, st -> {\r
+ st.setString(1, highresRealtivePath);\r
+ st.setLong(2, highresStore.getId());\r
+ st.setLong(3, highresMediaFileId);\r
+ });\r
+\r
+ }\r
+\r
+ return null;\r
+ }\r
+\r
+}\r
--- /dev/null
+package user.jobengine.server.steps.shared;\r
+\r
+import java.io.ByteArrayInputStream;\r
+import java.io.File;\r
+import java.io.IOException;\r
+import java.io.InputStream;\r
+import java.io.OutputStream;\r
+import java.io.StringWriter;\r
+import java.io.UnsupportedEncodingException;\r
+import java.nio.file.Files;\r
+import java.nio.file.Path;\r
+import java.nio.file.Paths;\r
+import java.nio.file.attribute.FileAttribute;\r
+import java.nio.file.attribute.PosixFilePermission;\r
+import java.nio.file.attribute.PosixFilePermissions;\r
+import java.text.SimpleDateFormat;\r
+import java.util.Calendar;\r
+import java.util.Date;\r
+import java.util.Set;\r
+\r
+import javax.xml.parsers.DocumentBuilder;\r
+import javax.xml.parsers.DocumentBuilderFactory;\r
+import javax.xml.transform.OutputKeys;\r
+import javax.xml.transform.Transformer;\r
+import javax.xml.transform.TransformerConfigurationException;\r
+import javax.xml.transform.TransformerException;\r
+import javax.xml.transform.TransformerFactory;\r
+import javax.xml.transform.TransformerFactoryConfigurationError;\r
+import javax.xml.transform.dom.DOMSource;\r
+import javax.xml.transform.stream.StreamResult;\r
+\r
+import org.apache.commons.lang.StringUtils;\r
+import org.apache.commons.net.ftp.FTPClient;\r
+import org.apache.logging.log4j.LogManager;\r
+import org.apache.logging.log4j.Logger;\r
+import org.apache.logging.log4j.Marker;\r
+import org.apache.logging.log4j.message.Message;\r
+import org.w3c.dom.DOMImplementation;\r
+import org.w3c.dom.Document;\r
+import org.w3c.dom.Element;\r
+\r
+import com.ibm.nosql.json.api.BasicDBObject;\r
+import com.ibm.nosql.json.util.JSON;\r
+\r
+import user.commons.CalendarUtils;\r
+import user.commons.MediaCubeMarker;\r
+import user.commons.StoreUri;\r
+import user.commons.remotestore.FtpDirectoryLister;\r
+\r
+public class EscortFiles {\r
+ private static final String RECORDTIMESTAMP = "RecordTimeStamp";\r
+ private static final String MODIFIEDTIMESTAMP = "ModifiedTimeStamp";\r
+ public static final String DOT_CATCHED = ".catched";\r
+ public static final String DOT_JSON = ".json";\r
+ private static final Logger logger = LogManager.getLogger();\r
+ private static final String EXTENDEDAGENCY = "ExtendedAgency";\r
+ private static final String EXTENDEDDESCRIPTION = "ExtendedDescription";\r
+ private static final String KILLDATE = "KillDate";\r
+ private static final String FORMAT_KILLDATE = "MM-dd-yyyy";\r
+ private static final String EXTENDEDID = "extendedId";\r
+ private static final String ID = "ID";\r
+ private static final String KILLDATE_FILENAME = "%s.%s.killdate";\r
+ private static final String FORMAT_KILLDATENAME = "yyyyMMdd";\r
+ public static final String STATUSFOLDER = ".STATUS";\r
+ public static final String CONFLICTFOLDER = ".CONFLICT";\r
+\r
+ public static String composeKillDate(int days) {\r
+ Calendar killDate = Calendar.getInstance();\r
+ killDate.add(Calendar.DAY_OF_YEAR, days);\r
+ SimpleDateFormat dateFormat = new SimpleDateFormat(FORMAT_KILLDATENAME);\r
+ return dateFormat.format(killDate.getTime());\r
+ }\r
+ /*\r
+ * <?xml version="1.0" encoding="UTF-16"?> <ID extendedId="aaaaaaa">\r
+ * <KillDate>02-02-2018</KillDate> <ExtendedDescription>TEST\r
+ * TEST</ExtendedDescription> <ExtendedAgency>AGENT AGENT</ExtendedAgency> </ID>\r
+ */\r
+\r
+ private static String composeKillDateFileName(String fileName, int days) {\r
+ return String.format(KILLDATE_FILENAME, fileName, composeKillDate(days));\r
+ }\r
+\r
+ public static void createCatchedFile(Path escortFile) throws IOException {\r
+ Path catchedFilePath = Paths.get(escortFile.toString() + DOT_CATCHED);\r
+ Files.createFile(catchedFilePath);\r
+ }\r
+\r
+ public static void createFellow(String escortFile, String extension) throws IOException {\r
+ Files.copy(Paths.get(escortFile), Paths.get(escortFile + "." + extension));\r
+ }\r
+\r
+ /***\r
+ * A media eleresi utjan alapjan a .STATUS almappaban letrehozza a .catch fajlt.\r
+ *\r
+ * @param mediaFile\r
+ * @throws IOException\r
+ */\r
+ public static void createMediaCatch(Path mediaFile) throws IOException {\r
+ Path catchedFile = createMediaCathFilePath(mediaFile);\r
+ ensureUNCFolder(catchedFile.getParent());\r
+ Files.createFile(catchedFile);\r
+ }\r
+\r
+ public static Path createMediaCathFilePath(Path mediaFile) {\r
+ String fileName = mediaFile.getFileName().toString() + DOT_CATCHED;\r
+ return Paths.get(mediaFile.getParent().toString(), STATUSFOLDER, fileName);\r
+ }\r
+\r
+ public static void createMetadata(String filePath, String fileName, String metadata) throws IOException {\r
+ ensureUNCFolder(filePath, STATUSFOLDER);\r
+ String metadataFileName = fileName + DOT_JSON;\r
+ Path metadataPath = Paths.get(filePath, STATUSFOLDER, metadataFileName);\r
+ Files.write(metadataPath, metadata.getBytes());\r
+ }\r
+\r
+ public static boolean createMetadataIfNotExists(String filePath, String fileName, String metadata)\r
+ throws IOException {\r
+ boolean result = false;\r
+ if (!EscortFiles.isMetadataExists(filePath, fileName)) {\r
+ EscortFiles.createMetadata(filePath, fileName, metadata);\r
+ result = true;\r
+ }\r
+ return result;\r
+ }\r
+\r
+ public static void createMorpheusXML(String filePath, String fileName, String content) throws IOException {\r
+ ensureUNCFolder(filePath, STATUSFOLDER);\r
+ Path xmlPath = Paths.get(filePath, fileName);\r
+ if (Files.exists(xmlPath))\r
+ throw new IOException(String.format("Az '%s' állomány már létezik.", xmlPath));\r
+ Files.write(xmlPath, content.getBytes());\r
+ }\r
+\r
+ public static byte[] createNEXIODatesMeta(String fileName, Date recorded, Date modified) throws Exception {\r
+ DocumentBuilderFactory dbf = DocumentBuilderFactory.newInstance();\r
+ DocumentBuilder db = dbf.newDocumentBuilder();\r
+ DOMImplementation impl = db.getDOMImplementation();\r
+ Document xmlDocument = impl.createDocument(null, null, null);\r
+\r
+ Element root = xmlDocument.createElement(ID);\r
+ root.setAttribute(EXTENDEDID, fileName);\r
+ // <ModifiedTimeStamp>07-13-2020 (19:36:52)</ModifiedTimeStamp>\r
+ // <RecordTimeStamp>05-18-2013 (18:52:24)</RecordTimeStamp>\r
+ SimpleDateFormat df = new SimpleDateFormat("MM-dd-yyyy (HH:mm:ss)");\r
+ root.appendChild(xmlDocument.createElement(MODIFIEDTIMESTAMP))\r
+ .appendChild(xmlDocument.createTextNode(df.format(modified)));\r
+ root.appendChild(xmlDocument.createElement(RECORDTIMESTAMP))\r
+ .appendChild(xmlDocument.createTextNode(df.format(recorded)));\r
+ xmlDocument.appendChild(root);\r
+\r
+ return xmDocumentToString(xmlDocument);\r
+ }\r
+\r
+ public static byte[] createNEXIOKillDateFile(String fileName, Date killDate, String description, String agency)\r
+ throws Exception {\r
+ DocumentBuilderFactory dbf = DocumentBuilderFactory.newInstance();\r
+ DocumentBuilder db = dbf.newDocumentBuilder();\r
+ DOMImplementation impl = db.getDOMImplementation();\r
+ Document xmlDocument = impl.createDocument(null, null, null);\r
+\r
+ Element root = xmlDocument.createElement(ID);\r
+ root.setAttribute(EXTENDEDID, fileName);\r
+ if (killDate != null) {\r
+ String sKillDate = CalendarUtils.toString(CalendarUtils.createCalendar(killDate), FORMAT_KILLDATE);\r
+ root.appendChild(xmlDocument.createElement(KILLDATE)).appendChild(xmlDocument.createTextNode(sKillDate));\r
+ }\r
+\r
+ if (StringUtils.isNotBlank(description))\r
+ root.appendChild(xmlDocument.createElement(EXTENDEDDESCRIPTION))\r
+ .appendChild(xmlDocument.createTextNode(description));\r
+ if (StringUtils.isNotBlank(agency))\r
+ root.appendChild(xmlDocument.createElement(EXTENDEDAGENCY)).appendChild(xmlDocument.createTextNode(agency));\r
+ xmlDocument.appendChild(root);\r
+\r
+ return xmDocumentToString(xmlDocument);\r
+ }\r
+\r
+ public static Document createNEXIOMeta(byte[] content) throws Exception {\r
+ DocumentBuilderFactory dbf = DocumentBuilderFactory.newInstance();\r
+ DocumentBuilder db = dbf.newDocumentBuilder();\r
+ DOMImplementation impl = db.getDOMImplementation();\r
+ Document xmlDocument = null;\r
+\r
+ try (InputStream is = new ByteArrayInputStream(content)) {\r
+ xmlDocument = db.parse(is);\r
+ } catch (Exception e) {\r
+ logger.catching(e);\r
+ }\r
+\r
+ return xmlDocument;\r
+ }\r
+\r
+ public static void createUNCKillDate(String filePath, String fileName, int days, Marker marker) throws IOException {\r
+ ensureUNCFolder(filePath, STATUSFOLDER);\r
+ String killDateFileName = composeKillDateFileName(fileName, days);\r
+ Path killDatePath = Paths.get(filePath, STATUSFOLDER, killDateFileName);\r
+ if (Files.exists(killDatePath))\r
+ logger.warn(marker, "Az '{}' állomány már létezik.", killDatePath);\r
+ else\r
+ Files.createFile(killDatePath);\r
+ }\r
+\r
+ @SuppressWarnings("unchecked")\r
+ public static <T> T decode(Path escortFile) {\r
+ T result = null;\r
+ try {\r
+ byte[] bytes = Files.readAllBytes(escortFile);\r
+ String content = new String(bytes);\r
+ result = (T) JSON.parse(content);\r
+ } catch (Exception e) {\r
+ logger.error("Decode error. System message is: ", e.getMessage());\r
+ }\r
+ return result;\r
+ }\r
+\r
+ public static void ensureUNCFolder(Path filePath) throws IOException {\r
+ File folder = filePath.toFile();\r
+ if (!folder.exists() || !folder.isDirectory()) {\r
+ try {\r
+ Set<PosixFilePermission> perms = PosixFilePermissions.fromString("rwxrwxrwx");\r
+ FileAttribute<Set<PosixFilePermission>> attr = PosixFilePermissions.asFileAttribute(perms);\r
+ Files.createDirectories(filePath, attr);\r
+ } catch (Exception e) {\r
+ // logger.catching(e);\r
+ try {\r
+ Files.createDirectories(filePath);\r
+ } catch (Exception e1) {\r
+ logger.catching(e1);\r
+ throw e1;\r
+ }\r
+ }\r
+ }\r
+ }\r
+\r
+ public static void ensureUNCFolder(String filePath, String folderName) throws IOException {\r
+ Path statusPath = Paths.get(filePath, folderName);\r
+ ensureUNCFolder(statusPath);\r
+ }\r
+\r
+ public static boolean isCatchedFileExists(Path escortFile) {\r
+ Path catchedFilePath = Paths.get(escortFile.toString() + DOT_CATCHED);\r
+ return catchedFilePath.toFile().exists();\r
+ }\r
+\r
+ /***\r
+ * A media eleresi utjan alapjan a .STATUS almappaban vizsgalja .catch fajl\r
+ * letezeset.\r
+ *\r
+ * @param mediaFile\r
+ * @return\r
+ */\r
+ public static boolean isMediaCatched(Path mediaFile) {\r
+ Path catchedFile = createMediaCathFilePath(mediaFile);\r
+ return catchedFile.toFile().exists();\r
+ }\r
+\r
+ public static boolean isMetadataExists(String filePath, String fileName) throws IOException {\r
+ boolean result = false;\r
+ String metadataFileName = fileName + DOT_JSON;\r
+ Path metadataPath = Paths.get(filePath, STATUSFOLDER, metadataFileName);\r
+ result = metadataPath.toFile().exists();\r
+ return result;\r
+ }\r
+\r
+ public static void notifyRecipient(Path escortFile, Logger logger, Message msg) {\r
+ if (escortFile.toFile().exists()) {\r
+ try {\r
+ BasicDBObject downloadable = EscortFiles.decode(escortFile);\r
+ String recipientKey = "recipient";\r
+ if (downloadable.containsKey(recipientKey)) {\r
+ String recipient = downloadable.getString(recipientKey);\r
+ logger.info(new MediaCubeMarker(recipient, "MediaCube rendszerüzenet"), msg);\r
+ }\r
+ } catch (Exception e) {\r
+ logger.catching(e);\r
+ }\r
+\r
+ }\r
+ }\r
+\r
+ public static void remove(Path file) {\r
+ try {\r
+ file.toFile().delete();\r
+ } catch (Exception e) {\r
+ logger.error("Unable to delete {}", file.toAbsolutePath().toString());\r
+ }\r
+ }\r
+\r
+ public static void removeCatchedFile(Path escortFile) {\r
+ remove(Paths.get(escortFile.toString() + DOT_CATCHED));\r
+ }\r
+\r
+ /***\r
+ * A media eleresi utjan alapjan a .STATUS almappabol torli a .catch fajlt.\r
+ *\r
+ * @param mediaFile\r
+ * @throws IOException\r
+ */\r
+ public static void removeMediaCatch(Path mediaFile) {\r
+ Path catchedFile = createMediaCathFilePath(mediaFile);\r
+ remove(catchedFile);\r
+ }\r
+\r
+ public static void setNEXIOKillDate(int killDateDays, String targetFileName, String nexioAgency, StoreUri targetUri)\r
+ throws Exception {\r
+ OutputStream outStream = null;\r
+ try {\r
+ FTPClient targetFTP = ((FtpDirectoryLister) targetUri.getLister()).connect();\r
+ Calendar killDate = CalendarUtils.createCalendar(new Date());\r
+ killDate.add(Calendar.DAY_OF_YEAR, killDateDays);\r
+ if (targetFileName.toLowerCase().contains(".mxf"))\r
+ targetFileName = targetFileName.substring(0, targetFileName.lastIndexOf('.'));\r
+ byte[] killDateFile = EscortFiles.createNEXIOKillDateFile(targetFileName, killDate.getTime(), null,\r
+ nexioAgency);\r
+ String xml = targetFileName + ".xml";\r
+ outStream = targetFTP.storeFileStream(xml);\r
+ if (outStream == null) {\r
+ throw new NullPointerException(\r
+ "Can not open: " + targetFileName.substring(0, targetFileName.lastIndexOf('.')) + ".xml"\r
+ + " Reply:" + targetFTP.getReplyString());\r
+ }\r
+ outStream.write(killDateFile);\r
+ outStream.flush();\r
+ } catch (Exception e) {\r
+ throw e;\r
+ } finally {\r
+ if (outStream != null)\r
+ outStream.close();\r
+ targetUri.cleanUp();\r
+ }\r
+ }\r
+\r
+ private static byte[] xmDocumentToString(Document xmlDocument) throws TransformerFactoryConfigurationError,\r
+ TransformerConfigurationException, TransformerException, IOException, UnsupportedEncodingException {\r
+ DOMSource domSource = new DOMSource(xmlDocument);\r
+ TransformerFactory tf = TransformerFactory.newInstance();\r
+ Transformer transformer = tf.newTransformer();\r
+ transformer.setOutputProperty(OutputKeys.METHOD, "xml");\r
+ transformer.setOutputProperty(OutputKeys.ENCODING, "UTF-16");\r
+ transformer.setOutputProperty(OutputKeys.INDENT, "yes");\r
+ StringWriter sw = new StringWriter();\r
+ StreamResult sr = new StreamResult(sw);\r
+ transformer.transform(domSource, sr);\r
+ String result = sw.toString();\r
+ sw.close();\r
+ return result.getBytes("UTF-16");\r
+ }\r
+}\r
--- /dev/null
+package user.jobengine.server.steps.shared;\r
+\r
+import java.io.BufferedReader;\r
+import java.io.InputStreamReader;\r
+import java.util.ArrayList;\r
+import java.util.List;\r
+\r
+import org.apache.logging.log4j.LogManager;\r
+import org.apache.logging.log4j.Logger;\r
+\r
+public class ExternalCommand {\r
+ private static final Logger logger = LogManager.getLogger();\r
+ private ExternalProfile profile;\r
+\r
+ public ExternalCommand(ExternalProfile profile) {\r
+ this.profile = profile;\r
+ }\r
+\r
+ public String execute(String input, String output, boolean firstResponse, IExternalCallback responseCallBack) throws Exception {\r
+ List<String> arguments = getArguments(input, output);\r
+ List<String> command = new ArrayList<>();\r
+ command.add(profile.getExecutable());\r
+ command.addAll(arguments);\r
+\r
+ ProcessBuilder processBuilder = new ProcessBuilder();\r
+ processBuilder.command(command);\r
+\r
+ String result = null;\r
+ try {\r
+ logger.info("Executing : {}", processBuilder.command());\r
+\r
+ Process process = processBuilder.start();\r
+ try (BufferedReader reader = new BufferedReader(new InputStreamReader(process.getInputStream()))) {\r
+ String line = null;\r
+ while ((line = reader.readLine()) != null) {\r
+ logger.debug("Process response: {}", line);\r
+ if (responseCallBack != null)\r
+ responseCallBack.onResponse(line);\r
+ //System.out.println(line);\r
+ if (line != null && line.length() > 0) {\r
+ result = line;\r
+ if (firstResponse)\r
+ break;\r
+ }\r
+ }\r
+ int exitCode = process.waitFor();\r
+ if (exitCode != 0) {\r
+ StringBuilder msg = new StringBuilder();\r
+ try (BufferedReader errReader = new BufferedReader(new InputStreamReader(process.getErrorStream()))) {\r
+ String errline = null;\r
+ while ((errline = errReader.readLine()) != null) {\r
+ msg.append(errline);\r
+ }\r
+ } catch (Exception ex) {\r
+ }\r
+\r
+ throw new Exception("Exited with error code : " + exitCode + ". " + msg);\r
+ }\r
+ } catch (Exception e) {\r
+ throw e;\r
+ }\r
+ } catch (Exception e) {\r
+ logger.error(e);\r
+ throw e;\r
+ }\r
+\r
+ return result;\r
+ }\r
+\r
+ private List<String> getArguments(String input, String output) {\r
+ List<String> result = new ArrayList<>();\r
+\r
+ profile.getArguments().forEach(i -> {\r
+ result.add(i.replace("%i", input).replace("%o", output));\r
+ });\r
+ return result;\r
+ }\r
+\r
+}
\ No newline at end of file
--- /dev/null
+package user.jobengine.server.steps.shared;\r
+\r
+import user.commons.configuration.SystemConfiguration;\r
+\r
+public class ExternalCommandExecutor {\r
+\r
+ public void execute(String profileName, String input, String output, IExternalCallback responseCallBack) throws Exception {\r
+ ExternalCommand externalCommand = getExternalCommand(profileName);\r
+ externalCommand.execute(input, output, false, responseCallBack);\r
+ }\r
+\r
+ private ExternalCommand getExternalCommand(String profileName) throws Exception {\r
+ ExternalProfilesConfig config = SystemConfiguration.getInstance().load("settings/external-commands.yaml", ExternalProfilesConfig.class);\r
+\r
+ if (config == null)\r
+ throw new Exception("Missing external-commands.yaml configuration");\r
+\r
+ ExternalProfile selectedProfile = null;\r
+ for (ExternalProfile profile : config.getProfiles()) {\r
+ if (profileName.equals(profile.getName())) {\r
+ selectedProfile = profile;\r
+ break;\r
+ }\r
+ }\r
+\r
+ if (selectedProfile == null)\r
+ throw new Exception("Missing profile " + profileName + " in external-commands.yaml configuration");\r
+\r
+ return new ExternalCommand(selectedProfile);\r
+ }\r
+\r
+}\r
--- /dev/null
+package user.jobengine.server.steps.shared;\r
+\r
+import java.util.List;\r
+\r
+public class ExternalProfile {\r
+ private String executable;\r
+ private String name;\r
+ private List<String> arguments;\r
+\r
+ public List<String> getArguments() {\r
+ return arguments;\r
+ }\r
+\r
+ public String getExecutable() {\r
+ return executable;\r
+ }\r
+\r
+ public String getName() {\r
+ return name;\r
+ }\r
+\r
+ public void setArguments(List<String> arguments) {\r
+ this.arguments = arguments;\r
+ }\r
+\r
+ public void setExecutable(String executable) {\r
+ this.executable = executable;\r
+ }\r
+\r
+ public void setName(String name) {\r
+ this.name = name;\r
+ }\r
+}
\ No newline at end of file
--- /dev/null
+package user.jobengine.server.steps.shared;\r
+\r
+import java.util.List;\r
+\r
+public class ExternalProfilesConfig {\r
+ private List<ExternalProfile> profiles;\r
+\r
+ public List<ExternalProfile> getProfiles() {\r
+ return profiles;\r
+ }\r
+\r
+ public void setProfiles(List<ExternalProfile> profiles) {\r
+ this.profiles = profiles;\r
+ }\r
+}
\ No newline at end of file
--- /dev/null
+package user.jobengine.server.steps.shared;\r
+\r
+public interface IExternalCallback {\r
+ void onResponse(String data);\r
+}
\ No newline at end of file
--- /dev/null
+package user.jobengine.server.steps.shared;\r
+\r
+import java.nio.file.Path;\r
+\r
+import com.ibm.nosql.json.api.BasicDBObject;\r
+\r
+import user.commons.RemoteFile;\r
+import user.commons.StoreUri;\r
+import user.commons.remotestore.RemoteStoreProtocol;\r
+import user.jobengine.db.IItemManager;\r
+import user.jobengine.db.IResultSetConsumer;\r
+import user.jobengine.db.Store;\r
+\r
+public class ItemManagerExtensions {\r
+\r
+ public static BasicDBObject getArchiveInfo(IItemManager manager, long houseid) {\r
+ final BasicDBObject[] result = { null };\r
+ StringBuilder query = new StringBuilder();\r
+ query.append("select count(*) as count, sum(length) as duration FROM media");\r
+ query.append(" ");\r
+ query.append(String.format("where houseid='%d' and itemtypeid = 82", houseid));\r
+ query.append(" ");\r
+ query.append("group by houseid");\r
+ IResultSetConsumer consumer = rs -> {\r
+ BasicDBObject o = new BasicDBObject();\r
+ o.put("count", rs.getLong("count"));\r
+ o.put("duration", rs.getLong("duration"));\r
+ result[0] = o;\r
+ return false;\r
+ };\r
+ manager.executeQuery(query.toString(), consumer, null);\r
+ return result[0];\r
+ }\r
+\r
+ public static long getExistingRundownMedia(IItemManager manager, String houseid) {\r
+ final long[] result = new long[] { 0 };\r
+ final String[] idToCheck = new String[] { houseid };\r
+ int pos = houseid.lastIndexOf("-");\r
+ //a hivas a CopyForArchiveNEXIOMaterialsStep-bol is johet, ott meg nincs idobelyegezve a nev!\r
+ if (pos > 0 && houseid.length() - pos > 4)\r
+ idToCheck[0] = houseid.substring(0, pos);\r
+ MetadataType metadataType = MetadataTypeDetector.GuessMetadataType(idToCheck[0]);\r
+ if (metadataType == MetadataType.OctopusPlaceholder) {\r
+ StringBuilder query = new StringBuilder();\r
+ query.append("select mediaid, mediafilehouseid, filename");\r
+ query.append(" ");\r
+ query.append(String.format("from vw_rundown_items where mediafilehouseid like '%s%%'", idToCheck[0]));\r
+ query.append(" ");\r
+ query.append("order by filename, mediaid");\r
+ IResultSetConsumer consumer = rs -> {\r
+ String fileName = rs.getString("filename");\r
+ if (idToCheck[0].equals(fileName)) {\r
+ result[0] = rs.getLong("mediaid");\r
+ return false;\r
+ } else\r
+ return true;\r
+ };\r
+ manager.executeQuery(query.toString(), consumer, null);\r
+ }\r
+\r
+ return result[0];\r
+ }\r
+\r
+ static public boolean isArchived(IItemManager manager, Path filePath) {\r
+ boolean result = false;\r
+ String name = filePath.getFileName().toString();\r
+ String[] tsmName = new String[] { null };\r
+ String query = String.format("SELECT relativepath FROM MEDIAFILE WHERE houseid = '%s'", name);\r
+ manager.executeQuery(query, rs -> {\r
+ tsmName[0] = rs.getString("relativepath");\r
+ return false;\r
+ }, null);\r
+\r
+ Store tsmStore = manager.getSystemStore(false);\r
+ if (tsmStore == null)\r
+ throw new NullPointerException("A TSM bejegyzés nem található!");\r
+\r
+ StoreUri tsmStoreUri = tsmStore.getSourceStoreUri(RemoteStoreProtocol.TSM);\r
+ if (tsmStoreUri == null)\r
+ throw new NullPointerException("A TSM forrás elérése nem található!");\r
+\r
+ if (tsmName[0] != null) {\r
+ try {\r
+ RemoteFile remoteFile = tsmStoreUri.getRemoteFile(tsmName[0]);\r
+ result = remoteFile != null;\r
+ } catch (Exception e) {\r
+ result = false;\r
+ } finally {\r
+ tsmStoreUri.cleanUp();\r
+ }\r
+ }\r
+ return result;\r
+ }\r
+}\r
--- /dev/null
+package user.jobengine.server.steps.shared;\r
+\r
+import javax.ws.rs.client.Entity;\r
+import javax.ws.rs.client.Invocation.Builder;\r
+import javax.ws.rs.core.MediaType;\r
+import javax.ws.rs.core.MultivaluedMap;\r
+import javax.ws.rs.core.Response;\r
+import javax.ws.rs.core.Response.Status;\r
+\r
+import org.apache.logging.log4j.LogManager;\r
+import org.apache.logging.log4j.Logger;\r
+import org.jboss.resteasy.client.jaxrs.ResteasyClient;\r
+import org.jboss.resteasy.client.jaxrs.ResteasyClientBuilder;\r
+import org.jboss.resteasy.client.jaxrs.ResteasyWebTarget;\r
+import org.jboss.resteasy.specimpl.MultivaluedMapImpl;\r
+\r
+import com.ibm.nosql.json.JSONUtil;\r
+import com.ibm.nosql.json.api.BasicDBObject;\r
+\r
+public class MediaCubeClient {\r
+ private static Logger logger = LogManager.getLogger();\r
+ private ResteasyWebTarget webTarget;\r
+\r
+ public MediaCubeClient(String address) {\r
+ ResteasyClient client = new ResteasyClientBuilder().build();\r
+ webTarget = client.target(address);\r
+ }\r
+\r
+ BasicDBObject getDbObject(String json) {\r
+ BasicDBObject result = (BasicDBObject) JSONUtil.jsonToDbObject(json);\r
+\r
+ if (result == null)\r
+ throw new NullPointerException("API Result is null!");\r
+\r
+ if (result.containsKey("exception")) {\r
+ BasicDBObject e = (BasicDBObject) result.get("exception");\r
+ throw new RuntimeException(e.getString("message"));\r
+ }\r
+ //{"exception":{"message":"Invalid credentials.","publicName":"AuthenticationFailedException"}}\r
+ return result;\r
+ }\r
+\r
+ public BasicDBObject getStatus(long jobId) {\r
+ MultivaluedMap<String, Object> vars = new MultivaluedMapImpl<>();\r
+ vars.add("jobId", jobId);\r
+ Response response = query("services/rest/jobengine/jobstatus", vars).get();\r
+ if (response.getStatus() != Status.OK.getStatusCode()) {\r
+ logger.error(response.readEntity(String.class));\r
+ System.out.println(response.readEntity(String.class));\r
+ return null;\r
+ }\r
+ String result = response.readEntity(String.class);\r
+ return getDbObject(result);\r
+ }\r
+\r
+ private Builder query(String path, MultivaluedMap<String, Object> vars) {\r
+ ResteasyWebTarget target = webTarget.path(path).queryParams(vars);\r
+ Builder result = target.request();\r
+ return result;\r
+ }\r
+\r
+ public long startjob(String template, String name, BasicDBObject jobParams) throws Exception {\r
+ MultivaluedMap<String, Object> vars = new MultivaluedMapImpl<>();\r
+ vars.add("template", template);\r
+ vars.add("name", name);\r
+ Response response = query("services/rest/jobengine/startjob", vars).post(Entity.entity(jobParams.toString(), MediaType.APPLICATION_JSON));\r
+\r
+ if (response.getStatus() != Status.OK.getStatusCode()) {\r
+ logger.error(response.readEntity(String.class));\r
+ return 0;\r
+ }\r
+\r
+ String resultObject = response.readEntity(String.class);\r
+ return Long.parseLong(resultObject);\r
+ }\r
+\r
+}\r
--- /dev/null
+<?xml version="1.0" encoding="UTF-8"?>\r
+<jobtemplate>\r
+ <commands>\r
+ <calljobstep remote="true" type="IntegrationTestStep.java" weight="1" />\r
+ </commands>\r
+</jobtemplate>
\ No newline at end of file
--- /dev/null
+<?xml version="1.0" encoding="UTF-8"?>\r
+<jobtemplate name="Metaadat szinkronizálás">\r
+ <commands>\r
+ <calljobstep type="MetadataUpdater.java" />\r
+ </commands>\r
+</jobtemplate>
\ No newline at end of file
--- /dev/null
+<?xml version="1.0" encoding="UTF-8"?>\r
+<jobtemplate name="Rekurzív törlés" useSessionLog="false">\r
+ <declarations>\r
+ <parameters>\r
+ <parameter name="sourcePath" type="java.lang.String" />\r
+ </parameters>\r
+ </declarations>\r
+ <commands>\r
+ <calljobstep id="step1" type="SafeDeleteRecursive.java" weight="1">\r
+ <inputs>\r
+ <input>\r
+ <parameter name="sourcePath" />\r
+ </input>\r
+ </inputs>\r
+ </calljobstep>\r
+ </commands>\r
+</jobtemplate>
\ No newline at end of file
\r
import java.io.File;\r
import java.io.IOException;\r
-import java.nio.file.Files;\r
-import java.nio.file.attribute.BasicFileAttributes;\r
import java.sql.Timestamp;\r
import java.util.Date;\r
import java.util.List;\r
\r
File sourceMediaFile = new File(archiveItem.getMediaFile());\r
String sourceFileName = sourceMediaFile.getName();\r
+ long fileSize = sourceMediaFile.length();\r
+\r
try {\r
Timecode timecode = new Timecode(mediaCubeMedia.getLength(), Type.PAL);\r
- String details = String.format("%s (%s, %d bytes)", sourceFileName, timecode.toString(), sourceMediaFile.length());\r
+ String details = String.format("%s (%s, %d bytes)", sourceFileName, timecode.toString(), fileSize);\r
+ logger.info(marker, details);\r
getJobRuntime().setDescription(details);\r
} catch (Exception e) {\r
- String details = String.format("%s (%d bytes)", sourceFileName, sourceMediaFile.length());\r
+ String details = String.format("%s (%d bytes)", sourceFileName, fileSize);\r
getJobRuntime().setDescription(details);\r
}\r
Timecode timecode = new Timecode(mediaCubeMedia.getLength(), Type.PAL);\r
- String details = String.format("%s (%s, %d bytes)", sourceFileName, timecode.toString(), sourceMediaFile.length());\r
+ String details = String.format("%s (%s, %d bytes)", sourceFileName, timecode.toString(), fileSize);\r
getJobRuntime().setDescription(details);\r
try {\r
\r
setAndCheck(archiveItem, mediaCubeMedia, getEngine());\r
\r
- //TODO mxf helyett az osszes kiterjesztest!!!!!\r
- //A dupla ellenorzes a napon beluli ismetlesek miatt kell\r
+ // TODO mxf helyett az osszes kiterjesztest!!!!!\r
+ // A dupla ellenorzes a napon beluli ismetlesek miatt kell\r
long existingMediaId = archiveItem.getExistingMediaId();\r
\r
- if (sourceMediaFile.length() == 0 && existingMediaId == 0) {\r
- existingMediaId = ItemManagerExtensions.getExistingRundownMedia(manager, sourceFileName.replace(MXFEXT, ""));\r
+ if (fileSize == 0 && existingMediaId == 0) {\r
+ existingMediaId = ItemManagerExtensions.getExistingRundownMedia(manager,\r
+ sourceFileName.replace(MXFEXT, ""));\r
if (existingMediaId == 0)\r
existingMediaId = -1;\r
}\r
\r
if (existingMediaId == 0)\r
- existingMediaId = ItemManagerExtensions.getExistingRundownMedia(manager, sourceFileName.replace(MXFEXT, ""));\r
+ existingMediaId = ItemManagerExtensions.getExistingRundownMedia(manager,\r
+ sourceFileName.replace(MXFEXT, ""));\r
String targetFileName;\r
if (RANDOMIZE_ARCHIVES) {\r
// a-z, A-Z, 0-9. For example: WRMcpIk7, s57JwCVA\r
- //veletlenszeru neveket adunk!\r
+ // veletlenszeru neveket adunk!\r
targetFileName = String.format("%s-%s", RandomStringUtils.randomAlphanumeric(8), sourceFileName);\r
} else\r
targetFileName = sourceFileName;\r
\r
if (existingMediaId == 0) {\r
- StoreUri sourceUri = manager.createStoreUri(RemoteStoreProtocol.LOCAL, sourceMediaFile.getParent().toString());\r
+ StoreUri sourceUri = manager.createStoreUri(RemoteStoreProtocol.LOCAL,\r
+ sourceMediaFile.getParent().toString());\r
\r
final IJobRuntime runtime = getJobRuntime();\r
sourceUri.addProgressListener(new IProgressEventListener() {\r
}\r
\r
if (existingMediaId > 0)\r
- logger.info(marker, "Az '{}' TSM mentése nem szükséges, mert már megtalálható az archívumban.", sourceFileName);\r
+ logger.info(marker, "Az '{}' TSM mentése nem szükséges, mert már megtalálható az archívumban.",\r
+ sourceFileName);\r
\r
- //Fel kell szabadítani, hogy a kovetkezo archivalaskor is nekifusson\r
+ // Fel kell szabadítani, hogy a kovetkezo archivalaskor is nekifusson\r
if (existingMediaId == -1) {\r
- logger.info(marker, "Az '{}' mentése jelenleg nem lehetséges, mert a szükséges metaadat még nem található meg az archívumban.", sourceFileName);\r
+ logger.info(marker,\r
+ "Az '{}' mentése jelenleg nem lehetséges, mert a szükséges metaadat még nem található meg az archívumban.",\r
+ sourceFileName);\r
if (!archiveItem.removeCatchedFile())\r
logger.error(marker,\r
"Az '{}' állomány .catched jelző állománya nem törölhető. Az újabb archiválási kísérlethez annak kézi eltávolítása szükséges!",\r
sourceMediaFile.getName());\r
mediaCubeMedia.remove();\r
} else {\r
- saveMetadata(mediaCubeMedia, sourceMediaFile, targetFileName, existingMediaId);\r
+ saveMetadata(mediaCubeMedia, sourceMediaFile, targetFileName, existingMediaId, fileSize,\r
+ archiveItem.isDisableProxy());\r
logger.info(marker, "Az '{}' archiválása sikeres.", sourceFileName);\r
if (killDateDays != 0)\r
EscortFiles.createUNCKillDate(sourceMediaFile.getParent(), sourceFileName, killDateDays, marker);\r
\r
} catch (Exception e) {\r
logger.catching(e);\r
- Message m = new ParameterizedMessage("Az '{}' állomány archiválása sikertelen. A rendszer hibaüzenete: {}", details, e.getMessage());\r
+ Message m = new ParameterizedMessage("Az '{}' állomány archiválása sikertelen. A rendszer hibaüzenete: {}",\r
+ details, e.getMessage());\r
logger.error(marker, m);\r
if (!archiveItem.removeCatchedFile())\r
logger.error(marker,\r
return null;\r
}\r
\r
- private void saveMetadata(Media mediaCubeMedia, File sourceFile, String targetFileName, long existingMediaId) {\r
+ private void saveMetadata(Media mediaCubeMedia, File sourceFile, String targetFileName, long existingMediaId,\r
+ long fileSize, boolean disableProxy) {\r
\r
if (existingMediaId == 0) {\r
- MediaFile mediaFile = manager.createMediaFile(targetFileName, fileType, tsmStore, mediaCubeMedia);\r
- mediaFile.setHouseId(sourceFile.getName());\r
- mediaFile.add();\r
+ MediaFile mf = manager.createMediaFile(targetFileName, fileType, tsmStore, mediaCubeMedia);\r
+ mf.setHouseId(sourceFile.getName());\r
+ mf.setFileSize(fileSize);\r
+ // 210617 proxy keszites tiltasa\r
+ mf.setDisableProxy(disableProxy);\r
+ mf.add();\r
} else {\r
Media existingMedia = manager.getMedia(existingMediaId);\r
List<MediaFile> mediaFiles = existingMedia.getMediaFiles();\r
mf.setPersister(manager);\r
mf.setId(0);\r
mf.setMedia(mediaCubeMedia);\r
+ // mivel itt masolat keszul, nem allitunk at semmit\r
+ // mf.setFileSize(fileSize);\r
+ // mf.setDisableProxy(disableProxy);\r
mf.add();\r
}\r
}\r
}\r
mediaCubeMedia.setPersister(manager);\r
\r
+ // 210614 megis maradjon az aktualis idopont\r
+ mediaCubeMedia.setArchived(new Timestamp(new Date().getTime()));\r
+\r
+ /*\r
try {\r
BasicFileAttributes attr = Files.readAttributes(sourceFile.toPath(), BasicFileAttributes.class);\r
mediaCubeMedia.setArchived(new Timestamp(attr.creationTime().toMillis()));\r
} catch (IOException e) {\r
logger.catching(e);\r
- mediaCubeMedia.setArchived(new Timestamp(new Date().getTime()));\r
}\r
-\r
+ */\r
mediaCubeMedia.modify();\r
}\r
\r
- private void setAndCheck(ArchiveItem archiveItem, Media mediaCubeMedia, IJobEngine jobEngine) throws JobEngineException, IOException {\r
+ private void setAndCheck(ArchiveItem archiveItem, Media mediaCubeMedia, IJobEngine jobEngine)\r
+ throws JobEngineException, IOException {\r
if (jobEngine == null) {\r
logger.error(marker, "Az folyamatkezelő réteg nem elérhető.");\r
throw new NullPointerException("Internal error, missing JobEngine reference.");\r
sourceMediaFile = new File(archiveItem.getMediaFile());\r
if (sourceMediaFile == null) {\r
logger.error(marker, "A folyamat 'archiveItem' bemeneti paraméter 'mediaFile' értéke üres.");\r
- throw new NullPointerException("System is not configured properly, missing 'mediaFile' value in 'archiveItem' input parameter.");\r
+ throw new NullPointerException(\r
+ "System is not configured properly, missing 'mediaFile' value in 'archiveItem' input parameter.");\r
}\r
if (!sourceMediaFile.exists()) {\r
logger.error(marker, "Az {} állomány nem létezik vagy nem érhető el.", sourceMediaFile.getName());\r
}\r
if (mediaCubeMedia == null) {\r
logger.error(marker, "A folyamat 'mediaCubeMedia' bemeneti paramétere üres.");\r
- throw new NullPointerException("System is not configured properly, 'mediaCubeMedia' input parameter missing.");\r
+ throw new NullPointerException(\r
+ "System is not configured properly, 'mediaCubeMedia' input parameter missing.");\r
}\r
\r
}\r
client = new TSMBufferedClient(NODENAME);\r
//client.connect(storeUri.getUserName(), storeUri.getPassword(), "\\");\r
client.connect(storeUri.getUserName(), storeUri.getPassword());\r
- logger.info("TSMBufferedClient connected, parameters: {} {}/{} {}, separator: {}", NODENAME, FSNAME, ALTERNATE_FSNAME, HLNAME, File.separator);\r
+ logger.info("TSMBufferedClient connected, parameters: {} {}/{} {}, separator: {}", NODENAME, FSNAME,\r
+ ALTERNATE_FSNAME, HLNAME, File.separator);\r
}\r
\r
@Override\r
connect();\r
String currentFileName = fileName;\r
logger.info("Getting {}, {}, {}", FSNAME, HLNAME, DELIMITER + currentFileName);\r
- TSMBackupFileObject backupFileObject = client.getActiveBackupFileObject(FSNAME, HLNAME, DELIMITER + currentFileName);\r
+ TSMBackupFileObject backupFileObject = client.getActiveBackupFileObject(FSNAME, HLNAME,\r
+ DELIMITER + currentFileName);\r
\r
//probaljuk meg a masik tarbol\r
if (backupFileObject == null) {\r
\r
@Override\r
public OutputStream getOutputStream(RemoteFile remoteFile) throws Exception {\r
- // try {\r
- // logger.info("Trying register filespace");\r
- // client.registerFilespace(FSNAME, "JOBENGINE", 'C', "JOBENGINE", 10L * 1024L * 1024L * 1024L, 0);\r
- // } catch (Exception e) {\r
- // logger.info(e.getMessage());\r
- // }\r
-\r
connect();\r
+\r
+ try {\r
+ logger.info("Trying register filespace");\r
+ client.registerFilespace(FSNAME, "JOBENGINE", 'C', "JOBENGINE", 10L * 1024L * 1024L * 1024L, 0);\r
+ } catch (Exception e) {\r
+ logger.info(e.getMessage());\r
+ }\r
+\r
TSMOutputStream outputStream = new TSMOutputStream(client, FSNAME, HLNAME, DELIMITER, remoteFile);\r
outputStream.open();\r
return outputStream;\r
private boolean success = true;\r
\r
//highLevelName == delimiter\r
- public TSMOutputStream(TSMBufferedClient client, String filespaceName, String highLevelName, String delimiter, RemoteFile remoteFile) {\r
+ public TSMOutputStream(TSMBufferedClient client, String filespaceName, String highLevelName, String delimiter,\r
+ RemoteFile remoteFile) {\r
this.client = client;\r
this.filespaceName = filespaceName;\r
this.highLevelName = highLevelName;\r
Connection connection = null;\r
DefaultContext context = null;\r
try {\r
- DataSource dataSource = MediaCubeDataSourceFactory.getNoSQLDatasource();\r
+ DataSource dataSource = MediaCubeDataSourceFactory.getDatasource();\r
connection = dataSource.getConnection();\r
context = new DefaultContext(connection);\r
} catch (Throwable t) {\r
private boolean missingProxy;\r
private long mediaId;\r
private int fromResult;\r
-\r
private int toResult;\r
+ private boolean countOnly;\r
\r
public StringBuilder getCriteriaForTag() {\r
StringBuilder result = null;\r
return this;\r
}\r
\r
+ public boolean isCountOnly() {\r
+ return countOnly;\r
+ }\r
+\r
+ public void setCountOnly(boolean countOnly) {\r
+ this.countOnly = countOnly;\r
+ }\r
+\r
}\r
\r
String query = sql.toString();\r
logger.info(query);\r
- PreparedStatement st = connection.prepareStatement(query, ResultSet.TYPE_SCROLL_INSENSITIVE,\r
+\r
+ PreparedStatement st = connection.prepareStatement(query, ResultSet.TYPE_FORWARD_ONLY,\r
ResultSet.CONCUR_UPDATABLE);\r
\r
// Ettol elszall a 11.5-on a kereses\r
DefaultContext context = manager.getDbContext();\r
Connection connection = context.getConnection();\r
try {\r
+ int maxRecords = options.getToResult() - options.getFromResult();\r
+\r
st = createStatement(connection, options.getText(), options);\r
+\r
+ logger.info("Executing query, fetch size is {}, automcommit {}", st.getFetchSize(),\r
+ connection.getAutoCommit());\r
rs = st.executeQuery();\r
+ logger.info("Checking size");\r
rs.last();\r
result.setItemCount(rs.getRow());\r
+ logger.info("Result size is {}", result.getItemCount());\r
\r
// set begining position\r
if (options.getFromResult() > -1) {\r
rs.beforeFirst();\r
}\r
\r
+ List<ArchivedMedia> resultList = null;\r
+ logger.info("Processing query result, maxRecords {}", maxRecords);\r
+ while (rs.next() && maxRecords > 0) {\r
+ ItemType it = new ItemType();\r
+ it.setId(rs.getLong("mediaitemtypeid"));\r
+ it.setName(rs.getString("mediaitemtypename"));\r
+\r
+ Item item = new Item();\r
+ item.setId(rs.getLong("itemid"));\r
+ item.setTitle(rs.getString("itemtitle"));\r
+\r
+ Media media = new Media();\r
+\r
+ media.setItemId(item.getId());\r
+ media.setItemType(it);\r
+ media.setId(rs.getLong("mediaid"));\r
+ media.setTitle(rs.getString("mediatitle"));\r
+ media.setHouseId(rs.getString("mediahouseid"));\r
+ media.setCreation(rs.getTimestamp("creation"));\r
+ media.setCreated(rs.getTimestamp("created"));\r
+ media.setModified(rs.getTimestamp("modified"));\r
+ media.setArchived(rs.getTimestamp("archived"));\r
+ media.setLength(rs.getLong("length"));\r
+ media.setMediaFilesCount(rs.getInt("mediafilecount"));\r
+ media.setMediaFilesName(rs.getString("mediafilehouseid"));\r
+\r
+ ArchivedMedia am = new ArchivedMedia();\r
+ am.setMedia(media);\r
+ am.setItem(item);\r
+\r
+ if (resultList == null)\r
+ resultList = new ArrayList<>();\r
+ resultList.add(am);\r
+\r
+ am.setRelevant(rs.getString("description"));\r
+ // setRelevant(criterias, am, rs.getString("description"));\r
+ maxRecords--;\r
+ }\r
+ logger.info("Query result processed");\r
+ result.setItems(resultList);\r
+ connection.commit();\r
+ } catch (Exception e) {\r
+ try {\r
+ connection.rollback();\r
+ } catch (Exception e1) {\r
+ e1.printStackTrace();\r
+ }\r
+ manager.throwError(e);\r
+ } finally {\r
+ try {\r
+\r
+ if (rs != null)\r
+ rs.close();\r
+ } catch (Exception e1) {\r
+ }\r
+ try {\r
+ if (st != null)\r
+ st.close();\r
+ } catch (Exception e1) {\r
+ }\r
+\r
+ manager.putDbContext(context);\r
+ }\r
+ manager.traceOut();\r
+\r
+ return result;\r
+ }\r
+\r
+ public SearchResult<ArchivedMedia> search_v1(SearchOptions options) {\r
+ SearchResult<ArchivedMedia> result = new SearchResult<>();\r
+\r
+ ResultSet rs = null;\r
+ PreparedStatement st = null;\r
+ DefaultContext context = manager.getDbContext();\r
+ Connection connection = context.getConnection();\r
+ try {\r
int maxRecords = options.getToResult() - options.getFromResult();\r
\r
+ st = createStatement(connection, options.getText(), options);\r
+\r
+ logger.info("Executing query, fetch size is {}, automcommit {}", st.getFetchSize(),\r
+ connection.getAutoCommit());\r
+ rs = st.executeQuery();\r
+ logger.info("Checking size");\r
+ rs.last();\r
+ result.setItemCount(rs.getRow());\r
+ logger.info("Result size is {}", result.getItemCount());\r
+\r
+ // set begining position\r
+ if (options.getFromResult() > -1) {\r
+ if (options.getFromResult() <= result.getItemCount()) {\r
+ if (options.getFromResult() > 1) {\r
+ rs.absolute(options.getFromResult() - 1);\r
+ } else {\r
+ rs.beforeFirst();\r
+ }\r
+ } else {\r
+ if (result.getItemCount() > 0) {\r
+ throw new Exception("Invalid searchResult 'from' position: " + options.getFromResult()\r
+ + " total records: " + result.getItemCount());\r
+ }\r
+ }\r
+ } else {\r
+ rs.beforeFirst();\r
+ }\r
+\r
List<ArchivedMedia> resultList = null;\r
logger.info("Processing query result, maxRecords {}", maxRecords);\r
while (rs.next() && maxRecords > 0) {\r
manager.throwError(e);\r
} finally {\r
try {\r
+\r
if (rs != null)\r
rs.close();\r
} catch (Exception e1) {\r
st.close();\r
} catch (Exception e1) {\r
}\r
+\r
manager.putDbContext(context);\r
}\r
manager.traceOut();\r
\r
@Override\r
public ArchivedMedia getElementAt(int index) {\r
- logger.info("getElementAt {}", index);\r
+ //logger.info("getElementAt {}", index);\r
ArchivedMedia ret = cache.get(index);\r
if (ret == null) {\r
loadCache(index);\r
int rowLimit = this.cacheSize; // 200\r
int startPos = 0;\r
if ((forIndex - halfSize) > 0) { // forIndex=60 startPos=0\r
- // cache=0-200, forIndex=150\r
+ // cache=0-200, forIndex=150\r
// startPos=50 cache=50-250\r
startPos = (forIndex - halfSize);\r
}\r
https://git.useribm.hu / mediacube.git / commitdiff