From 9c15259cb3e951605458f88286bc51028c2f2917 Mon Sep 17 00:00:00 2001 From: "vasary.daniel" Date: Tue, 10 Aug 2021 14:38:54 +0000 Subject: [PATCH] git-tfs-id: [http://tfs.userrendszerhaz.hu:8080/tfs/DefaultCollection]$/MediaCube;C32303 --- .../production/AMC/jobs/executors.xml | 36 ++ .../production/AMC/jobs/schedules.json | 146 +++++++ .../AMC/jobs/steps/ArchiveSubtitlesStep.java | 116 +++++ .../AMC/jobs/steps/CancelableStep.java | 35 ++ .../AMC/jobs/steps/DirMXFValidatorStep.java | 61 +++ .../AMC/jobs/steps/EscortFileUpdaterStep.java | 57 +++ .../steps/FILEZILLA_AVIDTransferToStep.java | 11 + .../FILEZILLA_PASARESTORETransferToStep.java | 85 ++++ .../AMC/jobs/steps/FileCleanupStep.java | 35 ++ .../jobs/steps/FileSearchFilterOptions.java | 41 ++ .../AMC/jobs/steps/FileValidatorStep.java | 146 +++++++ .../AMC/jobs/steps/ForkDownloadStep.java | 204 +++++++++ .../AMC/jobs/steps/ForkUploadStep.java | 243 +++++++++++ .../jobs/steps/GenericArchiveCheckerStep.java | 94 ++++ .../HarrisMissingMaterialCheckerStep.java | 172 ++++++++ .../AMC/jobs/steps/MetadataPersisterStep.java | 118 +++++ .../jobs/steps/NEXIO1TransferFromStep.java | 30 ++ .../AMC/jobs/steps/NEXIO1TransferToStep.java | 54 +++ .../jobs/steps/NEXIO2TransferFromStep.java | 30 ++ .../AMC/jobs/steps/NEXIO2TransferToStep.java | 53 +++ .../jobs/steps/NEXIOArchiveCheckerStep.java | 188 ++++++++ .../steps/NEXIOMetadataPersisterStep.java | 110 +++++ .../jobs/steps/PASAPOOLTransferToStep.java | 102 +++++ .../jobs/steps/PBARCHIVETransferFromStep.java | 21 + .../AMC/jobs/steps/PBArchiveCheckerStep.java | 151 +++++++ .../jobs/steps/PathItemsCollectorStep.java | 76 ++++ ...ableBeachMissingMaterialCheckerStep.210604 | 225 ++++++++++ ...PeableBeachMissingMaterialCheckerStep.java | 227 ++++++++++ .../jobs/steps/RegisterUserRestoreStep.java | 72 +++ .../jobs/steps/RegisterVODRestoreStep.java | 80 ++++ .../jobs/steps/RundownFilesCollectorStep.java | 42 ++ .../jobs/steps/ServerStatusReportStep.java | 39 ++ .../steps/SubtitleFilesCollectorStep.java | 60 +++ .../AMC/jobs/steps/TSMTransferFromStep.java | 27 ++ .../AMC/jobs/steps/TSMTransferToStep.java | 11 + .../jobs/steps/TestForkCancelableStep.java | 33 ++ .../AMC/jobs/steps/TransferStep.java | 100 +++++ .../AMC/jobs/steps/TransferToFTPStep.java | 11 + .../AMC/jobs/steps/VODTransferToStep.java | 22 + .../AMC/jobs/steps/shared/EscortFiles.java | 339 ++++++++++++++ .../AMC/jobs/steps/shared/PBQuery.java | 413 ++++++++++++++++++ .../FILEZILLA_AVID-validate-and-restore.xml | 95 ++++ ...ZILLA_PASARESTORE-validate-and-archive.xml | 94 ++++ ...ZILLA_PASARESTORE-validate-and-restore.xml | 95 ++++ .../templates/NEXIO1-validate-and-archive.xml | 94 ++++ .../templates/NEXIO1-validate-and-restore.xml | 95 ++++ .../templates/NEXIO2-validate-and-archive.xml | 94 ++++ .../templates/NEXIO2-validate-and-restore.xml | 95 ++++ .../PASAPOOL-validate-and-restore.xml | 95 ++++ .../AMC/jobs/templates/PB-archive-checker.xml | 26 ++ .../PBARCHIVE-validate-and-archive.xml | 85 ++++ .../templates/VOD-validate-and-restore.xml | 89 ++++ .../AMC/jobs/templates/cancelable.xml | 17 + .../jobs/templates/create-lowres-ondemand.xml | 106 +++++ .../templates/fork-validate-and-archive.xml | 29 ++ .../templates/fork-validate-and-restore.xml | 29 ++ .../templates/generic-archive-checker.xml | 34 ++ .../harris1-missingmaterial-checker.xml | 30 ++ .../harris2-missingmaterial-checker.xml | 30 ++ .../jobs/templates/nexio1-archive-checker.xml | 26 ++ .../jobs/templates/nexio2-archive-checker.xml | 26 ++ .../peablebeach-missingmaterial-checker.xml | 58 +++ .../jobs/templates/register-user-restore.xml | 29 ++ .../jobs/templates/register-vod-restore.xml | 21 + .../jobs/templates/server-status-report.xml | 6 + .../AMC/jobs/templates/sync-subtitles.xml | 40 ++ .../jobs/templates/test-fork-cancelable.xml | 6 + .../AMC/jobs/templates/validate-dir-mxf.xml | 17 + .../production/AMC/settings/application.yaml | 37 ++ .../-product/production/AMC/settings/dsm.opt | 6 + .../production/AMC/settings/jetty.xml | 35 ++ .../production/AMC/settings/log4j2-210810.xml | 202 +++++++++ .../production/AMC/settings/log4j2.xml | 209 +++++++++ .../production/AMC/settings/maestro.yaml | 38 ++ .../production/AMC/settings/mediacube.yaml | 55 +++ 75 files changed, 6159 insertions(+) create mode 100644 server/-product/production/AMC/jobs/executors.xml create mode 100644 server/-product/production/AMC/jobs/schedules.json create mode 100644 server/-product/production/AMC/jobs/steps/ArchiveSubtitlesStep.java create mode 100644 server/-product/production/AMC/jobs/steps/CancelableStep.java create mode 100644 server/-product/production/AMC/jobs/steps/DirMXFValidatorStep.java create mode 100644 server/-product/production/AMC/jobs/steps/EscortFileUpdaterStep.java create mode 100644 server/-product/production/AMC/jobs/steps/FILEZILLA_AVIDTransferToStep.java create mode 100644 server/-product/production/AMC/jobs/steps/FILEZILLA_PASARESTORETransferToStep.java create mode 100644 server/-product/production/AMC/jobs/steps/FileCleanupStep.java create mode 100644 server/-product/production/AMC/jobs/steps/FileSearchFilterOptions.java create mode 100644 server/-product/production/AMC/jobs/steps/FileValidatorStep.java create mode 100644 server/-product/production/AMC/jobs/steps/ForkDownloadStep.java create mode 100644 server/-product/production/AMC/jobs/steps/ForkUploadStep.java create mode 100644 server/-product/production/AMC/jobs/steps/GenericArchiveCheckerStep.java create mode 100644 server/-product/production/AMC/jobs/steps/HarrisMissingMaterialCheckerStep.java create mode 100644 server/-product/production/AMC/jobs/steps/MetadataPersisterStep.java create mode 100644 server/-product/production/AMC/jobs/steps/NEXIO1TransferFromStep.java create mode 100644 server/-product/production/AMC/jobs/steps/NEXIO1TransferToStep.java create mode 100644 server/-product/production/AMC/jobs/steps/NEXIO2TransferFromStep.java create mode 100644 server/-product/production/AMC/jobs/steps/NEXIO2TransferToStep.java create mode 100644 server/-product/production/AMC/jobs/steps/NEXIOArchiveCheckerStep.java create mode 100644 server/-product/production/AMC/jobs/steps/NEXIOMetadataPersisterStep.java create mode 100644 server/-product/production/AMC/jobs/steps/PASAPOOLTransferToStep.java create mode 100644 server/-product/production/AMC/jobs/steps/PBARCHIVETransferFromStep.java create mode 100644 server/-product/production/AMC/jobs/steps/PBArchiveCheckerStep.java create mode 100644 server/-product/production/AMC/jobs/steps/PathItemsCollectorStep.java create mode 100644 server/-product/production/AMC/jobs/steps/PeableBeachMissingMaterialCheckerStep.210604 create mode 100644 server/-product/production/AMC/jobs/steps/PeableBeachMissingMaterialCheckerStep.java create mode 100644 server/-product/production/AMC/jobs/steps/RegisterUserRestoreStep.java create mode 100644 server/-product/production/AMC/jobs/steps/RegisterVODRestoreStep.java create mode 100644 server/-product/production/AMC/jobs/steps/RundownFilesCollectorStep.java create mode 100644 server/-product/production/AMC/jobs/steps/ServerStatusReportStep.java create mode 100644 server/-product/production/AMC/jobs/steps/SubtitleFilesCollectorStep.java create mode 100644 server/-product/production/AMC/jobs/steps/TSMTransferFromStep.java create mode 100644 server/-product/production/AMC/jobs/steps/TSMTransferToStep.java create mode 100644 server/-product/production/AMC/jobs/steps/TestForkCancelableStep.java create mode 100644 server/-product/production/AMC/jobs/steps/TransferStep.java create mode 100644 server/-product/production/AMC/jobs/steps/TransferToFTPStep.java create mode 100644 server/-product/production/AMC/jobs/steps/VODTransferToStep.java create mode 100644 server/-product/production/AMC/jobs/steps/shared/EscortFiles.java create mode 100644 server/-product/production/AMC/jobs/steps/shared/PBQuery.java create mode 100644 server/-product/production/AMC/jobs/templates/FILEZILLA_AVID-validate-and-restore.xml create mode 100644 server/-product/production/AMC/jobs/templates/FILEZILLA_PASARESTORE-validate-and-archive.xml create mode 100644 server/-product/production/AMC/jobs/templates/FILEZILLA_PASARESTORE-validate-and-restore.xml create mode 100644 server/-product/production/AMC/jobs/templates/NEXIO1-validate-and-archive.xml create mode 100644 server/-product/production/AMC/jobs/templates/NEXIO1-validate-and-restore.xml create mode 100644 server/-product/production/AMC/jobs/templates/NEXIO2-validate-and-archive.xml create mode 100644 server/-product/production/AMC/jobs/templates/NEXIO2-validate-and-restore.xml create mode 100644 server/-product/production/AMC/jobs/templates/PASAPOOL-validate-and-restore.xml create mode 100644 server/-product/production/AMC/jobs/templates/PB-archive-checker.xml create mode 100644 server/-product/production/AMC/jobs/templates/PBARCHIVE-validate-and-archive.xml create mode 100644 server/-product/production/AMC/jobs/templates/VOD-validate-and-restore.xml create mode 100644 server/-product/production/AMC/jobs/templates/cancelable.xml create mode 100644 server/-product/production/AMC/jobs/templates/create-lowres-ondemand.xml create mode 100644 server/-product/production/AMC/jobs/templates/fork-validate-and-archive.xml create mode 100644 server/-product/production/AMC/jobs/templates/fork-validate-and-restore.xml create mode 100644 server/-product/production/AMC/jobs/templates/generic-archive-checker.xml create mode 100644 server/-product/production/AMC/jobs/templates/harris1-missingmaterial-checker.xml create mode 100644 server/-product/production/AMC/jobs/templates/harris2-missingmaterial-checker.xml create mode 100644 server/-product/production/AMC/jobs/templates/nexio1-archive-checker.xml create mode 100644 server/-product/production/AMC/jobs/templates/nexio2-archive-checker.xml create mode 100644 server/-product/production/AMC/jobs/templates/peablebeach-missingmaterial-checker.xml create mode 100644 server/-product/production/AMC/jobs/templates/register-user-restore.xml create mode 100644 server/-product/production/AMC/jobs/templates/register-vod-restore.xml create mode 100644 server/-product/production/AMC/jobs/templates/server-status-report.xml create mode 100644 server/-product/production/AMC/jobs/templates/sync-subtitles.xml create mode 100644 server/-product/production/AMC/jobs/templates/test-fork-cancelable.xml create mode 100644 server/-product/production/AMC/jobs/templates/validate-dir-mxf.xml create mode 100644 server/-product/production/AMC/settings/application.yaml create mode 100644 server/-product/production/AMC/settings/dsm.opt create mode 100644 server/-product/production/AMC/settings/jetty.xml create mode 100644 server/-product/production/AMC/settings/log4j2-210810.xml create mode 100644 server/-product/production/AMC/settings/log4j2.xml create mode 100644 server/-product/production/AMC/settings/maestro.yaml create mode 100644 server/-product/production/AMC/settings/mediacube.yaml diff --git a/server/-product/production/AMC/jobs/executors.xml b/server/-product/production/AMC/jobs/executors.xml new file mode 100644 index 00000000..498d3972 --- /dev/null +++ b/server/-product/production/AMC/jobs/executors.xml @@ -0,0 +1,36 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/server/-product/production/AMC/jobs/schedules.json b/server/-product/production/AMC/jobs/schedules.json new file mode 100644 index 00000000..66fc1e29 --- /dev/null +++ b/server/-product/production/AMC/jobs/schedules.json @@ -0,0 +1,146 @@ +{"joblist":[ + { + "template": "PB-archive-checker.xml", + "name" : "PEABLEBEACH archiválás előkészítése", + "active": true, + "executeimmediate": false, + "cronexpression": "0 0 */1 * * ?", + "parameters": + [ + {"name": "sourceStoreName", "value": "PBARCHIVE", "type": "java.lang.String"}, + {"name": "escortStoreName", "value": "MEDIACUBE_DOWNLOADS", "type": "java.lang.String"}, + {"name": "lastModifiedHours", "value": 3, "type": "java.lang.Integer"} + ] + }, + { + "template": "sync-subtitles.xml", + "name" : "Feliratok szinkronizálása", + "active": true, + "cronexpression": "0 */30 * * * ?", + "parameters": [ + {"name": "sourceFolder", "value": "x:\\PB_ORIG", "type": "java.lang.String"}, + {"name": "syncTarget", "value": "r:\\BeachPool", "type": "java.lang.String"}, + {"name": "filter", "value": ".*\\.(stl)$", "type": "java.lang.String" } + ] + }, + { + "template": "server-status-report.xml", + "name" : "MediaCube státusz", + "active": true, + "executeimmediate": false, + "cronexpression": "0 */30 * * * ?" + }, + { + "template": "nexio1-archive-checker.xml", + "name" : "NEXIO1 archiválás előkészítése", + "active": true, + "executeimmediate": false, + "cronexpression": "0 0 */1 * * ?", + "parameters": + [ + {"name": "sourceStoreName", "value": "NEXIO1", "type": "java.lang.String"}, + {"name": "escortStoreName", "value": "MEDIACUBE_DOWNLOADS", "type": "java.lang.String"}, + {"name": "lastModifiedHours", "value": 3, "type": "java.lang.Integer"} + ] + }, + { + "template": "nexio2-archive-checker.xml", + "name" : "NEXIO2 archiválás előkészítése", + "active": true, + "executeimmediate": false, + "cronexpression": "0 0 */1 * * ?", + "parameters": + [ + {"name": "sourceStoreName", "value": "NEXIO2", "type": "java.lang.String"}, + {"name": "escortStoreName", "value": "MEDIACUBE_DOWNLOADS", "type": "java.lang.String"}, + {"name": "lastModifiedHours", "value": 3, "type": "java.lang.Integer"} + ] + }, + { + "template": "fork-validate-and-archive.xml", + "name" : "Kötegelt validálás és archiválás", + "active": true, + "executeimmediate": false, + "cronexpression": "0 */30 * * * ?", + "parameters": + [ + {"name": "sourceStoreName", "value": "MEDIACUBE_DOWNLOADS", "type": "java.lang.String"}, + {"name": "template", "value": "validate-and-archive.xml", "type": "java.lang.String"}, + {"name": "expectedColorSpace", "value": "", "type": "java.lang.String"}, + {"name": "limit", "value": 100, "type": "java.lang.Integer"} + ] + }, + { + "template": "harris1-missingmaterial-checker.xml", + "name" : "HARRIS1 missing material lekérdezés", + "active": true, + "executeimmediate": false, + "cronexpression": "0 */5 * * * ?", + "parameters": + [ + {"name": "vicFiles", "value": ["z:\\NEXIO.VIC", "z:\\EDIT 9.VIC", "z:\\TRIM2.VIC"], "type": "com.ibm.nosql.json.api.BasicDBList"}, + {"name": "escortStoreName", "value": "MEDIACUBE_UPLOADS", "type": "java.lang.String"}, + {"name": "targetStoreName", "value": "NEXIO1", "type": "java.lang.String"}, + {"name": "targetProtocol", "value": "FTP", "type": "java.lang.String"} + ] + }, + { + "template": "harris2-missingmaterial-checker.xml", + "name" : "HARRIS2 missing material lekérdezés", + "active": true, + "executeimmediate": false, + "cronexpression": "0 */5 * * * ?", + "parameters": + [ + {"name": "vicFiles", "value": ["z:\\NEXIO2.VIC", "z:\\EDIT-4.VIC", "z:\\EDIT 10.VIC"], "type": "com.ibm.nosql.json.api.BasicDBList"}, + {"name": "escortStoreName", "value": "MEDIACUBE_UPLOADS", "type": "java.lang.String"}, + {"name": "targetStoreName", "value": "NEXIO2", "type": "java.lang.String"}, + {"name": "targetProtocol", "value": "FTP", "type": "java.lang.String"} + ] + }, + { + "template": "peablebeach-missingmaterial-checker.xml", + "name" : "PEABLEBEACH missing material lekérdezés", + "active": true, + "executeimmediate": false, + "cronexpression": "0 */5 * * * ?", + "parameters": + [ + {"name": "escortStoreName", "value": "MEDIACUBE_UPLOADS", "type": "java.lang.String"}, + {"name": "lookupStoreNames", "value": ["PEABLEBEACH", "PASAPOOL", "SELENIOPOOL", "SELENIOPOOL_16_9", "SELENIOPOOL_16_9_PROCESSING", "SELENIOPOOL_4_3", "SELENIOPOOL_4_3_PROCESSING"], "type": "com.ibm.nosql.json.api.BasicDBList"}, + {"name": "targetStoreName", "value": "PASAPOOL", "type": "java.lang.String"}, + {"name": "targetProtocol", "value": "FTP", "type": "java.lang.String"}, + {"name": "primaryEndPoint", "value": "http://10.170.100.61:18083", "type": "java.lang.String"}, + {"name": "primaryUserName", "value": "pbs", "type": "java.lang.String"}, + {"name": "primaryPassword", "value": "pbs", "type": "java.lang.String"}, + {"name": "secondaryEndPoint", "value": "http://10.170.100.62:18083", "type": "java.lang.String"}, + {"name": "secondaryUserName", "value": "pbs", "type": "java.lang.String"}, + {"name": "secondaryPassword", "value": "pbs", "type": "java.lang.String"}, + {"name": "rangeForwardHours", "value": 168, "type": "java.lang.Integer"} + ] + }, + { + "template": "fork-validate-and-restore.xml", + "name" : "Kötegelt validálás és visszatöltés", + "active": true, + "executeimmediate": false, + "cronexpression": "0 */1 * * * ?", + "parameters": + [ + {"name": "sourceStoreName", "value": "MEDIACUBE_UPLOADS", "type": "java.lang.String"}, + {"name": "template", "value": "validate-and-restore.xml", "type": "java.lang.String"}, + {"name": "expectedColorSpace", "value": "", "type": "java.lang.String"}, + {"name": "limit", "value": 0, "type": "java.lang.Integer"} + ] + }, + { + "template": "register-user-restore.xml", + "name" : "Felhasználói visszatöltés regisztráció", + "active": false, + "executeimmediate": false, + "parameters": + [ + {"name": "escortStoreName", "value": "MEDIACUBE_UPLOADS", "type": "java.lang.String"} + ] + } +]} diff --git a/server/-product/production/AMC/jobs/steps/ArchiveSubtitlesStep.java b/server/-product/production/AMC/jobs/steps/ArchiveSubtitlesStep.java new file mode 100644 index 00000000..4afb09b7 --- /dev/null +++ b/server/-product/production/AMC/jobs/steps/ArchiveSubtitlesStep.java @@ -0,0 +1,116 @@ +package user.jobengine.server.steps; + +import java.io.File; +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.nio.file.StandardCopyOption; +import java.nio.file.attribute.FileAttribute; +import java.nio.file.attribute.PosixFilePermission; +import java.nio.file.attribute.PosixFilePermissions; +import java.util.Arrays; +import java.util.List; +import java.util.Set; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + +public class ArchiveSubtitlesStep extends JobStep { + private static final Logger logger = LogManager.getLogger(); + + @StepEntry + public Object[] execute(List files, String syncTarget) { + int current = 0; + for (String file : files) { + Path source = Paths.get(file); + Path archive = archiveSubtitle(source); + + if (archive != null) { + Path sync = Paths.get(syncTarget); + syncSubtitle(archive, sync); + } + + current++; + int progress = current * 100 / files.size(); + setProgress(progress); + } + setProgress(100); + return null; + } + + private Path archiveSubtitle(Path source) { + Path archiveRoot = Paths.get(source.toString().replace("PB_ORIG", "PB_ARCH")).getParent().getParent(); + String lang = source.getParent().getFileName().toString().toUpperCase(); + // logger.info(getMarker(), "Archive root is {}, lang is {}", archiveRoot, + // lang); + + String sourceFileName = source.getFileName().toString(); + Path archive = Paths.get(archiveRoot.toString(), sourceFileName.replace(".stl", "_" + lang + ".stl")); + boolean isOverride = archive.toFile().exists(); + try { + ensureFolder(archiveRoot); + Files.copy(source, archive, StandardCopyOption.REPLACE_EXISTING); + + boolean isContentEquals = Arrays.equals(Files.readAllBytes(source), Files.readAllBytes(archive)); + logger.info(getMarker(), "Contents equals {}", isContentEquals); + if (isContentEquals) { + if (isOverride) + logger.info(getMarker(), "Successfully replaced {}", archive); + else + logger.info(getMarker(), "Successfully archived {}", archive); + + String doneFileName = sourceFileName.substring(0, sourceFileName.lastIndexOf(".")); + Path done = Paths.get(source.getParent().toString(), doneFileName + "_DONE.stl"); + Files.move(source, done, StandardCopyOption.REPLACE_EXISTING); + return archive; + } else + throw new Exception("Source and target content not equals"); + } catch (Exception e) { + logger.error(getMarker(), "Error copy {} to {}. System message: {}", source, archive, e.getMessage()); + } + return null; + } + + private void syncSubtitle(Path archive, Path syncTarget) { + Path sync = Paths.get(syncTarget.toString(), archive.getFileName().toString()); + logger.info(getMarker(), "Sync root {} exists {}", syncTarget, syncTarget.toFile().exists()); + + if (!sync.toFile().exists()) { + logger.info(getMarker(), "Skipping {} synchronization, target not exists", sync); + return; + } + + logger.info(getMarker(), "Target {} synchronization required", sync); + + try { + Files.copy(archive, sync, StandardCopyOption.REPLACE_EXISTING); + boolean isContentEquals = Arrays.equals(Files.readAllBytes(archive), Files.readAllBytes(sync)); + if (isContentEquals) + logger.info(getMarker(), "Successfully synchronized {}", archive); + else + throw new Exception("Synchronization source and target content not equals"); + } catch (Exception e) { + logger.error(getMarker(), "Error synchronize {} to {}. System message: {}", archive, sync, e.getMessage()); + } + } + + public void ensureFolder(Path filePath) throws IOException { + File folder = filePath.toFile(); + if (!folder.exists() || !folder.isDirectory()) { + try { + Set perms = PosixFilePermissions.fromString("rwxrwxrwx"); + FileAttribute> attr = PosixFilePermissions.asFileAttribute(perms); + Files.createDirectories(filePath, attr); + } catch (Exception e) { + try { + Files.createDirectories(filePath); + } catch (Exception e1) { + logger.catching(e1); + throw e1; + } + } + } + } + +} diff --git a/server/-product/production/AMC/jobs/steps/CancelableStep.java b/server/-product/production/AMC/jobs/steps/CancelableStep.java new file mode 100644 index 00000000..2b025e89 --- /dev/null +++ b/server/-product/production/AMC/jobs/steps/CancelableStep.java @@ -0,0 +1,35 @@ +package user.jobengine.server.steps; + +import org.apache.commons.net.ftp.FTPClient; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + +import user.commons.StoreUri; +import user.commons.remotestore.FtpDirectoryLister; +import user.commons.remotestore.RemoteStoreProtocol; + +public class CancelableStep extends JobStep { + private static final Logger logger = LogManager.getLogger(); + int count = 10; + + @StepEntry + public Object[] execute(int param) throws Exception { + try { + getJobRuntime().setDisableRetry(true); + for (int i = 0; i < count; i++) { + if (getJobRuntime().isWaitingCancel()) + break; + Thread.sleep(2000); + int progress = (i + 1) * 100 / count; + setProgress(progress); +// if (i==9) +// throw new Exception("Hiba teszt!"); + } + } catch (Exception e) { + e.printStackTrace(); + throw e; + } + return null; + } + +} diff --git a/server/-product/production/AMC/jobs/steps/DirMXFValidatorStep.java b/server/-product/production/AMC/jobs/steps/DirMXFValidatorStep.java new file mode 100644 index 00000000..f26d679b --- /dev/null +++ b/server/-product/production/AMC/jobs/steps/DirMXFValidatorStep.java @@ -0,0 +1,61 @@ +package user.jobengine.server.steps; + +import java.nio.file.DirectoryStream; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + +import user.commons.mediatool.MediaInfo; +import user.jobengine.db.Media; + +public class DirMXFValidatorStep extends JobStep { + private static final Logger logger = LogManager.getLogger(); + + @StepEntry + public Object[] execute(String source) throws Exception { + Path path = Paths.get(source); + DirectoryStream directoryStream = Files.newDirectoryStream(path); + for (Path file : directoryStream) { + if (getJobRuntime().isWaitingCancel()) { + cancel(); + return null; + } + + if (file.toFile().isDirectory()) + continue; + + String fileName = file.getFileName().toString(); + if (fileName.contains(".")) + fileName = fileName.substring(0, fileName.lastIndexOf(".")); + + if (file.toFile().length() == 0) { + logger.error("Zero: {}", fileName); + continue; + } + + Media media = getManager().getMedia(fileName); + long frames = 0; + if (media == null) { + logger.warn("Unchecked yet: {}", fileName); + } else + frames = media.getLength(); + + MediaInfo mi = null; + + try { + mi = new MediaInfo(file); + mi.process(); + } catch (Exception e) { + logger.error(getSessionMarker(), e.getMessage()); + logger.error("Unreadable: {}", fileName); + } + + logger.info("Valid: {} MC farmes {} FF frames {}", fileName, frames, mi.getFrames()); + } + + return null; + } +} diff --git a/server/-product/production/AMC/jobs/steps/EscortFileUpdaterStep.java b/server/-product/production/AMC/jobs/steps/EscortFileUpdaterStep.java new file mode 100644 index 00000000..5dcb5239 --- /dev/null +++ b/server/-product/production/AMC/jobs/steps/EscortFileUpdaterStep.java @@ -0,0 +1,57 @@ +package user.jobengine.server.steps; + +import java.io.FileNotFoundException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + +import user.commons.DownloadableMedia; +import user.commons.StoreUri; +import user.commons.mediatool.MediaInfo; +import user.commons.remotestore.RemoteStoreProtocol; +import user.jobengine.server.steps.shared.EscortFiles; + +public class EscortFileUpdaterStep extends JobStep { + private static final Logger logger = LogManager.getLogger(); + + @StepEntry + public Object[] execute(StoreUri sourceStoreUri, String fileName, String escortFile) throws Exception { + if (!RemoteStoreProtocol.LOCAL.equals(sourceStoreUri.getProtocol())) + throw new Exception("Store URI protocol must be local."); + Path filePath = Paths.get(sourceStoreUri.toString(true), fileName); + if (!filePath.toFile().exists()) + throw new FileNotFoundException("File not exists: " + filePath.toString()); + + Path escortFilePath = Paths.get(escortFile); + DownloadableMedia downloadable = null; + try { + downloadable = EscortFiles.decode(escortFilePath); + } catch (Exception e) { + logger.error(getSessionMarker(), e.getMessage()); + } + + if (downloadable != null) + storeCurrentFrames(filePath, downloadable, escortFilePath); + + return null; + } + + private void storeCurrentFrames(Path filePath, DownloadableMedia downloadable, Path escortFilePath) { + MediaInfo mi = null; + try { + mi = new MediaInfo(filePath); + mi.process(); + long frames = mi.getFrames(); + downloadable.put("frames", frames); + Files.write(escortFilePath, downloadable.toPrettyString("").getBytes()); + logger.info(getMarker(), "{} length {} is saved to escort file", filePath, frames, + escortFilePath.getFileName().toString()); + } catch (Exception e) { + logger.error(getSessionMarker(), e.getMessage()); + } + } + +} diff --git a/server/-product/production/AMC/jobs/steps/FILEZILLA_AVIDTransferToStep.java b/server/-product/production/AMC/jobs/steps/FILEZILLA_AVIDTransferToStep.java new file mode 100644 index 00000000..97747098 --- /dev/null +++ b/server/-product/production/AMC/jobs/steps/FILEZILLA_AVIDTransferToStep.java @@ -0,0 +1,11 @@ +package user.jobengine.server.steps; + +import user.commons.StoreUri; + +public class FILEZILLA_AVIDTransferToStep extends TransferStep { + @Override + @StepEntry + public Object[] execute(StoreUri sourceStoreUri, String sourceFileName, StoreUri targetStoreUri, String targetFileName) throws Exception { + return super.execute(sourceStoreUri, sourceFileName, targetStoreUri, targetFileName); + } +} diff --git a/server/-product/production/AMC/jobs/steps/FILEZILLA_PASARESTORETransferToStep.java b/server/-product/production/AMC/jobs/steps/FILEZILLA_PASARESTORETransferToStep.java new file mode 100644 index 00000000..925549ba --- /dev/null +++ b/server/-product/production/AMC/jobs/steps/FILEZILLA_PASARESTORETransferToStep.java @@ -0,0 +1,85 @@ +package user.jobengine.server.steps; + +import java.nio.file.Path; +import java.nio.file.Paths; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.apache.logging.log4j.Marker; +import org.apache.logging.log4j.MarkerManager; + +import user.commons.StoreUri; +import user.commons.mediaarea.MediaArea; + +public class FILEZILLA_PASARESTORETransferToStep extends TransferStep { + private static final Logger logger = LogManager.getLogger(); + private static final String CODEC_PROFILE_HIGH = "HIGH"; + private static final String CODEC_PROFILE_MAIN = "MAIN"; + private static final String CODEC_PROFILE_422 = "4:2:2"; + private static final String DISPLAY_ASPECT_16_9 = "16:9"; + private static final String DISPLAY_ASPECT_4_3 = "4:3"; + private StoreUri sourceStoreUri; + private String sourceFileName; + + @Override + @StepEntry + public Object[] execute(StoreUri sourceStoreUri, String sourceFileName, StoreUri targetStoreUri, String targetFileName) throws Exception { + this.sourceStoreUri = sourceStoreUri; + this.sourceFileName = sourceFileName; + logMediaProfile(); + Object[] result = super.execute(sourceStoreUri, sourceFileName, targetStoreUri, targetFileName); + return result; + } + + private boolean isHD(MediaArea mi) { + return mi.getHeight() == 1080; + } + + private boolean isSD_HIGH_16_9(MediaArea mi) { + String formatProfileName = mi.getFormatProfileName(); + return mi.getHeight() < 650 && formatProfileName.contains(CODEC_PROFILE_HIGH) && DISPLAY_ASPECT_16_9.equals(mi.getDisplayAspect()); + } + + private boolean isSD_HIGH_4_3(MediaArea mi) { + String formatProfileName = mi.getFormatProfileName(); + return mi.getHeight() < 650 && formatProfileName.contains(CODEC_PROFILE_HIGH) && DISPLAY_ASPECT_4_3.equals(mi.getDisplayAspect()); + } + + private boolean isSD_MAIN_422(MediaArea mi) { + String formatProfileName = mi.getFormatProfileName(); + return mi.getHeight() < 650 && (formatProfileName.contains(CODEC_PROFILE_MAIN) || formatProfileName.contains(CODEC_PROFILE_422)); + } + + protected void logMediaProfile() { + Marker marker = MarkerManager.getMarker("MEDIAPROFILE"); + try { + Path filePath = Paths.get(sourceStoreUri.toString(true), sourceFileName); + MediaArea mi = new MediaArea(filePath); + mi.process(); + logger.info(marker, "{} data : {} {} {}", sourceFileName, mi.getFormatProfileName(), mi.getHeight(), mi.getDisplayAspect()); + + if (isHD(mi)) { + logger.info(marker, "{} is MOVED_HD", sourceFileName); + return; + } + + if (isSD_HIGH_16_9(mi)) { + logger.info(marker, "{} is CONVSD16_9", sourceFileName); + return; + } + + if (isSD_HIGH_4_3(mi)) { + logger.info(marker, "{} is CONVSD4_3", sourceFileName); + return; + } + + if (isSD_MAIN_422(mi)) { + logger.info(marker, "{} is MOVED_SD", sourceFileName); + return; + } + } catch (Exception e) { + logger.error(marker, e.getMessage()); + } + } + +} diff --git a/server/-product/production/AMC/jobs/steps/FileCleanupStep.java b/server/-product/production/AMC/jobs/steps/FileCleanupStep.java new file mode 100644 index 00000000..8e1a6f4c --- /dev/null +++ b/server/-product/production/AMC/jobs/steps/FileCleanupStep.java @@ -0,0 +1,35 @@ +package user.jobengine.server.steps; + +import java.nio.file.Path; +import java.nio.file.Paths; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.apache.logging.log4j.message.Message; +import org.apache.logging.log4j.message.ParameterizedMessage; + +import user.commons.StoreUri; +import user.jobengine.server.steps.shared.EscortFiles; + +public class FileCleanupStep extends JobStep { + private static final Logger logger = LogManager.getLogger(); + + @StepEntry + public Object[] execute(StoreUri sourceStoreUri, String fileName, String escortFile) throws Exception { + try { + + Message msg = new ParameterizedMessage("A {} fájl visszatöltése befejeződött.", fileName); + EscortFiles.notifyRecipient(Paths.get(escortFile), logger, msg); + + Path filePath = Paths.get(sourceStoreUri.toString(true), fileName); + EscortFiles.remove(filePath); + EscortFiles.remove(Paths.get(escortFile)); + EscortFiles.remove(Paths.get(escortFile + EscortFiles.DOT_CATCHED)); + } catch (Exception e) { + logger.error(e.getMessage()); + throw e; + } + return null; + } + +} diff --git a/server/-product/production/AMC/jobs/steps/FileSearchFilterOptions.java b/server/-product/production/AMC/jobs/steps/FileSearchFilterOptions.java new file mode 100644 index 00000000..436c3be0 --- /dev/null +++ b/server/-product/production/AMC/jobs/steps/FileSearchFilterOptions.java @@ -0,0 +1,41 @@ +package user.jobengine.server.steps; + +import java.nio.file.Path; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +import com.ibm.nosql.json.api.BasicDBObject; + +public class FileSearchFilterOptions { + + private BasicDBObject filter; + + public FileSearchFilterOptions(BasicDBObject filter) { + this.filter = filter; + } + + public boolean acceptFile(Path file) { + if (filter == null) + return true; + + if (filter.containsKey("fileName")) { + //.*\.(sh|ini|conf|vhost|xml|php)$ + String fileNamePattern = filter.getString("fileName"); + if (fileNamePattern == null || fileNamePattern.trim().length() == 0) + return true; + + Pattern pattern = Pattern.compile(fileNamePattern, Pattern.CASE_INSENSITIVE); + + Matcher matcher = pattern.matcher(file.getFileName().toString()); + if (matcher.find()) + return true; + + } + + return false; + } + + public boolean preAcceptDirectory(Path file) { + return true; + } +} diff --git a/server/-product/production/AMC/jobs/steps/FileValidatorStep.java b/server/-product/production/AMC/jobs/steps/FileValidatorStep.java new file mode 100644 index 00000000..362155f5 --- /dev/null +++ b/server/-product/production/AMC/jobs/steps/FileValidatorStep.java @@ -0,0 +1,146 @@ +package user.jobengine.server.steps; + +import java.io.FileNotFoundException; +import java.nio.file.Path; +import java.nio.file.Paths; + +import org.apache.commons.lang.StringUtils; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.apache.logging.log4j.message.Message; +import org.apache.logging.log4j.message.ParameterizedMessage; + +import user.commons.DownloadableMedia; +import user.commons.StoreUri; +import user.commons.mediatool.MediaInfo; +import user.commons.remotestore.RemoteStoreProtocol; +import user.jobengine.db.Media; +import user.jobengine.server.steps.shared.EscortFiles; + +public class FileValidatorStep extends JobStep { + private static final String COLORSPACE = "COLORSPACE"; + private static final Logger logger = LogManager.getLogger(); + + @StepEntry + public Object[] execute(StoreUri sourceStoreUri, String fileName, long expectedFrameNumber, long expectedSize, String exceptedColorSpace, String escortFile) + throws Exception { + if (!RemoteStoreProtocol.LOCAL.equals(sourceStoreUri.getProtocol())) + throw new Exception("Store URI protocol must be local."); + Path filePath = Paths.get(sourceStoreUri.toString(true), fileName); + if (!filePath.toFile().exists()) + throw new FileNotFoundException("File not exists: " + filePath.toString()); + + Path escortFilePath = Paths.get(escortFile); + + if (getJobRuntime().isWaitingCancel()) { + EscortFiles.remove(filePath); + //EscortFiles.remove(escortFilePath); + EscortFiles.remove(Paths.get(escortFile + EscortFiles.DOT_CATCHED)); + + Message msg = new ParameterizedMessage("A {} fájl visszatöltése kezelői beavatkozás miatt megszakadt.", fileName); + EscortFiles.notifyRecipient(escortFilePath, logger, msg); + + cancel(); + return null; + } + + DownloadableMedia downloadable = null; + try { + downloadable = EscortFiles.decode(escortFilePath); + } catch (Exception e) { + logger.error(getSessionMarker(), e.getMessage()); + } + + if (downloadable != null && expectedFrameNumber == 0) + storeCurrentFrames(filePath, downloadable); + + if (downloadable != null && downloadable.containsKey("skipValidation") && downloadable.getBoolean("skipValidation")) + return null; + + if (expectedFrameNumber > 0 || StringUtils.isNotBlank(exceptedColorSpace)) + validateMedia(filePath, expectedFrameNumber, exceptedColorSpace, escortFile); + + if (expectedSize > 0) + validateSize(filePath, expectedSize, escortFile); + + //A metadata persister miatt torlunk mindet teszt modban + /* + if (StringUtils.isNotBlank(testSimulate)) { + EscortFiles.remove(escortFilePath); + EscortFiles.remove(Paths.get(escortFile + EscortFiles.DOT_CATCHED)); + logger.info(getMarker(), "{} deleted to prevent real processing", escortFile); + } + */ + logger.info(getMarker(), "{} passed validation", fileName); + return null; + } + + private void storeCurrentFrames(Path filePath, DownloadableMedia downloadable) { + MediaInfo mi = null; + try { + mi = new MediaInfo(filePath); + mi.process(); + long frames = mi.getFrames(); + + if (downloadable.containsKey("mediaId")) { + Media media = getManager().getMedia(downloadable.getLong("mediaId")); + if (media != null) { + media.setLength(frames); + getManager().modify(media); + } + + } + + } catch (Exception e) { + logger.error(getSessionMarker(), e.getMessage()); + } + } + + private void validateMedia(Path filePath, long expectedFrameNumber, String exceptedColorSpace, String escortFile) throws Exception { + MediaInfo mi = null; + try { + mi = new MediaInfo(filePath); + mi.process(); + + if (expectedFrameNumber > 0) { + long frames = mi.getFrames(); + if (Math.abs(frames - expectedFrameNumber) > 3) { + Message msg = new ParameterizedMessage("Validációs hiba miatt a {} fájl másolása nem lehetséges.", filePath.getFileName()); + EscortFiles.notifyRecipient(Paths.get(escortFile), logger, msg); + getJobRuntime().setDisableRetry(true); + throw new Exception("Length mismatch in " + filePath + ". Expected: " + expectedFrameNumber + ", found: " + frames); + } + } + + if (StringUtils.isNotBlank(exceptedColorSpace)) { + String colorSpace = mi.getColorSpace(); + if (colorSpace == null) { + EscortFiles.createFellow(escortFile, COLORSPACE); + throw new Exception("Can not determine color space in " + filePath); + } + + if (!colorSpace.equals(exceptedColorSpace)) { + EscortFiles.createFellow(escortFile, COLORSPACE); + throw new Exception("Expected color space is " + exceptedColorSpace + ", but found " + colorSpace); + } + } + + } catch (Exception e) { + logger.error(getSessionMarker(), e.getMessage()); + throw e; + } + } + + private void validateSize(Path filePath, long expectedSize, String escortFile) throws Exception { + try { + long length = filePath.toFile().length(); + if (length != expectedSize) + throw new Exception("Size mismatch. Expected: " + expectedSize + ", found: " + length); + + } catch (Exception e) { + logger.error(getSessionMarker(), e.getMessage()); + throw e; + } + } + +} diff --git a/server/-product/production/AMC/jobs/steps/ForkDownloadStep.java b/server/-product/production/AMC/jobs/steps/ForkDownloadStep.java new file mode 100644 index 00000000..6722c822 --- /dev/null +++ b/server/-product/production/AMC/jobs/steps/ForkDownloadStep.java @@ -0,0 +1,204 @@ +package user.jobengine.server.steps; + +import java.io.IOException; +import java.nio.file.DirectoryStream; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + +import user.commons.DownloadableMedia; +import user.commons.JobStatus; +import user.commons.RemoteFile; +import user.commons.StoreUri; +import user.commons.remotestore.FtpDirectoryLister; +import user.commons.remotestore.RemoteStoreProtocol; +import user.jobengine.db.Store; +import user.jobengine.server.IJobEngine; +import user.jobengine.server.IJobRuntime; +import user.jobengine.server.steps.shared.EscortFiles; + +public class ForkDownloadStep extends JobStep { + private static final Logger logger = LogManager.getLogger(); + + @StepEntry + public Object[] execute(String tempStoreName, String template, String expectedColorSpace, int limit) + throws Exception { + DirectoryStream directoryStream = null; + int count = limit; + int allCount = 0; + int currentCount = 0; + + try { + + StoreUri tempStoreUri = getManager().getStoreUri(tempStoreName, RemoteStoreProtocol.LOCAL); + Path path = Paths.get(tempStoreUri.toString(true), ".STATUS"); + + List escortFiles = new ArrayList<>(); + directoryStream = Files.newDirectoryStream(path); + for (Path escortFile : directoryStream) { + if (getJobRuntime().isWaitingCancel()) { + cancel(); + return null; + } + escortFiles.add(escortFile); + } + allCount = escortFiles.size(); + escortFiles.sort((p1, p2) -> { + return Long.valueOf(p1.toFile().lastModified()).compareTo(p2.toFile().lastModified()); + }); + + if (allCount > 0) { + for (Path escortFile : escortFiles) { + if (getJobRuntime().isWaitingCancel()) { + cancel(); + break; + } + + if (processPathItem(tempStoreUri, template, escortFile)) { + count--; + if (count == 0) + break; + } + currentCount++; + int progress = currentCount * 100 / allCount; + setProgress(progress); + } + } + + setProgress(100); + } catch (Exception e) { + logger.error(getSessionMarker(), e.getMessage()); + throw e; + } finally { + if (directoryStream != null) { + try { + directoryStream.close(); + } catch (IOException e) { + } + } + } + + return null; + } + + private boolean processPathItem(StoreUri tempStoreUri, String template, Path escortFile) throws IOException { + if (escortFile.toFile().isDirectory()) + return false; + + //ami nem JSON azt nem kezeljuk + if (!escortFile.getFileName().toString().toLowerCase().endsWith(EscortFiles.DOT_JSON)) + return false; + + //aminek van COLORSPACE kiterjesztesu valtozata azt nem kezeljuk + // Path colorspaceFile = Paths.get(escortFile.toString() + ".colorspace"); + // if (colorspaceFile.toFile().exists()) + // return false; + + if (EscortFiles.isCatchedFileExists(escortFile)) + return false; + + String escortFileName = escortFile.getFileName().toString(); + //minimum 3 karakter: STORENAME.FILENAME.MXF.JSON + long dotCount = escortFileName.chars().filter(ch -> ch == '.').count(); + if (dotCount < 3) + return false; + + try { + DownloadableMedia downloadable = EscortFiles.decode(escortFile); + if (downloadable == null) { + logger.error(getSessionMarker(), "Can't encode {}", escortFile); + return false; + } + + StoreUri sourceStoreUri = getManager().getStoreUri(downloadable.getLong("sourceStoreUriId")); + StoreUri targetStoreUri = getManager().getStoreUri(downloadable.getLong("targetStoreUriId")); + + Store sourceStore = getManager().getStore(sourceStoreUri.getStoreId()); + String sourceStoreName = sourceStore.getName(); + template = sourceStoreName + "-" + template; + logger.info(getMarker(), "Starting {} for {}", template, escortFileName); + + String sourceFileName = downloadable.getString("fileName"); + Map parameters = new HashMap<>(); + parameters.put("sourceStoreUri", sourceStoreUri); + parameters.put("fileName", sourceFileName); + parameters.put("tempStoreUri", tempStoreUri); + parameters.put("targetStoreUri", targetStoreUri); + parameters.put("expectedFrameNumber", downloadable.getLong("frames")); + parameters.put("expectedSize", downloadable.getLong("size")); + parameters.put("expectedColorSpace", downloadable.getString("colorSpace")); + parameters.put("escortFile", escortFile.toAbsolutePath().toString()); + + int priority = 50; + + //megszakitasok miatt a mar lejott mozikat vegyuk elore 20.09.09 + if (targetExists(tempStoreUri, sourceFileName)) + priority = 51; + + //parameterek: IJobRuntime parent, String template, String name, int priority, Map parameters + IJobRuntime child = getEngine().submit(null, e -> { + IJobRuntime r = (IJobRuntime) e.getSource(); + if (r.isDisableRetry()) + return; + + if (e.getStatus().equals(JobStatus.CANCELED) || e.getStatus().equals(JobStatus.SUSPENDED)) + EscortFiles.removeCatchedFile(escortFile); + + if (e.getStatus().equals(JobStatus.FINISHED) && "PBARCHIVE".equals(sourceStoreName)) { + try { + FtpDirectoryLister lister = (FtpDirectoryLister) sourceStoreUri.getLister(); + RemoteFile remoteFile = lister.get(sourceFileName); + if (remoteFile == null) + logger.info(getMarker(), "Remote file for {} is not available", sourceFileName); + else { + logger.info(getMarker(), "Remote file {} successfully archived, deleting...", + sourceFileName); + lister.delete(remoteFile); + } + } catch (Exception ex) { + logger.error(getMarker(), ex.getMessage()); + } finally { + if (sourceStoreUri != null) + sourceStoreUri.cleanUp(); + } + } + + }, template, "Archiválás", priority, IJobEngine.DEFAULT_OWNER, parameters); + if (child == null) + throw new Exception("Submit error."); + + child.setDescription(sourceFileName); + + child.setRelated(downloadable.getString("title") + " / " + sourceStoreName); + + EscortFiles.createCatchedFile(escortFile); + + } catch (Exception e) { + logger.error(getSessionMarker(), e.getMessage()); + return false; + } + + return true; + } + + private boolean targetExists(StoreUri storeUri, String fileName) { + try { + Path targetFile = Paths.get(storeUri.toString(true), fileName); + if (targetFile.toFile().exists()) { + logger.info(getMarker(), "Retrieved file {} exists", fileName); + return true; + } + } catch (Exception e) { + logger.error(getMarker(), e.getMessage()); + } + logger.info(getMarker(), "Retrieved file {} not exists yet", fileName); + return false; + } +} diff --git a/server/-product/production/AMC/jobs/steps/ForkUploadStep.java b/server/-product/production/AMC/jobs/steps/ForkUploadStep.java new file mode 100644 index 00000000..57b1801d --- /dev/null +++ b/server/-product/production/AMC/jobs/steps/ForkUploadStep.java @@ -0,0 +1,243 @@ +package user.jobengine.server.steps; + +import java.io.IOException; +import java.nio.file.DirectoryStream; +import java.nio.file.FileStore; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.nio.file.attribute.BasicFileAttributes; +import java.nio.file.attribute.FileTime; +import java.text.NumberFormat; +import java.util.ArrayList; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import org.apache.commons.lang.StringUtils; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + +import user.commons.DownloadableMedia; +import user.commons.JobStatus; +import user.commons.StoreUri; +import user.commons.remotestore.RemoteStoreProtocol; +import user.jobengine.db.Media; +import user.jobengine.db.MediaFile; +import user.jobengine.db.Store; +import user.jobengine.server.IJobEngine; +import user.jobengine.server.IJobRuntime; +import user.jobengine.server.steps.shared.EscortFiles; + +public class ForkUploadStep extends JobStep { + private static final Logger logger = LogManager.getLogger(); + + @StepEntry + public Object[] execute(String tempStoreName, String template, String expectedColorSpace, int limit) throws Exception { + DirectoryStream directoryStream = null; + int count = limit; + int allCount = 0; + int currentCount = 0; + + try { + StoreUri tempStoreUri = getManager().getStoreUri(tempStoreName, RemoteStoreProtocol.LOCAL); + + try { + Path tempPath = Paths.get(tempStoreUri.toString(true)); + FileStore store = Files.getFileStore(tempPath.getRoot()); + NumberFormat nf = NumberFormat.getNumberInstance(); + logger.info(getMarker(), "Free space on temp {}", nf.format(store.getUsableSpace())); + } catch (IOException e) { + logger.error(getMarker(), e.getMessage()); + } + + Path path = Paths.get(tempStoreUri.toString(true), ".STATUS"); + + List escortFiles = new ArrayList<>(); + directoryStream = Files.newDirectoryStream(path); + for (Path escortFile : directoryStream) { + if (getJobRuntime().isWaitingCancel()) { + cancel(); + return null; + } + escortFiles.add(escortFile); + } + allCount = escortFiles.size(); + //a file neve az adasbakerules idopontjaval kezdodik + // escortFiles.sort((p1, p2) -> { + // return p1.compareTo(p2); + // }); + + //200911 megis inkabb a letrehozas idopontja legyen + try { + escortFiles.sort((p1, p2) -> { + int result = 0; + try { + FileTime mod1 = Files.readAttributes(p1, BasicFileAttributes.class).lastModifiedTime(); + FileTime mod2 = Files.readAttributes(p2, BasicFileAttributes.class).lastModifiedTime(); + result = mod1.compareTo(mod2); + } catch (Throwable e) { + } + return result; + }); + } catch (Throwable e) { + } + + if (allCount > 0) { + for (Path escortFile : escortFiles) { + if (getJobRuntime().isWaitingCancel()) { + cancel(); + break; + } + + if (processPathItem(tempStoreUri, template, escortFile)) { + count--; + if (count == 0) + break; + } + currentCount++; + int progress = currentCount * 100 / allCount; + setProgress(progress); + } + } + + setProgress(100); + } catch (Exception e) { + logger.error(getSessionMarker(), e.getMessage()); + throw e; + } finally { + if (directoryStream != null) { + try { + directoryStream.close(); + } catch (IOException e) { + } + } + } + + return null; + } + + private String getTSMFileName(long mediaId, StoreUri sourceStoreUri) { + String result = null; + Media media = getManager().getMedia(mediaId); + + if (media != null) { + List mediaFiles = media.getMediaFiles(); + for (MediaFile mediaFile : mediaFiles) { + if (mediaFile.getStoreId() == sourceStoreUri.getStoreId()) { + result = mediaFile.getRelativePath(); + break; + } + } + } + + return result; + } + + private boolean processPathItem(StoreUri tempStoreUri, String template, Path escortFile) throws IOException { + if (escortFile.toFile().isDirectory()) + return false; + + //ami nem JSON azt nem kezeljuk + if (!escortFile.getFileName().toString().toLowerCase().endsWith(EscortFiles.DOT_JSON)) + return false; + + //aminek van COLORSPACE kiterjesztésű + // Path colorspaceFile = Paths.get(escortFile.toString() + ".colorspace"); + // if (colorspaceFile.toFile().exists()) + // return false; + + if (EscortFiles.isCatchedFileExists(escortFile)) + return false; + + String escortFileName = escortFile.getFileName().toString(); + //minimum 3 karakter: STORENAME.FILENAME.MXF.JSON + long dotCount = escortFileName.chars().filter(ch -> ch == '.').count(); + if (dotCount < 3) + return false; + + try { + DownloadableMedia downloadable = EscortFiles.decode(escortFile); + + if (downloadable == null) { + logger.error(getSessionMarker(), "Can't decode {}", escortFile); + return false; + } + + StoreUri sourceStoreUri = getManager().getStoreUri("TSM", RemoteStoreProtocol.TSM); + + long mediaId = downloadable.getLong("mediaId"); + String tsmFileName = getTSMFileName(mediaId, sourceStoreUri); + if (StringUtils.isBlank(tsmFileName)) { + logger.error(getMarker(), "Media ID not exists anymore {}", mediaId); + return false; + } + + StoreUri targetStoreUri = getManager().getStoreUri(downloadable.getLong("targetStoreUriId")); + + Store targetStore = getManager().getStore(targetStoreUri.getStoreId()); + template = targetStore.getName() + "-" + template; + logger.info(getMarker(), "Starting {} for {}", template, escortFileName); + + String targetFileName = downloadable.getString("fileName"); + + Map parameters = new HashMap<>(); + parameters.put("sourceStoreUri", sourceStoreUri); + parameters.put("tsmFileName", tsmFileName); + parameters.put("fileName", targetFileName); + parameters.put("tempStoreUri", tempStoreUri); + parameters.put("targetStoreUri", targetStoreUri); + parameters.put("relativeTargetPath", downloadable.getString("relativeTargetPath")); + parameters.put("expectedFrameNumber", downloadable.getLong("frames")); + parameters.put("expectedSize", downloadable.getLong("size")); + parameters.put("expectedColorSpace", downloadable.getString("colorSpace")); + parameters.put("escortFile", escortFile.toAbsolutePath().toString()); + + //int priority = downloadable.containsKey("priority") ? downloadable.getInt("priority") : 0; + //mindenki egyforman induljon 20.09.07 + //a NEXIO egy kicsit fontosabb 20.09.08. megsem 20.09.09 + //int priority = escortFile.toString().contains("NEXIO") ? 51 : 50; + //megszakitasok miatt a mar lejott mozikat vegyuk elore 20.09.09 + + int priority = 50; + if (targetExists(tempStoreUri, targetFileName)) + priority = 51; + + //parameterek: IJobRuntime parent, String template, String name, int priority, Map parameters + IJobRuntime child = getEngine().submit(null, e -> { + IJobRuntime r = (IJobRuntime) e.getSource(); + if (r.isDisableRetry()) + return; + + if (e.getStatus().equals(JobStatus.CANCELED) || e.getStatus().equals(JobStatus.SUSPENDED)) + EscortFiles.removeCatchedFile(escortFile); + }, template, "Visszatöltés", priority, IJobEngine.DEFAULT_OWNER, parameters); + if (child == null) + throw new Exception("Submit error."); + + child.setDescription(escortFileName); + + child.setRelated(downloadable.getString("title") + "|" + targetStore.getName()); + EscortFiles.createCatchedFile(escortFile); + } catch (Exception e) { + logger.error(getSessionMarker(), e.getMessage()); + return false; + } + + return true; + } + + private boolean targetExists(StoreUri storeUri, String fileName) { + try { + Path targetFile = Paths.get(storeUri.toString(true), fileName); + if (targetFile.toFile().exists()) { + logger.info(getMarker(), "Retrieved file {} exists", fileName); + return true; + } + } catch (Exception e) { + logger.error(getMarker(), e.getMessage()); + } + logger.info(getMarker(), "Retrieved file {} not exists yet", fileName); + return false; + } +} diff --git a/server/-product/production/AMC/jobs/steps/GenericArchiveCheckerStep.java b/server/-product/production/AMC/jobs/steps/GenericArchiveCheckerStep.java new file mode 100644 index 00000000..60858b2c --- /dev/null +++ b/server/-product/production/AMC/jobs/steps/GenericArchiveCheckerStep.java @@ -0,0 +1,94 @@ +package user.jobengine.server.steps; + +import java.io.IOException; +import java.nio.file.Paths; +import java.sql.Timestamp; +import java.util.List; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + +import user.commons.DownloadableMedia; +import user.commons.RemoteFile; +import user.commons.StoreUri; +import user.commons.remotestore.RemoteStoreProtocol; +import user.jobengine.db.Media; +import user.jobengine.server.steps.shared.EscortFiles; + +public class GenericArchiveCheckerStep extends JobStep { + private static final String DOT_MXF = ".MXF"; + private static final Logger logger = LogManager.getLogger(); + + private boolean check(Timestamp modified, Timestamp created, Media media) { + boolean create = media == null; + create |= create || !media.getModified().equals(modified); + create |= create || !media.getCreated().equals(created); + return create; + } + + @StepEntry + public Object[] execute(String sourceStoreName, String protocol, String wildCard, String escortStoreName, int limit) throws Exception { + StoreUri sourceStoreUri = null; + int count = limit; + + try { + sourceStoreUri = getManager().getStoreUri(sourceStoreName, Enum.valueOf(RemoteStoreProtocol.class, protocol)); + StoreUri escortSoreUri = getManager().getStoreUri(escortStoreName, RemoteStoreProtocol.LOCAL); + StoreUri targetStoreUri = getManager().getStoreUri("TSM", RemoteStoreProtocol.TSM); + + String outputPath = Paths.get(escortSoreUri.toString(true)).toString(); + + //az FTP store nem tamogatja az asszinkron listazast + List remoteFiles = sourceStoreUri.getRemoteFiles(wildCard); + for (RemoteFile remoteFile : remoteFiles) { + if (processClip(sourceStoreName, remoteFile, sourceStoreUri, targetStoreUri, outputPath)) + count--; + + if (count == 0) { + logger.info("Limit reached: {}", limit); + break; + } + + } + } catch (Exception e) { + throw e; + } finally { + if (sourceStoreUri != null) + sourceStoreUri.cleanUp(); + } + + return null; + } + + private boolean processClip(String storeName, RemoteFile remoteFile, StoreUri sourceStoreUri, StoreUri targetStoreUri, String outputPath) { + boolean result = false; + String title = remoteFile.getName(); + Timestamp modified = Timestamp.from(remoteFile.getModify().toInstant()); + Timestamp created = Timestamp.from(remoteFile.getCreated().toInstant()); + + //TODO a masolas alatt allo klippeket kiszurni + if (title.contains(".")) + title = title.substring(0, title.indexOf(".")); + long size = remoteFile.getSize(); + + Media media = getManager().getMedia(title); + + boolean createEscort = check(modified, created, media); + + if (createEscort) { + long mediaId = media == null ? 0 : media.getId(); + String fileName = title + DOT_MXF; + DownloadableMedia archive = DownloadableMedia.create(title, fileName, modified, created, 0, size, sourceStoreUri.getId(), targetStoreUri.getId(), + mediaId); + String escortFileName = storeName + "." + remoteFile.getName(); + try { + result = EscortFiles.createMetadataIfNotExists(outputPath, escortFileName, archive.toPrettyString("")); + } catch (IOException e) { + logger.error("Escort file creation error.", e); + } + } + + return result; + } + +} diff --git a/server/-product/production/AMC/jobs/steps/HarrisMissingMaterialCheckerStep.java b/server/-product/production/AMC/jobs/steps/HarrisMissingMaterialCheckerStep.java new file mode 100644 index 00000000..db18fdf1 --- /dev/null +++ b/server/-product/production/AMC/jobs/steps/HarrisMissingMaterialCheckerStep.java @@ -0,0 +1,172 @@ +package user.jobengine.server.steps; + +import java.io.IOException; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.Arrays; +import java.util.List; + +import org.apache.commons.lang.StringUtils; +import org.apache.commons.net.ftp.FTPClient; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + +import com.ibm.nosql.json.api.BasicDBList; + +import user.commons.DownloadableMedia; +import user.commons.StoreUri; +import user.commons.harris.HarrisRecord; +import user.commons.harris.VICFileParser; +import user.commons.remotestore.FtpDirectoryLister; +import user.commons.remotestore.RemoteStoreProtocol; +import user.jobengine.db.Media; +import user.jobengine.server.steps.shared.EscortFiles; + +public class HarrisMissingMaterialCheckerStep extends JobStep { + private static final Logger logger = LogManager.getLogger(HarrisMissingMaterialCheckerStep.class); + private static final String lineFormat = System.getProperty("harris.vic.lineformat", + "A TTTTTTTT LLLLLLLLLLL MMMMMMMMMMMMMMMMMMMMMMM X DDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDD"); + private static final String validTypeCodes = System.getProperty("harris.vic.validtypecodes", "SPACE,E"); + private static final String appendExtension = System.getProperty("missingmaterial.appendextension", ".mxf"); + + @StepEntry + public Object[] execute(BasicDBList vicFiles, String escortStoreName, String targetStoreName, String targetProtocol) throws Exception { + StoreUri escortStoreUri = null; + StoreUri targetStoreUri = null; + try { + getJobRuntime().setRelated(targetStoreName + " restore"); + + escortStoreUri = getManager().getStoreUri(escortStoreName, RemoteStoreProtocol.LOCAL); + targetStoreUri = getManager().getStoreUri(targetStoreName, Enum.valueOf(RemoteStoreProtocol.class, targetProtocol)); + StoreUri sourceStoreUri = getManager().getStoreUri("TSM", RemoteStoreProtocol.TSM); + List records = null; + for (Object vicFile : vicFiles) { + if (getJobRuntime().isWaitingCancel()) { + cancel(); + break; + } + + Path vicFilePath = Paths.get(String.valueOf(vicFile)); + + if (!vicFilePath.toFile().exists()) { + logger.warn("File not exists: {}", vicFilePath); + continue; + } + + logger.info(getMarker(), "Processing {}", vicFilePath); + records = processVICFile(records, vicFilePath); + } + + if (records != null && !getJobRuntime().isWaitingCancel()) { + FTPClient client = ((FtpDirectoryLister) targetStoreUri.getLister()).connect(); + logger.info(getMarker(), "Reading target file list"); + List fileNames = Arrays.asList(client.listNames()); + logger.info(getMarker(), "Target file list reading completed, mediabase size is {}", fileNames.size()); + int i = 0; + for (HarrisRecord record : records) { + + if (getJobRuntime().isWaitingCancel()) { + cancel(); + break; + } + + boolean contains = fileNames.contains(record.getFileName() + ".lxf"); + if (contains) + logger.info(getMarker(), "File {} already exists on {}", record.getFileName(), targetStoreName); + else + processRecord(record, targetStoreName, sourceStoreUri, targetStoreUri, escortStoreUri); + i++; + int progress = i * 100 / records.size(); + setProgress(progress); + } + } + + if (records == null || records.size() == 0) + logger.info(getMarker(), "Nothing to do"); + else + logger.info(getMarker(), "Completed"); + + setProgress(100); + } catch (Exception e) { + logger.error(e.getMessage()); + throw e; + } finally { + if (escortStoreUri != null) + escortStoreUri.cleanUp(); + if (targetStoreUri != null) + targetStoreUri.cleanUp(); + } + + return null; + } + + private boolean processRecord(HarrisRecord record, String targetStoreName, StoreUri sourceStoreUri, StoreUri targetStoreUri, StoreUri escortStoreUri) + throws Exception { + + String fileName = record.getFileName(); + String outputPath = Paths.get(escortStoreUri.toString(true)).toString(); + + if (StringUtils.isNotBlank(appendExtension)) + fileName += appendExtension; + + String escortFileName = targetStoreName + "." + fileName; + + //ha mar letezik, nem toltjuk fel ujra (a vic fajlbol kezzel torlik valamikor...) + try { + if (EscortFiles.isMetadataExists(outputPath, escortFileName)) { + logger.debug(getMarker(), "Status file already exists {}", escortFileName); + return true; + } + } catch (IOException e1) { + logger.info(getMarker(), "Status file check error for {}", fileName); + return true; + } + + //nincs bent MC-ben + Media media = getManager().getMedia(record.getFileName()); + if (media == null) { + logger.warn(getSessionMarker(), "File is not available in archive {}", record.getFileName()); + return true; + } + + // if (existsOnNexio(record, targetStoreName, targetStoreUri)) { + // logger.info(getMarker(), "File {} already exists on {}", record.getFileName(), targetStoreName); + // return false; + // } + + DownloadableMedia downloadable = DownloadableMedia.create(media.getTitle(), fileName, media.getModified(), media.getCreated(), media.getLength(), 0L, + sourceStoreUri.getId(), targetStoreUri.getId(), media.getId()); + + //alapbol magasabb (1-99) + downloadable.put("priority", 50); + downloadable.put("isNexio", true); + + try { + EscortFiles.createMetadata(outputPath, escortFileName, downloadable.toPrettyString("")); + logger.info(getMarker(), "Status file created {}", escortFileName); + } catch (Exception e) { + logger.error("Can't create escort file {}. System message is: {}", escortFileName, e.getMessage()); + } + return true; + } + + private List processVICFile(List records, Path vicFilePath) { + VICFileParser parser = null; + try { + parser = new VICFileParser(vicFilePath, lineFormat, validTypeCodes); + + List currentRecords = parser.getRecords(); + if (records == null) + records = currentRecords; + else { + if (currentRecords != null) + records.addAll(currentRecords); + } + + } catch (Exception e) { + logger.error(e.getMessage()); + } + return records; + } + +} diff --git a/server/-product/production/AMC/jobs/steps/MetadataPersisterStep.java b/server/-product/production/AMC/jobs/steps/MetadataPersisterStep.java new file mode 100644 index 00000000..2a04cc47 --- /dev/null +++ b/server/-product/production/AMC/jobs/steps/MetadataPersisterStep.java @@ -0,0 +1,118 @@ +package user.jobengine.server.steps; + +import java.nio.file.Path; +import java.nio.file.Paths; +import java.sql.Timestamp; +import java.time.Instant; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + +import user.commons.DownloadableMedia; +import user.commons.StoreUri; +import user.jobengine.db.IItemManager; +import user.jobengine.db.Item; +import user.jobengine.db.Media; +import user.jobengine.db.MediaFile; +import user.jobengine.server.steps.shared.EscortFiles; + +public class MetadataPersisterStep extends JobStep { + private static final String GENERIC = "Generic"; + private static final Logger logger = LogManager.getLogger(); + + @StepEntry + public Object[] execute(StoreUri targetStoreUri, String escortFile) throws Exception { + try { + + Path escortFilePath = Paths.get(escortFile); + //ha nem ment at a letoltott file az ellenorzesen, akkor már torolve lett + if (escortFilePath.toFile().exists()) { + String escortFileName = escortFilePath.getFileName().toString(); + String sourceStoreName = escortFileName.substring(0, escortFileName.indexOf(".")); + DownloadableMedia downloadable = EscortFiles.decode(escortFilePath); + String title = downloadable.getString("title"); + String fileName = downloadable.getString("fileName"); + long mediaId = downloadable.containsKey("mediaId") ? downloadable.getLong("mediaId") : 0; + String fileTypeName = fileName.substring(fileName.lastIndexOf(".") + 1).toUpperCase(); + + IItemManager manager = getManager(); + + if (mediaId == 0) { + mediaId = tryGetMediaId(title, mediaId); + } + + //TODO ha van mediaId, akkor update!!!!!!!!! + + Timestamp modified = downloadable.getTimestamp("modified"); + Timestamp created = downloadable.getTimestamp("created"); + long frames = downloadable.getLong("frames"); + + String targetStoreName = getManager().getStore(targetStoreUri.getStoreId()).getName(); + if (mediaId == 0) { + //ez elszallhat, mert nincs filetype vagy store + MediaFile mediaFile = manager.createMediaFile(fileName, fileTypeName, targetStoreName); + //ez elszallhat, mert nincs itemtype + Item item = manager.createItem(GENERIC, title, sourceStoreName, null); + //ez elszallhat, mert nincs itemtype + Media media = manager.createMedia(GENERIC, title, sourceStoreName, null); + + //TODO megoldani egy tranzakcioban + item.add(); + media.setItemId(item.getId()); + media.setLength(frames); + media.setCreated(created); + media.setModified(modified); + media.setArchived(Timestamp.from(Instant.now())); + media.add(); + + //TODO + //mediaFile.setFileSize(fileSize); + mediaFile.setMediaId(media.getId()); + mediaFile.setLastModified(modified); + mediaFile.add(); + } else { + Media media = manager.getMedia(mediaId); + media.setLength(frames); + media.setCreated(created); + media.setModified(modified); + media.setArchived(Timestamp.from(Instant.now())); + manager.modify(media); + + //TODO + //mediaFile.setFileSize(fileSize); + MediaFile mediaFile = manager.getSystemMediaFile(media); + if (mediaFile == null) { + mediaFile = manager.createMediaFile(fileName, fileTypeName, targetStoreName); + mediaFile.setMediaId(media.getId()); + mediaFile.setLastModified(modified); + mediaFile.add(); + } else { + mediaFile.setRelativePath(fileName); + mediaFile.setLastModified(modified); + manager.modify(mediaFile); + } + } + + } else + logger.error(getMarker(), "Status file does not exists {}", escortFilePath); + } catch (Exception e) { + logger.error(getSessionMarker(), e.getMessage()); + throw e; + } + return null; + } + + private long tryGetMediaId(String title, long mediaId) { + Media media = null; + //duplikatum miatt + try { + media = getManager().getMedia(title); + if (media != null) + mediaId = media.getId(); + } catch (Exception e) { + logger.error(getMarker(), "Media with title {} is not available. System message is: {}", title, e.getMessage()); + } + return mediaId; + } + +} diff --git a/server/-product/production/AMC/jobs/steps/NEXIO1TransferFromStep.java b/server/-product/production/AMC/jobs/steps/NEXIO1TransferFromStep.java new file mode 100644 index 00000000..7963459f --- /dev/null +++ b/server/-product/production/AMC/jobs/steps/NEXIO1TransferFromStep.java @@ -0,0 +1,30 @@ +package user.jobengine.server.steps; + +import java.nio.file.Path; +import java.nio.file.Paths; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + +import user.commons.StoreUri; + +public class NEXIO1TransferFromStep extends TransferStep { + private static final Logger logger = LogManager.getLogger(); + + @Override + @StepEntry + public Object[] execute(StoreUri sourceStoreUri, String sourceFileName, StoreUri targetStoreUri, String targetFileName) throws Exception { + Object[] result = null; + try { + Path targetFile = Paths.get(targetStoreUri.toString(true), targetFileName); + if (targetFile.toFile().exists()) { + logger.info(getMarker(), "Using previously retrived file {}", targetFile); + } else { + result = super.execute(sourceStoreUri, sourceFileName, targetStoreUri, targetFileName); + } + } catch (Exception e) { + logger.error(getMarker(), e.getMessage()); + } + return result; + } +} diff --git a/server/-product/production/AMC/jobs/steps/NEXIO1TransferToStep.java b/server/-product/production/AMC/jobs/steps/NEXIO1TransferToStep.java new file mode 100644 index 00000000..5af53df1 --- /dev/null +++ b/server/-product/production/AMC/jobs/steps/NEXIO1TransferToStep.java @@ -0,0 +1,54 @@ +package user.jobengine.server.steps; + +import java.util.Arrays; +import java.util.List; + +import org.apache.commons.net.ftp.FTPClient; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + +import user.commons.StoreUri; +import user.commons.remotestore.FtpDirectoryLister; + +public class NEXIO1TransferToStep extends TransferStep { + private static final Logger logger = LogManager.getLogger(); + + @Override + @StepEntry + public Object[] execute(StoreUri sourceStoreUri, String sourceFileName, StoreUri targetStoreUri, String targetFileName) throws Exception { + // + // try { + // String title = targetFileName; + // if (title.contains(".")) + // title = title.substring(0, title.lastIndexOf(".")); + // RemoteFile remoteFile = targetStoreUri.getFileWithContent(title + ".xml"); + // if (remoteFile != null && remoteFile.getContent() != null) { + // logger.info("File already exists on NEXIO1 {}", targetFileName); + // return null; + // } + // } catch (Exception e) { + // logger.error(e.getMessage()); + // } finally { + // targetStoreUri.cleanUp(); + // } +/* + try { + FTPClient client = ((FtpDirectoryLister) targetStoreUri.getLister()).connect(); + logger.info(getMarker(), "Reading target file list"); + List fileNames = Arrays.asList(client.listNames()); + logger.info(getMarker(), "Target file list reading completed, mediabase size is {}", fileNames.size()); + String title = targetFileName.replace(".mxf", ".lxf"); + boolean contains = fileNames.contains(title); + if (contains) { + logger.info(getMarker(), "File already exists {} on {}", title, getManager().getStore(targetStoreUri.getStoreId()).getName()); + return null; + } + } catch (Exception e) { + logger.catching(e); + } finally { + targetStoreUri.cleanUp(); + } +*/ + return super.execute(sourceStoreUri, sourceFileName, targetStoreUri, targetFileName); + } +} diff --git a/server/-product/production/AMC/jobs/steps/NEXIO2TransferFromStep.java b/server/-product/production/AMC/jobs/steps/NEXIO2TransferFromStep.java new file mode 100644 index 00000000..a698c6e8 --- /dev/null +++ b/server/-product/production/AMC/jobs/steps/NEXIO2TransferFromStep.java @@ -0,0 +1,30 @@ +package user.jobengine.server.steps; + +import java.nio.file.Path; +import java.nio.file.Paths; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + +import user.commons.StoreUri; + +public class NEXIO2TransferFromStep extends TransferStep { + private static final Logger logger = LogManager.getLogger(); + + @Override + @StepEntry + public Object[] execute(StoreUri sourceStoreUri, String sourceFileName, StoreUri targetStoreUri, String targetFileName) throws Exception { + Object[] result = null; + try { + Path targetFile = Paths.get(targetStoreUri.toString(true), targetFileName); + if (targetFile.toFile().exists()) { + logger.info(getMarker(), "Using previously retrived file {}", targetFile); + } else { + result = super.execute(sourceStoreUri, sourceFileName, targetStoreUri, targetFileName); + } + } catch (Exception e) { + logger.error(getMarker(), e.getMessage()); + } + return result; + } +} diff --git a/server/-product/production/AMC/jobs/steps/NEXIO2TransferToStep.java b/server/-product/production/AMC/jobs/steps/NEXIO2TransferToStep.java new file mode 100644 index 00000000..20c81a68 --- /dev/null +++ b/server/-product/production/AMC/jobs/steps/NEXIO2TransferToStep.java @@ -0,0 +1,53 @@ +package user.jobengine.server.steps; + +import java.util.Arrays; +import java.util.List; + +import org.apache.commons.net.ftp.FTPClient; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + +import user.commons.StoreUri; +import user.commons.remotestore.FtpDirectoryLister; + +public class NEXIO2TransferToStep extends TransferStep { + private static final Logger logger = LogManager.getLogger(); + + @Override + @StepEntry + public Object[] execute(StoreUri sourceStoreUri, String sourceFileName, StoreUri targetStoreUri, String targetFileName) throws Exception { + // try { + // String title = targetFileName; + // if (title.contains(".")) + // title = title.substring(0, title.lastIndexOf(".")); + // RemoteFile remoteFile = targetStoreUri.getFileWithContent(title + ".xml"); + // if (remoteFile != null && remoteFile.getContent() != null) { + // logger.info("File already exists on NEXIO2 {}", targetFileName); + // return null; + // } + // } catch (Exception e) { + // logger.error(e.getMessage()); + // } finally { + // targetStoreUri.cleanUp(); + // } +/* + try { + FTPClient client = ((FtpDirectoryLister) targetStoreUri.getLister()).connect(); + logger.info(getMarker(), "Reading target file list"); + List fileNames = Arrays.asList(client.listNames()); + logger.info(getMarker(), "Target file list reading completed, mediabase size is {}", fileNames.size()); + String title = targetFileName.replace(".mxf", ".lxf"); + boolean contains = fileNames.contains(title); + if (contains) { + logger.info(getMarker(), "File already exists {} on {}", title, getManager().getStore(targetStoreUri.getStoreId()).getName()); + return null; + } + } catch (Exception e) { + logger.catching(e); + } finally { + targetStoreUri.cleanUp(); + } +*/ + return super.execute(sourceStoreUri, sourceFileName, targetStoreUri, targetFileName); + } +} diff --git a/server/-product/production/AMC/jobs/steps/NEXIOArchiveCheckerStep.java b/server/-product/production/AMC/jobs/steps/NEXIOArchiveCheckerStep.java new file mode 100644 index 00000000..05485171 --- /dev/null +++ b/server/-product/production/AMC/jobs/steps/NEXIOArchiveCheckerStep.java @@ -0,0 +1,188 @@ +package user.jobengine.server.steps; + +import java.io.IOException; +import java.nio.file.Paths; +import java.sql.Timestamp; +import java.text.SimpleDateFormat; +import java.time.Duration; +import java.time.Instant; +import java.util.Iterator; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + +import user.commons.DownloadableMedia; +import user.commons.StoreUri; +import user.commons.nexio.api.Clip; +import user.commons.nexio.api.ClipNotFoundException; +import user.commons.nexio.api.Controller; +import user.commons.nexio.api.Mediabase; +import user.commons.nexio.server.protocol.ProtocolException; +import user.commons.remotestore.RemoteStoreProtocol; +import user.jobengine.db.Media; +import user.jobengine.server.steps.shared.EscortFiles; +import user.jobengine.db.MediaFile; + +public class NEXIOArchiveCheckerStep extends JobStep { + private static final String DOT_MXF = ".mxf"; + private static final Logger logger = LogManager.getLogger(); + private SimpleDateFormat df = new SimpleDateFormat("yyyyMMdd-HHmmss"); + + private boolean check(Timestamp modified, Timestamp created, int duration, Media media, int videoFormat, int lastModifiedHours) { + String title = media.getTitle(); + + //formatum + if (!(videoFormat == 3 || videoFormat == 19)) { + logger.debug(getMarker(), "{} unsupported video format", videoFormat); + return false; + } + + if (!checkModifiedByTime(title, modified, lastModifiedHours)) + return false; + + //utolso modositas + MediaFile mediaFile = getManager().getSystemMediaFile(media); + boolean isModified = true; + if (mediaFile == null) { + logger.info(getMarker(), "{} system media file not available", title); + return false; + } + + String mcMod = df.format(mediaFile.getLastModified()); + String nexioMod = df.format(modified); + isModified = mcMod.compareTo(nexioMod) < 0; + if (isModified) { + logger.info(getMarker(), "{} modification time: MC {} < NEXIO {}", title, mcMod, nexioMod); + return true; + } + + boolean isLengthChanged = media.getLength() > 0 && Math.abs(media.getLength() - duration) > 3; + if (isLengthChanged) { + logger.info(getMarker(), "{} length: MC {} != NEXIO {}", title, media.getLength(), duration); + return true; + } + + return false; + } + + private boolean checkModifiedByTime(String title, Timestamp modified, int lastModifiedHours) { + //utolso modositas a mosthoz kepest + //The result of this method can be a negative period if the end is before the start. + long modifiedHours = Duration.between(modified.toInstant(), Instant.now()).toHours(); + if (modifiedHours < 0) { + logger.info(getMarker(), "{} modification time greather than now", title); + return false; + } + if (modifiedHours < lastModifiedHours) { + logger.debug(getMarker(), "{} modification time is too cloose for now", title); + return false; + } + + return true; + } + + @StepEntry + public Object[] execute(String sourceStoreName, String escortStoreName, int lastModifiedHours) throws Exception { + Controller controller = null; + + try { + getJobRuntime().setRelated(sourceStoreName + " archive"); + + StoreUri nexioStoreUri = getManager().getStoreUri(sourceStoreName, RemoteStoreProtocol.NEXIO); + StoreUri sourceStoreUri = getManager().getStoreUri(sourceStoreName, RemoteStoreProtocol.FTP); + StoreUri targetStoreUri = getManager().getStoreUri("TSM", RemoteStoreProtocol.TSM); + StoreUri escortStoreUri = getManager().getStoreUri(escortStoreName, RemoteStoreProtocol.LOCAL); + String outputPath = Paths.get(escortStoreUri.toString(true)).toString(); + + logger.info(getMarker(), "Connecting mediabase {}", nexioStoreUri.toString()); + if (nexioStoreUri.getPortNumber() == 0) + controller = new Controller(nexioStoreUri.getUri(), nexioStoreUri.getPortNumber()); + else + controller = new Controller(nexioStoreUri.getUri()); + controller.connect(); + Mediabase mediabase = controller.getMediabase(); + + Iterator clips = mediabase.getClips(); + logger.info(getMarker(), "Processing clips"); + + processClips(sourceStoreName, sourceStoreUri, targetStoreUri, outputPath, clips, lastModifiedHours); + } catch (Exception e) { + logger.error(getSessionMarker(), e.getMessage() == null ? "Unknown error" : e.getMessage()); + throw e; + } finally { + setProgress(100); + if (controller != null) + controller.disconnect(); + } + + return null; + } + + private void processClips(String storeName, StoreUri sourceStoreUri, StoreUri targetStoreUri, String outputPath, Iterator clips, + int lastModifiedHours) throws ClipNotFoundException, IOException, ProtocolException { + int processed = 0; + while (clips.hasNext()) { + + if (getJobRuntime().isWaitingCancel()) { + cancel(); + break; + } + + Clip clip = clips.next(); + + try { + String title = null; + Timestamp created = null; + Timestamp modified = null; + int frames = 0; + int videoFormat = 0; + frames = clip.getDuration(); + + //ures clip + if (frames == 1) + continue; + + title = clip.getXid().get(); + created = Timestamp.from(clip.getRecordDateTimestamp().toInstant()); + modified = Timestamp.from(clip.getModifiedTimestamp().toInstant()); + videoFormat = clip.getVideoFormat(); + + Media media = getManager().getMedia(title); + logger.debug(getMarker(), "Checking {}", title); + boolean createEscort = false; + + long mediaId = 0; + if (media == null) { + if (checkModifiedByTime(title, modified, lastModifiedHours)) { + createEscort = true; + logger.info(getMarker(), "{} not archived yet", title); + } + } else { + mediaId = media.getId(); + createEscort = check(modified, created, frames, media, videoFormat, lastModifiedHours); + } + + if (createEscort && !title.contains("*")) { + String fileName = title + DOT_MXF; + DownloadableMedia downloadable = DownloadableMedia.create(title, fileName, modified, created, frames, 0, sourceStoreUri.getId(), + targetStoreUri.getId(), mediaId); + String escortFileName = storeName + "." + downloadable.getString("fileName"); + if (EscortFiles.createMetadataIfNotExists(outputPath, escortFileName, downloadable.toPrettyString(""))) { + logger.info(getMarker(), "Archive status file created for {}", fileName); + } else { + logger.info(getMarker(), "Archive status file already exists for {}", fileName); + } + + } + + processed++; + if (processed % 100 == 0) + logger.info(getMarker(), "Processed {} records", processed); + } catch (Exception e) { + logger.error(getSessionMarker(), e.getMessage()); + continue; + } + } + } + +} diff --git a/server/-product/production/AMC/jobs/steps/NEXIOMetadataPersisterStep.java b/server/-product/production/AMC/jobs/steps/NEXIOMetadataPersisterStep.java new file mode 100644 index 00000000..ffaf54a3 --- /dev/null +++ b/server/-product/production/AMC/jobs/steps/NEXIOMetadataPersisterStep.java @@ -0,0 +1,110 @@ +package user.jobengine.server.steps; + +import java.nio.file.Path; +import java.nio.file.Paths; +import java.sql.Timestamp; +import java.text.SimpleDateFormat; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.w3c.dom.Document; +import org.w3c.dom.Node; +import org.w3c.dom.NodeList; + +import user.commons.DownloadableMedia; +import user.commons.RemoteFile; +import user.commons.StoreUri; +import user.jobengine.db.Media; +import user.jobengine.db.MediaFile; +import user.jobengine.server.steps.shared.EscortFiles; + +public class NEXIOMetadataPersisterStep extends JobStep { + private static final Logger logger = LogManager.getLogger(); + private static final String DOT_XML = ".xml"; + + @StepEntry + public Object[] execute(StoreUri targetStoreUri, String escortFile) throws Exception { + try { + Path escortFilePath = Paths.get(escortFile); + if (!escortFilePath.toFile().exists()) { + logger.error(getMarker(), "Status file does not exists {}", escortFilePath); + return null; + } + + DownloadableMedia downloadable = EscortFiles.decode(escortFilePath); + if (!downloadable.containsKey("isNexio") || !downloadable.getBoolean("isNexio")) + return null; + + if (!downloadable.containsKey("mediaId")) + throw new NullPointerException("No media ID found for " + downloadable.getString("title")); + + Media media = getManager().getMedia(downloadable.getLong("mediaId")); + MediaFile mediaFile = getManager().getSystemMediaFile(media); + if (mediaFile == null) + throw new NullPointerException("No TSM media file found for " + downloadable.getString("title")); + + RemoteFile remoteFile = targetStoreUri.getFileWithContent(downloadable.getString("title") + ".xml"); + + //TODO mediafile lastmodified = feltoltott file nexio xml lastmodified + Timestamp lastModified = getLastModified(remoteFile); + logger.info("Setting last modification time on {} to {}", downloadable.getString("title"), lastModified); + mediaFile.setLastModified(lastModified); + getManager().modify(mediaFile); + + // if (!RemoteStoreProtocol.FTP.equals(targetStoreUri.getProtocol())) + // return null; + + // + // String title = downloadable.getString("title"); + // String fileName = downloadable.getString("fileName"); + // Date created = downloadable.getDate("created"); + // Date modified = downloadable.getDate("modified"); + // byte[] metadata = EscortFiles.createNEXIODatesMeta(title, created, modified); + // uploadMeta(targetStoreUri, fileName, metadata); + } catch (Exception e) { + logger.error(getSessionMarker(), e.getMessage()); + throw e; + } finally { + if (targetStoreUri != null) + targetStoreUri.cleanUp(); + } + return null; + } + + private Timestamp getLastModified(RemoteFile remoteFile) throws Exception { + Timestamp result = null; + Document document = EscortFiles.createNEXIOMeta(remoteFile.getContent()); + Node firstChild = document.getFirstChild(); + NodeList list = firstChild.getChildNodes(); + for (int i = 0; i < list.getLength(); i++) { + Node item = list.item(i); + if (item.getNodeName().equals("ModifiedTimeStamp")) { + SimpleDateFormat df = new SimpleDateFormat("MM-dd-yyyy (HH:mm:ss)"); + result = Timestamp.from(df.parse(item.getTextContent()).toInstant()); + break; + } + } + return result; + } + + // private void uploadMeta(StoreUri storeUri, String fileName, byte[] metadata) throws Exception { + // logger.info(getMarker(), "Uploading metadata for {}", fileName); + // OutputStream outStream = null; + // try { + // FTPClient sourceFtp = ((FtpDirectoryLister) storeUri.getLister()).connect(); + // outStream = sourceFtp.storeFileStream(fileName + DOT_XML); + // if (outStream == null) + // throw new NullPointerException("Can not create: " + fileName + DOT_XML + " on FTP. Reply is:" + sourceFtp.getReplyString()); + // outStream.write(metadata); + // outStream.flush(); + // } catch (Exception e) { + // logger.catching(e); + // throw e; + // } finally { + // if (outStream != null) + // outStream.close(); + // storeUri.cleanUp(); + // } + // } + +} diff --git a/server/-product/production/AMC/jobs/steps/PASAPOOLTransferToStep.java b/server/-product/production/AMC/jobs/steps/PASAPOOLTransferToStep.java new file mode 100644 index 00000000..0c69ee3a --- /dev/null +++ b/server/-product/production/AMC/jobs/steps/PASAPOOLTransferToStep.java @@ -0,0 +1,102 @@ +package user.jobengine.server.steps; + +import java.nio.file.Path; +import java.nio.file.Paths; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.apache.logging.log4j.Marker; +import org.apache.logging.log4j.MarkerManager; + +import user.commons.RemoteFile; +import user.commons.StoreUri; +import user.commons.mediaarea.MediaArea; +import user.commons.strings.FileSizeUtils; + +public class PASAPOOLTransferToStep extends TransferStep { + private static final Logger logger = LogManager.getLogger(); + private static final String CODEC_PROFILE_HIGH = "HIGH"; + private static final String CODEC_PROFILE_MAIN = "MAIN"; + private static final String CODEC_PROFILE_422 = "4:2:2"; + private static final String DISPLAY_ASPECT_16_9 = "16:9"; + private static final String DISPLAY_ASPECT_4_3 = "4:3"; + private StoreUri sourceStoreUri; + private String sourceFileName; + private Marker marker = MarkerManager.getMarker("MEDIAPROFILE"); + + @Override + @StepEntry + public Object[] execute(StoreUri sourceStoreUri, String sourceFileName, StoreUri targetStoreUri, String targetFileName) throws Exception { + this.sourceStoreUri = sourceStoreUri; + this.sourceFileName = sourceFileName; + long start = System.currentTimeMillis(); + Object[] result = super.execute(sourceStoreUri, sourceFileName, targetStoreUri, targetFileName); + RemoteFile remoteFile = sourceStoreUri.getRemoteFile(targetFileName); + if (remoteFile != null && remoteFile.getSize() > 0) { + long diff = (System.currentTimeMillis() - start) / 1000; + if (diff > 0) { + long bytesSpeed = remoteFile.getSize() / diff; + String speed = FileSizeUtils.sizeAsString(bytesSpeed); + logger.info(getMarker(), "{} size is {}, upload speed was {}/sec", targetFileName, FileSizeUtils.sizeAsString(remoteFile.getSize()), speed); + } + } + return result; + } + + @Override + protected StoreUri getTargetStoreUri(StoreUri targetStoreUri) { + StoreUri result = targetStoreUri; + try { + Path filePath = Paths.get(sourceStoreUri.toString(true), sourceFileName); + MediaArea mi = new MediaArea(filePath); + mi.process(); + + if (isHD(mi)) { + logger.info(marker, "{} is MOVED_HD", sourceFileName); + result = getManager().getStoreUri("PEABLEBEACH", targetStoreUri.getProtocol()); + return result; + } + + if (isSD_HIGH_16_9(mi)) { + logger.info(marker, "{} is CONVSD16_9", sourceFileName); + result = getManager().getStoreUri("SELENIOPOOL_16_9", targetStoreUri.getProtocol()); + return result; + } + + if (isSD_HIGH_4_3(mi)) { + logger.info(marker, "{} is CONVSD4_3", sourceFileName); + result = getManager().getStoreUri("SELENIOPOOL_4_3", targetStoreUri.getProtocol()); + return result; + } + + if (isSD_MAIN_422(mi)) { + logger.info(marker, "{} is MOVED_SD", sourceFileName); + result = getManager().getStoreUri("PEABLEBEACH", targetStoreUri.getProtocol()); + return result; + } + } catch (Exception e) { + logger.error(getSessionMarker(), e.getMessage()); + } + return result; + } + + private boolean isHD(MediaArea mi) { + return mi.getHeight() == 1080; + } + + private boolean isSD_HIGH_16_9(MediaArea mi) { + String formatProfileName = mi.getFormatProfileName(); + return mi.getHeight() < 650 && formatProfileName.contains(CODEC_PROFILE_HIGH) && DISPLAY_ASPECT_16_9.equals(mi.getDisplayAspect()); + } + + private boolean isSD_HIGH_4_3(MediaArea mi) { + String formatProfileName = mi.getFormatProfileName(); + return mi.getHeight() < 650 && formatProfileName.contains(CODEC_PROFILE_HIGH) && DISPLAY_ASPECT_4_3.equals(mi.getDisplayAspect()); + } + + private boolean isSD_MAIN_422(MediaArea mi) { + String formatProfileName = mi.getFormatProfileName(); + return mi.getHeight() < 650 && (formatProfileName.contains(CODEC_PROFILE_MAIN) || formatProfileName.contains(CODEC_PROFILE_422)); + } + +} diff --git a/server/-product/production/AMC/jobs/steps/PBARCHIVETransferFromStep.java b/server/-product/production/AMC/jobs/steps/PBARCHIVETransferFromStep.java new file mode 100644 index 00000000..d47c2b79 --- /dev/null +++ b/server/-product/production/AMC/jobs/steps/PBARCHIVETransferFromStep.java @@ -0,0 +1,21 @@ +package user.jobengine.server.steps; + +import java.nio.file.Path; +import java.nio.file.Paths; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + +import user.commons.StoreUri; + +public class PBARCHIVETransferFromStep extends TransferStep { + private static final Logger logger = LogManager.getLogger(); + + @Override + @StepEntry + public Object[] execute(StoreUri sourceStoreUri, String sourceFileName, StoreUri targetStoreUri, String targetFileName) throws Exception { + Object[] result = null; + result = super.execute(sourceStoreUri, sourceFileName, targetStoreUri, targetFileName); + return result; + } +} diff --git a/server/-product/production/AMC/jobs/steps/PBArchiveCheckerStep.java b/server/-product/production/AMC/jobs/steps/PBArchiveCheckerStep.java new file mode 100644 index 00000000..75b1a357 --- /dev/null +++ b/server/-product/production/AMC/jobs/steps/PBArchiveCheckerStep.java @@ -0,0 +1,151 @@ +package user.jobengine.server.steps; + +import java.io.IOException; +import java.nio.file.Paths; +import java.sql.Timestamp; +import java.time.Duration; +import java.time.Instant; +import java.util.ArrayList; +import java.util.List; + +import org.apache.commons.net.ftp.FTPClient; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + +import user.commons.DownloadableMedia; +import user.commons.RemoteFile; +import user.commons.StoreUri; +import user.commons.nexio.api.ClipNotFoundException; +import user.commons.nexio.server.protocol.ProtocolException; +import user.commons.remotestore.FtpDirectoryLister; +import user.commons.remotestore.RemoteStoreProtocol; +import user.jobengine.db.Media; +import user.jobengine.server.steps.shared.EscortFiles; + +public class PBArchiveCheckerStep extends JobStep { + private static final String DOT_MXF = ".mxf"; + private static final Logger logger = LogManager.getLogger(); + + private boolean checkModifiedByTime(String title, Timestamp modified, int lastModifiedHours) { + long modifiedHours = Duration.between(modified.toInstant(), Instant.now()).toHours(); + if (modifiedHours < 0) { + logger.info(getMarker(), "{} modification time greather than now", title); + return false; + } + if (modifiedHours < lastModifiedHours) { + logger.info(getMarker(), "{} modification time is too cloose for now", title); + return false; + } + + return true; + } + + @StepEntry + public Object[] execute(String sourceStoreName, String escortStoreName, int lastModifiedHours) throws Exception { + logger.info(getMarker(), "Started {}", sourceStoreName); + StoreUri sourceStoreUri = null; + try { + getJobRuntime().setRelated(sourceStoreName + " archive"); + + sourceStoreUri = getManager().getStoreUri(sourceStoreName, RemoteStoreProtocol.FTP); + + StoreUri targetStoreUri = getManager().getStoreUri("TSM", RemoteStoreProtocol.TSM); + StoreUri escortStoreUri = getManager().getStoreUri(escortStoreName, RemoteStoreProtocol.LOCAL); + String outputPath = Paths.get(escortStoreUri.toString(true)).toString(); + + FtpDirectoryLister lister = (FtpDirectoryLister) sourceStoreUri.getLister(); + FTPClient client = lister.connect(); + //a StoreUri alapu listazas nem mukodik, miota a working directory beallitas fix + List remoteFiles = new ArrayList<>(); + + String[] listNames = client.listNames(); + for (String f : listNames) { + logger.info(getMarker(), "File name is {}", f); + + try { + RemoteFile remoteFile = lister.get(f); + if (remoteFile == null) + logger.info(getMarker(), "Remote file for {} is not available", f); + else + remoteFiles.add(remoteFile); + + } catch (Exception iex) { + logger.error(getMarker(), iex.getMessage()); + } + } + + logger.info(getMarker(), "Found {} files in working dir {}", remoteFiles.size(), + client.printWorkingDirectory()); + processClips(sourceStoreName, sourceStoreUri, targetStoreUri, outputPath, remoteFiles, lastModifiedHours); + + } catch (Exception e) { + logger.error(getMarker(), e.getMessage()); + logger.error(getSessionMarker(), e.getMessage() == null ? "Unknown error" : e.getMessage()); + throw e; + } finally { + if (sourceStoreUri != null) + sourceStoreUri.cleanUp(); + setProgress(100); + } + + return null; + } + + private void processClips(String storeName, StoreUri sourceStoreUri, StoreUri targetStoreUri, String outputPath, + List remoteFiles, int lastModifiedHours) + throws ClipNotFoundException, IOException, ProtocolException { + for (RemoteFile remoteFile : remoteFiles) { + + if (getJobRuntime().isWaitingCancel()) { + //logger.info("CANCELED"); + cancel(); + break; + } + try { + + String title = null; + Timestamp created = null; + Timestamp modified = null; + title = remoteFile.getName(); + + int dotIdx = title.lastIndexOf("."); + if (dotIdx > -1) + title = title.substring(0, dotIdx); + + created = remoteFile.getCreated(); + modified = remoteFile.getModify(); + + logger.info(getMarker(), "{} created {} modified {}", title, created, modified); + Media media = getManager().getMedia(title); + boolean createEscort = false; + + if (checkModifiedByTime(title, modified, lastModifiedHours)) + createEscort = true; + + if (!createEscort) + continue; + + if (createEscort) { + long mediaId = 0; + if (media != null) + mediaId = media.getId(); + String fileName = title + DOT_MXF; + DownloadableMedia downloadable = DownloadableMedia.create(title, fileName, modified, created, 0, 0, + sourceStoreUri.getId(), targetStoreUri.getId(), mediaId); + String escortFileName = storeName + "." + downloadable.getString("fileName"); + if (EscortFiles.createMetadataIfNotExists(outputPath, escortFileName, + downloadable.toPrettyString(""))) { + logger.info(getMarker(), "Archive status file created for {}", fileName); + } else { + logger.info(getMarker(), "Archive status file already exists for {}", fileName); + } + } + + } catch (Exception e) { + logger.error(getSessionMarker(), e.getMessage()); + continue; + } + } + } + +} diff --git a/server/-product/production/AMC/jobs/steps/PathItemsCollectorStep.java b/server/-product/production/AMC/jobs/steps/PathItemsCollectorStep.java new file mode 100644 index 00000000..58c379b1 --- /dev/null +++ b/server/-product/production/AMC/jobs/steps/PathItemsCollectorStep.java @@ -0,0 +1,76 @@ +package user.jobengine.server.steps; + +import java.io.IOException; +import java.nio.file.FileVisitResult; +import java.nio.file.FileVisitor; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.nio.file.SimpleFileVisitor; +import java.nio.file.attribute.BasicFileAttributes; +import java.util.ArrayList; +import java.util.List; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + +import com.ibm.nosql.json.api.BasicDBObject; + +/* DO NOT REMOVE! +import user.jobengine.server.steps.FileSearchFilterOptions; +*/ +public class PathItemsCollectorStep extends JobStep { + private static final Logger logger = LogManager.getLogger(); + + protected FileSearchFilterOptions createFileFilter(BasicDBObject filter) { + return new FileSearchFilterOptions(filter); + } + + @StepEntry + public Object[] execute(String sourceFolder, BasicDBObject filter) throws Exception { + Path sourcePath = Paths.get(sourceFolder); + + List files = new ArrayList<>(); + + //teljes rekurzivitas szuressel + FileSearchFilterOptions filterOptions = createFileFilter(filter); + FileVisitor visitor = new SimpleFileVisitor() { + @Override + public FileVisitResult preVisitDirectory(Path dir, BasicFileAttributes attrs) throws IOException { + if (!filterOptions.preAcceptDirectory(dir)) + return FileVisitResult.SKIP_SUBTREE; + return FileVisitResult.CONTINUE; + } + + @Override + public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) throws IOException { + if (filterOptions.acceptFile(file)) { + files.add(file.toString()); + logger.info(file); + } + return FileVisitResult.CONTINUE; + } + + @Override + public FileVisitResult visitFileFailed(Path file, IOException exc) throws IOException { + logger.error(exc); + return FileVisitResult.CONTINUE; + } + + }; + + try { + logger.info(getMarker(), "Visiting {} which exists {}", sourceFolder, sourcePath.toFile().exists()); + Files.walkFileTree(sourcePath, visitor); + } catch (Exception e) { + logger.error(getMarker(), "Error processing '{}'. System message: {}", sourcePath, e.getMessage()); + logger.catching(e); + throw e; + } finally { + } + + logger.info(getMarker(), "Found {} files", files.size()); + setProgress(100); + return new Object[] { files }; + } +} diff --git a/server/-product/production/AMC/jobs/steps/PeableBeachMissingMaterialCheckerStep.210604 b/server/-product/production/AMC/jobs/steps/PeableBeachMissingMaterialCheckerStep.210604 new file mode 100644 index 00000000..3b1a1fa0 --- /dev/null +++ b/server/-product/production/AMC/jobs/steps/PeableBeachMissingMaterialCheckerStep.210604 @@ -0,0 +1,225 @@ +package user.jobengine.server.steps; + +import java.nio.file.DirectoryStream; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collection; +import java.util.LinkedHashSet; +import java.util.List; + +import org.apache.commons.lang.StringUtils; +import org.apache.commons.net.ftp.FTPClient; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + +import com.ibm.nosql.json.api.BasicDBList; + +import user.commons.DownloadableMedia; +import user.commons.StoreUri; +import user.commons.remotestore.FtpDirectoryLister; +import user.commons.remotestore.RemoteStoreProtocol; +import user.jobengine.db.Media; +import user.jobengine.server.steps.PBQuery.MMMedia; + +public class PeableBeachMissingMaterialCheckerStep extends JobStep { + private static final Logger logger = LogManager.getLogger(); + private static final String appendExtension = System.getProperty("missingmaterial.appendextension", ".mxf"); + + @StepEntry + public Object[] execute(String escortStoreName, BasicDBList lookupStoreNames, String targetStoreName, String targetProtocol, String primaryEndPoint, + String primaryUserName, String primaryPassword, String secondaryEndPoint, String secondaryUserName, String secondaryPassword, int rangeForwardHours) + throws Exception { + StoreUri escortStoreUri = null; + try { + getJobRuntime().setRelated(targetStoreName + " restore"); + + escortStoreUri = getManager().getStoreUri(escortStoreName, RemoteStoreProtocol.LOCAL); + StoreUri sourceStoreUri = getManager().getStoreUri("TSM", RemoteStoreProtocol.TSM); + StoreUri targetStoreUri = getManager().getStoreUri(targetStoreName, Enum.valueOf(RemoteStoreProtocol.class, targetProtocol)); + + LinkedHashSet poolContent = new LinkedHashSet<>(); + + int progress = 0; + int count = lookupStoreNames.size(); + int processed = 0; + + for (Object lookup : lookupStoreNames) { + if (getJobRuntime().isWaitingCancel()) { + cancel(); + return null; + } + String lookupStoreName = String.valueOf(lookup); + + StoreUri lookupStoreUri = getManager().getStoreUri(lookupStoreName, Enum.valueOf(RemoteStoreProtocol.class, targetProtocol)); + logger.info(getMarker(), "Checking pool {}", lookupStoreName); + List lookupContent = getPoolContent(lookupStoreName, lookupStoreUri); + poolContent.addAll(lookupContent); + processed++; + progress = processed * 50 / count; + setProgress(progress); + } + /* + if (1 == 1) { + + logger.info(getMarker(), "Kikapcsolva"); + return null; + } + */ + + PBQuery query = new PBQuery(); + query.init(primaryEndPoint, primaryUserName, primaryPassword, secondaryEndPoint, secondaryUserName, secondaryPassword, rangeForwardHours); + Collection medias = query.getPossibelMissingMaterialNames(new ArrayList<>(poolContent)); + logger.info(getMarker(), "API returned {} items", medias == null ? 0 : medias.size()); + + if (medias != null && medias.size() > 0) { + count = medias.size(); + for (MMMedia media : medias) { + logger.info(getMarker(), "API response contains {}", media.getName()); + } + + for (MMMedia media : medias) { + if (getJobRuntime().isWaitingCancel()) { + cancel(); + break; + } + processRecord(media, sourceStoreUri, targetStoreName, targetStoreUri, escortStoreUri); + processed++; + progress = 50 + (processed * 50 / count); + setProgress(progress); + } + + } + setProgress(100); + } catch (Exception e) { + logger.error(getSessionMarker(), e.getMessage()); + throw e; + } finally { + if (escortStoreUri != null) + escortStoreUri.cleanUp(); + } + + return null; + } + + private List getPoolContent(String storeName, StoreUri targetStoreUri) throws Exception { + List poolContent = new ArrayList<>(); + + FtpDirectoryLister lister = (FtpDirectoryLister) targetStoreUri.getLister(); + FTPClient client = lister.connect(); + logger.info(getMarker(), "Working dir is {}", client.printWorkingDirectory()); + String[] listNames = client.listNames(); + targetStoreUri.cleanUp(); + + if (listNames != null) { + List remoteFileNames = Arrays.asList(listNames); + logger.info(getMarker(), "Remote file list size :{}", remoteFileNames.size()); + for (String name : remoteFileNames) { + try { + if (name.toLowerCase().endsWith(".mxf")) { + int pos = name.lastIndexOf("."); + if (pos > -1) { + name = name.substring(0, pos); + //logger.info(getMarker(), "Adding {}", name); + poolContent.add(name); + } + } + } catch (Exception e) { + logger.error(e.getMessage()); + } + + } + } + + // List remoteFiles = targetStoreUri.getRemoteFiles(); + // if (remoteFiles != null) { + // logger.info(getMarker(), "Remote file list size :{}", remoteFiles.size()); + // for (RemoteFile rf : remoteFiles) { + // try { + // String name = rf.getName(); + // if (name.toLowerCase().endsWith(".mxf")) { + // int pos = name.lastIndexOf("."); + // if (pos > -1) { + // name = name.substring(0, pos); + // //logger.info(getMarker(), "Adding {}", name); + // poolContent.add(name); + // } + // } + // } catch (Exception e) { + // logger.error(e.getMessage()); + // } + // + // } + // } + logger.info(getMarker(), "Pool {} contains {} items", storeName, poolContent == null ? 0 : poolContent.size()); + return poolContent; + } + + //csak visszaterunk + private String normalizeName(String mediaName) { + String result = mediaName; + if (StringUtils.isNotBlank(appendExtension)) + result += appendExtension; + // if (StringUtils.isNotBlank(mediaName)) { + // result = mediaName.trim().toLowerCase(); + // } + return result; + } + + private void processRecord(MMMedia mmedia, StoreUri sourceStoreUri, String targetStoreName, StoreUri targetStoreUri, StoreUri escortStoreUri) { + //TODO kisbetu/nagybetu problema kezelese + + Media media = null; + //duplikatum miatt + try { + media = getManager().getMedia(mmedia.getName()); + } catch (Exception e) { + logger.error(getMarker(), "Can't identify missing material {}. System message is: {}", mmedia.getName(), e.getMessage()); + return; + } + + String fileName = normalizeName(mmedia.getName()); + + if (media == null) { + logger.warn(getMarker(), "File {} not archived yet", mmedia.getName()); + return; + } + + DownloadableMedia downloadable = DownloadableMedia.create(mmedia.getName(), fileName, media.getModified(), media.getCreated(), media.getLength(), 0L, + sourceStoreUri.getId(), targetStoreUri.getId(), media.getId()); + downloadable.put("priority", 50); + String usage = mmedia.getUsage(); + usage = usage.replace("-", "").replace(":", ""); + String escortFileName = usage + "." + targetStoreName + "." + fileName; + Path outputPath = null; + try { + outputPath = Paths.get(escortStoreUri.toString(true)); + boolean exsists = statusFileExists(outputPath, fileName); + if (exsists) + return; + EscortFiles.createMetadata(outputPath.toString(), escortFileName, downloadable.toPrettyString("")); + logger.info(getMarker(), "Status file created {}", escortFileName); + } catch (Exception e) { + logger.error(getSessionMarker(), "Can't create status file {}", Paths.get(outputPath.toString(), escortFileName)); + } + + } + + private boolean statusFileExists(Path outputPath, String fileName) { + boolean exsists = false; + Path statusPath = Paths.get(outputPath.toString(), EscortFiles.STATUSFOLDER); + try (DirectoryStream p = Files.newDirectoryStream(statusPath, "*" + fileName + EscortFiles.DOT_JSON)) { + if (p.iterator().hasNext()) { + logger.info(getMarker(), "Status file for {} already exists", fileName); + exsists = true; + } + + } catch (Exception e1) { + logger.error(getSessionMarker(), e1.getMessage()); + } + return exsists; + } + +} diff --git a/server/-product/production/AMC/jobs/steps/PeableBeachMissingMaterialCheckerStep.java b/server/-product/production/AMC/jobs/steps/PeableBeachMissingMaterialCheckerStep.java new file mode 100644 index 00000000..e2313678 --- /dev/null +++ b/server/-product/production/AMC/jobs/steps/PeableBeachMissingMaterialCheckerStep.java @@ -0,0 +1,227 @@ +package user.jobengine.server.steps; + +import java.nio.file.DirectoryStream; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collection; +import java.util.LinkedHashSet; +import java.util.List; + +import org.apache.commons.lang.StringUtils; +import org.apache.commons.net.ftp.FTPClient; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + +import com.ibm.nosql.json.api.BasicDBList; + +import user.commons.DownloadableMedia; +import user.commons.StoreUri; +import user.commons.remotestore.FtpDirectoryLister; +import user.commons.remotestore.RemoteStoreProtocol; +import user.jobengine.db.Media; +import user.jobengine.server.steps.shared.EscortFiles; +import user.jobengine.server.steps.shared.PBQuery; +import user.jobengine.server.steps.shared.PBQuery.MMMedia; + +public class PeableBeachMissingMaterialCheckerStep extends JobStep { + private static final Logger logger = LogManager.getLogger(); + private static final String appendExtension = ".mxf"; + + @StepEntry + public Object[] execute(String escortStoreName, BasicDBList lookupStoreNames, String targetStoreName, String targetProtocol, String primaryEndPoint, + String primaryUserName, String primaryPassword, String secondaryEndPoint, String secondaryUserName, String secondaryPassword, int rangeForwardHours) + throws Exception { + StoreUri escortStoreUri = null; + try { + getJobRuntime().setRelated(targetStoreName + " restore"); + + escortStoreUri = getManager().getStoreUri(escortStoreName, RemoteStoreProtocol.LOCAL); + StoreUri sourceStoreUri = getManager().getStoreUri("TSM", RemoteStoreProtocol.TSM); + StoreUri targetStoreUri = getManager().getStoreUri(targetStoreName, Enum.valueOf(RemoteStoreProtocol.class, targetProtocol)); + + LinkedHashSet poolContent = new LinkedHashSet<>(); + + int progress = 0; + int count = lookupStoreNames.size(); + int processed = 0; + + for (Object lookup : lookupStoreNames) { + if (getJobRuntime().isWaitingCancel()) { + cancel(); + return null; + } + String lookupStoreName = String.valueOf(lookup); + + StoreUri lookupStoreUri = getManager().getStoreUri(lookupStoreName, Enum.valueOf(RemoteStoreProtocol.class, targetProtocol)); + logger.info(getMarker(), "Checking pool {}", lookupStoreName); + List lookupContent = getPoolContent(lookupStoreName, lookupStoreUri); + poolContent.addAll(lookupContent); + processed++; + progress = processed * 50 / count; + setProgress(progress); + } + /* + if (1 == 1) { + + logger.info(getMarker(), "Kikapcsolva"); + return null; + } + */ + + PBQuery query = new PBQuery(); + query.init(primaryEndPoint, primaryUserName, primaryPassword, secondaryEndPoint, secondaryUserName, secondaryPassword, rangeForwardHours); + Collection medias = query.getPossibelMissingMaterialNames(new ArrayList<>(poolContent)); + logger.info(getMarker(), "API returned {} items", medias == null ? 0 : medias.size()); + + if (medias != null && medias.size() > 0) { + count = medias.size(); + for (MMMedia media : medias) { + logger.info(getMarker(), "API response contains {}", media.getName()); + } + + for (MMMedia media : medias) { + if (getJobRuntime().isWaitingCancel()) { + cancel(); + break; + } + processRecord(media, sourceStoreUri, targetStoreName, targetStoreUri, escortStoreUri); + processed++; + progress = 50 + (processed * 50 / count); + setProgress(progress); + } + + } + setProgress(100); + } catch (Exception e) { + logger.error(getSessionMarker(), e.getMessage()); + throw e; + } finally { + if (escortStoreUri != null) + escortStoreUri.cleanUp(); + } + + return null; + } + + private List getPoolContent(String storeName, StoreUri targetStoreUri) throws Exception { + List poolContent = new ArrayList<>(); + + FtpDirectoryLister lister = (FtpDirectoryLister) targetStoreUri.getLister(); + FTPClient client = lister.connect(); + logger.info(getMarker(), "Working dir is {}", client.printWorkingDirectory()); + String[] listNames = client.listNames(); + targetStoreUri.cleanUp(); + + if (listNames != null) { + List remoteFileNames = Arrays.asList(listNames); + logger.info(getMarker(), "Remote file list size :{}", remoteFileNames.size()); + for (String name : remoteFileNames) { + try { + if (name.toLowerCase().endsWith(".mxf")) { + int pos = name.lastIndexOf("."); + if (pos > -1) { + name = name.substring(0, pos); + //logger.info(getMarker(), "Adding {}", name); + poolContent.add(name); + } + } + } catch (Exception e) { + logger.error(e.getMessage()); + } + + } + } + + // List remoteFiles = targetStoreUri.getRemoteFiles(); + // if (remoteFiles != null) { + // logger.info(getMarker(), "Remote file list size :{}", remoteFiles.size()); + // for (RemoteFile rf : remoteFiles) { + // try { + // String name = rf.getName(); + // if (name.toLowerCase().endsWith(".mxf")) { + // int pos = name.lastIndexOf("."); + // if (pos > -1) { + // name = name.substring(0, pos); + // //logger.info(getMarker(), "Adding {}", name); + // poolContent.add(name); + // } + // } + // } catch (Exception e) { + // logger.error(e.getMessage()); + // } + // + // } + // } + logger.info(getMarker(), "Pool {} contains {} items", storeName, poolContent == null ? 0 : poolContent.size()); + return poolContent; + } + + //csak visszaterunk + private String normalizeName(String mediaName) { + String result = mediaName; + if (StringUtils.isNotBlank(appendExtension)) + result += appendExtension; + // if (StringUtils.isNotBlank(mediaName)) { + // result = mediaName.trim().toLowerCase(); + // } + return result; + } + + private void processRecord(MMMedia mmedia, StoreUri sourceStoreUri, String targetStoreName, StoreUri targetStoreUri, StoreUri escortStoreUri) { + //TODO kisbetu/nagybetu problema kezelese + + Media media = null; + //duplikatum miatt + try { + media = getManager().getMedia(mmedia.getName()); + } catch (Exception e) { + logger.error(getMarker(), "Can't identify missing material {}. System message is: {}", mmedia.getName(), e.getMessage()); + return; + } + + String fileName = normalizeName(mmedia.getName()); + + if (media == null) { + logger.warn(getMarker(), "File {} not archived yet", mmedia.getName()); + return; + } + + DownloadableMedia downloadable = DownloadableMedia.create(mmedia.getName(), fileName, media.getModified(), media.getCreated(), media.getLength(), 0L, + sourceStoreUri.getId(), targetStoreUri.getId(), media.getId()); + downloadable.put("priority", 50); + String usage = mmedia.getUsage(); + usage = usage.replace("-", "").replace(":", ""); + String escortFileName = usage + "." + targetStoreName + "." + fileName; + Path outputPath = null; + try { + outputPath = Paths.get(escortStoreUri.toString(true)); + boolean exsists = statusFileExists(outputPath, fileName); + if (exsists) + return; + EscortFiles.createMetadata(outputPath.toString(), escortFileName, downloadable.toPrettyString("")); + logger.info(getMarker(), "Status file created {}", escortFileName); + } catch (Exception e) { + logger.error(getSessionMarker(), "Can't create status file {}", Paths.get(outputPath.toString(), escortFileName)); + } + + } + + private boolean statusFileExists(Path outputPath, String fileName) { + boolean exsists = false; + Path statusPath = Paths.get(outputPath.toString(), EscortFiles.STATUSFOLDER); + try (DirectoryStream p = Files.newDirectoryStream(statusPath, "*" + fileName + EscortFiles.DOT_JSON)) { + if (p.iterator().hasNext()) { + logger.info(getMarker(), "Status file for {} already exists", fileName); + exsists = true; + } + + } catch (Exception e1) { + logger.error(getSessionMarker(), e1.getMessage()); + } + return exsists; + } + +} diff --git a/server/-product/production/AMC/jobs/steps/RegisterUserRestoreStep.java b/server/-product/production/AMC/jobs/steps/RegisterUserRestoreStep.java new file mode 100644 index 00000000..c743653e --- /dev/null +++ b/server/-product/production/AMC/jobs/steps/RegisterUserRestoreStep.java @@ -0,0 +1,72 @@ +package user.jobengine.server.steps; + +import java.nio.file.Paths; +import java.util.List; + +import org.apache.commons.lang.StringUtils; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + +import user.commons.DownloadableMedia; +import user.commons.StoreUri; +import user.commons.remotestore.RemoteStoreProtocol; +import user.jobengine.db.ArchivedMedia; +import user.jobengine.db.Media; +import user.jobengine.server.steps.shared.EscortFiles; + +public class RegisterUserRestoreStep extends JobStep { + private static final Logger logger = LogManager.getLogger(); + private static final String appendExtension = System.getProperty("missingmaterial.appendextension", ".mxf"); + + @StepEntry + public Object[] execute(String escortStoreName, String targetStoreName, List basket, String recipient) throws Exception { + StoreUri escortStoreUri = null; + try { + escortStoreUri = getManager().getStoreUri(escortStoreName, RemoteStoreProtocol.LOCAL); + StoreUri targetStoreUri = getManager().getStore(targetStoreName).getTargetStoreUri(); + StoreUri sourceStoreUri = getManager().getStoreUri("TSM", RemoteStoreProtocol.TSM); + int processed = 0; + for (ArchivedMedia media : basket) { + processRecord(media, targetStoreName, sourceStoreUri, targetStoreUri, escortStoreUri, recipient); + processed++; + int progress = processed * 100 / basket.size(); + setProgress(progress); + } + } catch (Exception e) { + logger.error(getSessionMarker(), e.getMessage()); + throw e; + } finally { + if (escortStoreUri != null) + escortStoreUri.cleanUp(); + } + + return null; + } + + private void processRecord(ArchivedMedia archivedMedia, String targetStoreName, StoreUri sourceStoreUri, StoreUri targetStoreUri, StoreUri escortStoreUri, + String recipient) { + Media media = archivedMedia.getMedia(); + + String fileName = media.getTitle(); + if (StringUtils.isNotBlank(appendExtension)) + fileName += appendExtension; + + DownloadableMedia downloadable = DownloadableMedia.create(media.getTitle(), fileName, media.getModified(), media.getCreated(), media.getLength(), 0L, + sourceStoreUri.getId(), targetStoreUri.getId(), media.getId()); + downloadable.put("recipient", recipient); + downloadable.put("skipValidation", true); + String escortFileName = targetStoreName + "." + downloadable.getString("fileName"); + String outputPath = null; + try { + outputPath = Paths.get(escortStoreUri.toString(true)).toString(); + if (EscortFiles.createMetadataIfNotExists(outputPath, escortFileName, downloadable.toPrettyString(""))) + logger.info(getSessionMarker(), "Status file created {}", escortFileName); + else + logger.info(getSessionMarker(), "Status file already exists {}", escortFileName); + } catch (Exception e) { + logger.error("Can't create escort file {}", Paths.get(outputPath.toString(), escortFileName)); + } + + } + +} diff --git a/server/-product/production/AMC/jobs/steps/RegisterVODRestoreStep.java b/server/-product/production/AMC/jobs/steps/RegisterVODRestoreStep.java new file mode 100644 index 00000000..174056d4 --- /dev/null +++ b/server/-product/production/AMC/jobs/steps/RegisterVODRestoreStep.java @@ -0,0 +1,80 @@ +package user.jobengine.server.steps; + +import java.nio.file.Paths; + +import org.apache.commons.lang.StringUtils; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + +import com.ibm.nosql.json.api.BasicDBList; +import com.ibm.nosql.json.api.BasicDBObject; + +import user.commons.DownloadableMedia; +import user.commons.StoreUri; +import user.commons.remotestore.RemoteStoreProtocol; +import user.jobengine.db.Media; +import user.jobengine.server.steps.shared.EscortFiles; + +public class RegisterVODRestoreStep extends JobStep { + private static final Logger logger = LogManager.getLogger(); + private static final String appendExtension = System.getProperty("missingmaterial.appendextension", ".mxf"); + + @StepEntry + public Object[] execute(String targetStoreName, BasicDBList basket) throws Exception { + try { + StoreUri sourceStoreUri = getManager().getStoreUri("TSM", RemoteStoreProtocol.TSM); + + StoreUri targetStoreUri = getManager().getStore(targetStoreName).getTargetStoreUri(); + StoreUri escortStoreUri = getManager().getStoreUri("MEDIACUBE_UPLOADS", RemoteStoreProtocol.LOCAL); + int processed = 0; + for (int i = 0; i < basket.size(); i++) { + BasicDBObject item = (BasicDBObject) basket.get(i); + //logger.info(item.toPrettyString()); + processRecord(targetStoreName, sourceStoreUri, targetStoreUri, escortStoreUri, item); + processed++; + int progress = processed * 100 / basket.size(); + setProgress(progress); + } + } catch (Exception e) { + logger.error(getSessionMarker(), e.getMessage()); + throw e; + } finally { + } + + return null; + } + + private void processRecord(String targetStoreName, StoreUri sourceStoreUri, StoreUri targetStoreUri, StoreUri escortStoreUri, BasicDBObject item) { + String mediaTitle = item.getString("title"); + String relativeTargetPath = item.getString("path"); + + Media media = getManager().getMedia(mediaTitle); + + if (media == null) { + logger.error("Media {} is not available", mediaTitle); + return; + } + + String fileName = media.getTitle(); + if (StringUtils.isNotBlank(appendExtension)) + fileName += appendExtension; + + DownloadableMedia downloadable = DownloadableMedia.create(media.getTitle(), fileName, media.getModified(), media.getCreated(), media.getLength(), 0L, + sourceStoreUri.getId(), targetStoreUri.getId(), media.getId()); + //downloadable.put("skipValidation", true); + downloadable.put("relativeTargetPath", relativeTargetPath); + String escortFileName = targetStoreName + "." + downloadable.getString("fileName"); + String outputPath = null; + try { + outputPath = Paths.get(escortStoreUri.toString(true)).toString(); + if (EscortFiles.createMetadataIfNotExists(outputPath, escortFileName, downloadable.toPrettyString(""))) + logger.info(getSessionMarker(), "Status file created {}", escortFileName); + else + logger.info(getSessionMarker(), "Status file already exists {}", escortFileName); + } catch (Exception e) { + logger.error("Can't create escort file {}", Paths.get(outputPath.toString(), escortFileName)); + } + + } + +} diff --git a/server/-product/production/AMC/jobs/steps/RundownFilesCollectorStep.java b/server/-product/production/AMC/jobs/steps/RundownFilesCollectorStep.java new file mode 100644 index 00000000..57750aa7 --- /dev/null +++ b/server/-product/production/AMC/jobs/steps/RundownFilesCollectorStep.java @@ -0,0 +1,42 @@ +package user.jobengine.server.steps; + +import java.nio.file.Path; +import java.text.SimpleDateFormat; +import java.util.Calendar; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + +import com.ibm.nosql.json.api.BasicDBObject; + +/* DO NOT REMOVE! +import user.jobengine.server.steps.FileSearchFilterOptions; +*/ +public class RundownFilesCollectorStep extends PathItemsCollectorStep { + private static final Logger logger = LogManager.getLogger(); + + @Override + protected FileSearchFilterOptions createFileFilter(BasicDBObject filter) { + SimpleDateFormat df = new SimpleDateFormat("yyyy\\MM"); + Calendar thisMonthCal = Calendar.getInstance(); + Calendar nextMonthCal = Calendar.getInstance(); + nextMonthCal.add(Calendar.MONTH, 1); + + FileSearchFilterOptions result = new FileSearchFilterOptions(filter) { + @Override + public boolean acceptFile(Path file) { + if (file.toString().contains(df.format(thisMonthCal.getTime())) || file.toString().contains(df.format(nextMonthCal.getTime()))) + return super.acceptFile(file); + + return false; + } + }; + return result; + } + + @Override + @StepEntry + public Object[] execute(String sourceFolder, BasicDBObject filter) throws Exception { + return super.execute(sourceFolder, filter); + } +} diff --git a/server/-product/production/AMC/jobs/steps/ServerStatusReportStep.java b/server/-product/production/AMC/jobs/steps/ServerStatusReportStep.java new file mode 100644 index 00000000..787a07e8 --- /dev/null +++ b/server/-product/production/AMC/jobs/steps/ServerStatusReportStep.java @@ -0,0 +1,39 @@ +package user.jobengine.server.steps; + +import java.time.Instant; +import java.util.Map; +import java.util.Set; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + +import user.commons.JobStatus; +import user.commons.MediaCubeMarker; +import user.jobengine.server.IJobEngine; +import user.jobengine.server.IJobRuntime; + +public class ServerStatusReportStep extends JobStep { + private static final Logger logger = LogManager.getLogger(); + + @StepEntry + public Object[] execute() throws Exception { + IJobEngine engine = getEngine(); + int jobCount = 0; + StringBuilder sb = new StringBuilder(); + Map jobs = engine.getJobs(); + if (jobs != null) { + Set keys = jobs.keySet(); + jobCount = keys.size(); + for (Long key : keys) { + IJobRuntime runtime = jobs.get(key); + if (JobStatus.SUSPENDED.equals(runtime.getStatus())) + sb.append(runtime.getRelated() + " felfüggesztve: " + runtime.getDescription() + "
"); + } + } + MediaCubeMarker marker = new MediaCubeMarker(); + + marker.setSubject(String.format("%s [%d db] AMC MediaCube feldolgozás", Instant.now(), jobCount)); + logger.info(marker, sb.toString()); + return null; + } +} diff --git a/server/-product/production/AMC/jobs/steps/SubtitleFilesCollectorStep.java b/server/-product/production/AMC/jobs/steps/SubtitleFilesCollectorStep.java new file mode 100644 index 00000000..f27a5a64 --- /dev/null +++ b/server/-product/production/AMC/jobs/steps/SubtitleFilesCollectorStep.java @@ -0,0 +1,60 @@ +package user.jobengine.server.steps; + +import java.io.IOException; +import java.nio.file.FileVisitResult; +import java.nio.file.FileVisitor; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.nio.file.SimpleFileVisitor; +import java.nio.file.attribute.BasicFileAttributes; +import java.util.ArrayList; +import java.util.List; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + +public class SubtitleFilesCollectorStep extends JobStep { + private static final Logger logger = LogManager.getLogger(); + + @StepEntry + public Object[] execute(String sourceFolder, String fileNamePattern) throws Exception { + Path sourcePath = Paths.get(sourceFolder); + List files = new ArrayList<>(); + + Pattern pattern = Pattern.compile(fileNamePattern, Pattern.CASE_INSENSITIVE); + + FileVisitor visitor = new SimpleFileVisitor() { + @Override + public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) throws IOException { + if (!file.getFileName().toString().contains("_DONE")) { + Matcher matcher = pattern.matcher(file.getFileName().toString()); + if (matcher.find()) + files.add(file.toString()); + } + return FileVisitResult.CONTINUE; + } + + @Override + public FileVisitResult visitFileFailed(Path file, IOException exc) throws IOException { + return FileVisitResult.CONTINUE; + } + + }; + + try { + Files.walkFileTree(sourcePath, visitor); + } catch (Exception e) { + logger.error(getMarker(), "Error processing '{}'. System message: {}", sourcePath, e.getMessage()); + logger.catching(e); + throw e; + } finally { + } + + logger.info(getMarker(), "Found {} files", files.size()); + return new Object[] { files }; + } + +} diff --git a/server/-product/production/AMC/jobs/steps/TSMTransferFromStep.java b/server/-product/production/AMC/jobs/steps/TSMTransferFromStep.java new file mode 100644 index 00000000..ab1b6bbf --- /dev/null +++ b/server/-product/production/AMC/jobs/steps/TSMTransferFromStep.java @@ -0,0 +1,27 @@ +package user.jobengine.server.steps; + +import java.nio.file.Path; +import java.nio.file.Paths; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + +import user.commons.StoreUri; + +public class TSMTransferFromStep extends TransferStep { + private static final Logger logger = LogManager.getLogger(); + + @Override + @StepEntry + public Object[] execute(StoreUri sourceStoreUri, String sourceFileName, StoreUri targetStoreUri, String targetFileName) throws Exception { + Object[] result = null; + Path targetFile = Paths.get(targetStoreUri.toString(true), targetFileName); + if (targetFile.toFile().exists()) { + logger.info(getMarker(), "Using previously retrived file {}", targetFile); + } else { + result = super.execute(sourceStoreUri, sourceFileName, targetStoreUri, targetFileName); + } + return result; + } + +} diff --git a/server/-product/production/AMC/jobs/steps/TSMTransferToStep.java b/server/-product/production/AMC/jobs/steps/TSMTransferToStep.java new file mode 100644 index 00000000..efff055e --- /dev/null +++ b/server/-product/production/AMC/jobs/steps/TSMTransferToStep.java @@ -0,0 +1,11 @@ +package user.jobengine.server.steps; + +import user.commons.StoreUri; + +public class TSMTransferToStep extends TransferStep { + @Override + @StepEntry + public Object[] execute(StoreUri sourceStoreUri, String sourceFileName, StoreUri targetStoreUri, String targetFileName) throws Exception { + return super.execute(sourceStoreUri, sourceFileName, targetStoreUri, targetFileName); + } +} diff --git a/server/-product/production/AMC/jobs/steps/TestForkCancelableStep.java b/server/-product/production/AMC/jobs/steps/TestForkCancelableStep.java new file mode 100644 index 00000000..7651a137 --- /dev/null +++ b/server/-product/production/AMC/jobs/steps/TestForkCancelableStep.java @@ -0,0 +1,33 @@ +package user.jobengine.server.steps; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + +import user.commons.IJob; +import user.commons.JobStatus; +import user.commons.ListUtils; +import user.jobengine.server.IJobEngine; +import user.jobengine.server.IJobRuntime; + +public class TestForkCancelableStep extends JobStep { + private static final String CHILD_TITLE = "Párhuzamosított alfolyamat"; + private static final String CHILD_TEMPLATE = "cancelable.xml"; + private static final Logger logger = LogManager.getLogger(); + int count = 20; + + @StepEntry + public Object[] execute(IJobEngine jobEngine, IJobRuntime jobRuntime) throws Exception { + for (int i = 0; i < count; i++) { + IJobRuntime runtime = getEngine().submit(null, e -> { + if (e.getStatus().equals(JobStatus.CANCELED) || e.getStatus().equals(JobStatus.SUSPENDED)) + logger.info("Cleanup occured because status is {} {}", e.getStatus(), ((IJob) e.getSource()).getId()); + }, CHILD_TEMPLATE, "JOB " + i, 0, IJobEngine.DEFAULT_OWNER, ListUtils.asMap("param", i)); + + runtime.setRelated("TEST" + runtime.getId()); + } + + // jobRuntime.forkWaitComplete(); + logger.info("Done"); + return null; + } +} diff --git a/server/-product/production/AMC/jobs/steps/TransferStep.java b/server/-product/production/AMC/jobs/steps/TransferStep.java new file mode 100644 index 00000000..67cc025c --- /dev/null +++ b/server/-product/production/AMC/jobs/steps/TransferStep.java @@ -0,0 +1,100 @@ +package user.jobengine.server.steps; + +import java.nio.file.Path; +import java.nio.file.Paths; + +import org.apache.commons.net.ftp.FTPClient; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + +import user.commons.StoreUri; +import user.commons.remotestore.FtpDirectoryLister; +import user.commons.remotestore.RemoteStoreProtocol; +import user.jobengine.db.Store; + +public class TransferStep extends JobStep { + private static final String DOT_PART = ".part"; + private static final Logger logger = LogManager.getLogger(); + + @StepEntry + public Object[] execute(StoreUri sourceStoreUri, String sourceFileName, StoreUri targetStoreUri, String targetFileName) throws Exception { + try { + getJobRuntime().setCancelable(false); + + Store sourceStore = getManager().getStore(sourceStoreUri.getStoreId()); + Store targetStore = getManager().getStore(targetStoreUri.getStoreId()); + + String description = String.format("%s -> %s : %s", sourceStore.getName(), targetStore.getName(), sourceFileName); + getJobRuntime().setDescription(description); + + sourceStoreUri.addProgressListener(e -> setProgress(e.getProgress())); + + StoreUri currentTargetStoreUri = getTargetStoreUri(targetStoreUri); + tryCopy(sourceStoreUri, sourceFileName, currentTargetStoreUri, targetFileName); + } catch (Exception e) { + logger.error(getMarker(), "Error in transfer of {} when copying from {} to {}.", sourceFileName, sourceStoreUri, targetStoreUri); + throw e; + } finally { + if (sourceStoreUri != null) + sourceStoreUri.cleanUp(); + if (targetStoreUri != null) + targetStoreUri.cleanUp(); + } + return null; + } + + protected StoreUri getTargetStoreUri(StoreUri targetStoreUri) { + return targetStoreUri; + } + + protected String getTmpExtension() { + return DOT_PART; + } + + private void tryCopy(StoreUri sourceStoreUri, String sourceFileName, StoreUri targetStoreUri, String targetFileName) throws Exception { + String currentTargetFileName = targetFileName; + + boolean renameAfterCopy = false; + boolean renameAfterFTP = false; + if (getTmpExtension() != null) { + if (RemoteStoreProtocol.LOCAL.equals(targetStoreUri.getProtocol())) { + currentTargetFileName += getTmpExtension(); + renameAfterCopy = true; + } + Store targetStore = getManager().getStore(targetStoreUri.getStoreId()); + + if (RemoteStoreProtocol.FTP.equals(targetStoreUri.getProtocol()) && !"NEXIO1".equals(targetStore.getName()) + && !"NEXIO2".equals(targetStore.getName())) { + currentTargetFileName += getTmpExtension(); + renameAfterFTP = true; + } + } + + sourceStoreUri.transferFrom(targetStoreUri, sourceFileName, currentTargetFileName); + + logger.info(getMarker(), "Transfer of {} completed from {} to {}", sourceFileName, sourceStoreUri, currentTargetFileName); + + if (renameAfterCopy) { + Path tmpTargetFile = Paths.get(targetStoreUri.toString(true), currentTargetFileName); + Path targetFile = Paths.get(targetStoreUri.toString(true), targetFileName); + try { + logger.info(getMarker(), "Renaming LOCAL file from {} to {} on {}", currentTargetFileName, targetFileName, sourceStoreUri); + tmpTargetFile.toFile().renameTo(targetFile.toFile()); + } catch (Exception e) { + logger.error(getMarker(), e.getMessage()); + } + } + if (renameAfterFTP) { + try { + FtpDirectoryLister lister = (FtpDirectoryLister) targetStoreUri.getLister(); + FTPClient client = lister.connect(); + logger.info(getMarker(), "Renaming FTP file from {} to {} on {}", currentTargetFileName, targetFileName, sourceStoreUri); + client.rename(currentTargetFileName, targetFileName); + } catch (Exception e) { + logger.error(getMarker(), e.getMessage()); + } finally { + targetStoreUri.cleanUp(); + } + } + } +} diff --git a/server/-product/production/AMC/jobs/steps/TransferToFTPStep.java b/server/-product/production/AMC/jobs/steps/TransferToFTPStep.java new file mode 100644 index 00000000..b42d5d53 --- /dev/null +++ b/server/-product/production/AMC/jobs/steps/TransferToFTPStep.java @@ -0,0 +1,11 @@ +package user.jobengine.server.steps; + +import user.commons.StoreUri; + +public class TransferToFTPStep extends TransferStep { + @Override + @StepEntry + public Object[] execute(StoreUri sourceStoreUri, String sourceFileName, StoreUri targetStoreUri, String targetFileName) throws Exception { + return super.execute(sourceStoreUri, sourceFileName, targetStoreUri, targetFileName); + } +} diff --git a/server/-product/production/AMC/jobs/steps/VODTransferToStep.java b/server/-product/production/AMC/jobs/steps/VODTransferToStep.java new file mode 100644 index 00000000..d17eaa1e --- /dev/null +++ b/server/-product/production/AMC/jobs/steps/VODTransferToStep.java @@ -0,0 +1,22 @@ +package user.jobengine.server.steps; + +import user.commons.StoreUri; + +public class VODTransferToStep extends TransferStep { + private String relativeTargetPath; + + @StepEntry + public Object[] execute(StoreUri sourceStoreUri, String sourceFileName, StoreUri targetStoreUri, String relativeTargetPath, String targetFileName) + throws Exception { + this.relativeTargetPath = relativeTargetPath; + return super.execute(sourceStoreUri, sourceFileName, targetStoreUri, targetFileName); + } + + @Override + protected StoreUri getTargetStoreUri(StoreUri targetStoreUri) { + StoreUri result = getManager().getStoreUri(targetStoreUri.getId()); + result.setRootPath(relativeTargetPath); + return result; + } + +} diff --git a/server/-product/production/AMC/jobs/steps/shared/EscortFiles.java b/server/-product/production/AMC/jobs/steps/shared/EscortFiles.java new file mode 100644 index 00000000..da308d3c --- /dev/null +++ b/server/-product/production/AMC/jobs/steps/shared/EscortFiles.java @@ -0,0 +1,339 @@ +package user.jobengine.server.steps.shared; + +import java.io.ByteArrayInputStream; +import java.io.File; +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; +import java.io.StringWriter; +import java.io.UnsupportedEncodingException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.nio.file.attribute.FileAttribute; +import java.nio.file.attribute.PosixFilePermission; +import java.nio.file.attribute.PosixFilePermissions; +import java.text.SimpleDateFormat; +import java.util.Calendar; +import java.util.Date; +import java.util.Set; + +import javax.xml.parsers.DocumentBuilder; +import javax.xml.parsers.DocumentBuilderFactory; +import javax.xml.transform.OutputKeys; +import javax.xml.transform.Transformer; +import javax.xml.transform.TransformerConfigurationException; +import javax.xml.transform.TransformerException; +import javax.xml.transform.TransformerFactory; +import javax.xml.transform.TransformerFactoryConfigurationError; +import javax.xml.transform.dom.DOMSource; +import javax.xml.transform.stream.StreamResult; + +import org.apache.commons.lang.StringUtils; +import org.apache.commons.net.ftp.FTPClient; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.apache.logging.log4j.Marker; +import org.apache.logging.log4j.message.Message; +import org.w3c.dom.DOMImplementation; +import org.w3c.dom.Document; +import org.w3c.dom.Element; + +import com.ibm.nosql.json.api.BasicDBObject; +import com.ibm.nosql.json.util.JSON; + +import user.commons.CalendarUtils; +import user.commons.MediaCubeMarker; +import user.commons.StoreUri; +import user.commons.remotestore.FtpDirectoryLister; + +public class EscortFiles { + private static final String RECORDTIMESTAMP = "RecordTimeStamp"; + private static final String MODIFIEDTIMESTAMP = "ModifiedTimeStamp"; + public static final String DOT_CATCHED = ".catched"; + public static final String DOT_JSON = ".json"; + private static final Logger logger = LogManager.getLogger(); + private static final String EXTENDEDAGENCY = "ExtendedAgency"; + private static final String EXTENDEDDESCRIPTION = "ExtendedDescription"; + private static final String KILLDATE = "KillDate"; + private static final String FORMAT_KILLDATE = "MM-dd-yyyy"; + private static final String EXTENDEDID = "extendedId"; + private static final String ID = "ID"; + private static final String KILLDATE_FILENAME = "%s.%s.killdate"; + private static final String FORMAT_KILLDATENAME = "yyyyMMdd"; + public static final String STATUSFOLDER = ".STATUS"; + public static final String CONFLICTFOLDER = ".CONFLICT"; + + public static String composeKillDate(int days) { + Calendar killDate = Calendar.getInstance(); + killDate.add(Calendar.DAY_OF_YEAR, days); + SimpleDateFormat dateFormat = new SimpleDateFormat(FORMAT_KILLDATENAME); + return dateFormat.format(killDate.getTime()); + } + /* + * + * 02-02-2018 TEST + * TEST AGENT AGENT + * + */ + + private static String composeKillDateFileName(String fileName, int days) { + return String.format(KILLDATE_FILENAME, fileName, composeKillDate(days)); + } + + public static void createCatchedFile(Path escortFile) throws IOException { + Path catchedFilePath = Paths.get(escortFile.toString() + DOT_CATCHED); + Files.createFile(catchedFilePath); + } + + public static void createFellow(String escortFile, String extension) throws IOException { + Files.copy(Paths.get(escortFile), Paths.get(escortFile + "." + extension)); + } + + /*** + * A media eleresi utjan alapjan a .STATUS almappaban letrehozza a .catch fajlt. + * + * @param mediaFile + * @throws IOException + */ + public static void createMediaCatch(Path mediaFile) throws IOException { + Path catchedFile = createMediaCathFilePath(mediaFile); + ensureUNCFolder(catchedFile.getParent()); + Files.createFile(catchedFile); + } + + private static Path createMediaCathFilePath(Path mediaFile) { + return Paths.get(mediaFile.getParent().toString(), STATUSFOLDER, mediaFile.getFileName() + DOT_CATCHED); + } + + public static void createMetadata(String filePath, String fileName, String metadata) throws IOException { + ensureUNCFolder(filePath, STATUSFOLDER); + String metadataFileName = fileName + DOT_JSON; + Path metadataPath = Paths.get(filePath, STATUSFOLDER, metadataFileName); + Files.write(metadataPath, metadata.getBytes()); + } + + public static boolean createMetadataIfNotExists(String filePath, String fileName, String metadata) throws IOException { + boolean result = false; + if (!EscortFiles.isMetadataExists(filePath, fileName)) { + EscortFiles.createMetadata(filePath, fileName, metadata); + result = true; + } + return result; + } + + public static void createMorpheusXML(String filePath, String fileName, String content) throws IOException { + ensureUNCFolder(filePath, STATUSFOLDER); + Path xmlPath = Paths.get(filePath, fileName); + if (Files.exists(xmlPath)) + throw new IOException(String.format("Az '%s' állomány már létezik.", xmlPath)); + Files.write(xmlPath, content.getBytes()); + } + + public static byte[] createNEXIODatesMeta(String fileName, Date recorded, Date modified) throws Exception { + DocumentBuilderFactory dbf = DocumentBuilderFactory.newInstance(); + DocumentBuilder db = dbf.newDocumentBuilder(); + DOMImplementation impl = db.getDOMImplementation(); + Document xmlDocument = impl.createDocument(null, null, null); + + Element root = xmlDocument.createElement(ID); + root.setAttribute(EXTENDEDID, fileName); + // 07-13-2020 (19:36:52) + // 05-18-2013 (18:52:24) + SimpleDateFormat df = new SimpleDateFormat("MM-dd-yyyy (HH:mm:ss)"); + root.appendChild(xmlDocument.createElement(MODIFIEDTIMESTAMP)).appendChild(xmlDocument.createTextNode(df.format(modified))); + root.appendChild(xmlDocument.createElement(RECORDTIMESTAMP)).appendChild(xmlDocument.createTextNode(df.format(recorded))); + xmlDocument.appendChild(root); + + return xmDocumentToString(xmlDocument); + } + + public static byte[] createNEXIOKillDateFile(String fileName, Date killDate, String description, String agency) throws Exception { + DocumentBuilderFactory dbf = DocumentBuilderFactory.newInstance(); + DocumentBuilder db = dbf.newDocumentBuilder(); + DOMImplementation impl = db.getDOMImplementation(); + Document xmlDocument = impl.createDocument(null, null, null); + + Element root = xmlDocument.createElement(ID); + root.setAttribute(EXTENDEDID, fileName); + if (killDate != null) { + String sKillDate = CalendarUtils.toString(CalendarUtils.createCalendar(killDate), FORMAT_KILLDATE); + root.appendChild(xmlDocument.createElement(KILLDATE)).appendChild(xmlDocument.createTextNode(sKillDate)); + } + + if (StringUtils.isNotBlank(description)) + root.appendChild(xmlDocument.createElement(EXTENDEDDESCRIPTION)).appendChild(xmlDocument.createTextNode(description)); + if (StringUtils.isNotBlank(agency)) + root.appendChild(xmlDocument.createElement(EXTENDEDAGENCY)).appendChild(xmlDocument.createTextNode(agency)); + xmlDocument.appendChild(root); + + return xmDocumentToString(xmlDocument); + } + + public static Document createNEXIOMeta(byte[] content) throws Exception { + DocumentBuilderFactory dbf = DocumentBuilderFactory.newInstance(); + DocumentBuilder db = dbf.newDocumentBuilder(); + DOMImplementation impl = db.getDOMImplementation(); + Document xmlDocument = null; + + try (InputStream is = new ByteArrayInputStream(content)) { + xmlDocument = db.parse(is); + } catch (Exception e) { + logger.catching(e); + } + + return xmlDocument; + } + + public static void createUNCKillDate(String filePath, String fileName, int days, Marker marker) throws IOException { + ensureUNCFolder(filePath, STATUSFOLDER); + String killDateFileName = composeKillDateFileName(fileName, days); + Path killDatePath = Paths.get(filePath, STATUSFOLDER, killDateFileName); + if (Files.exists(killDatePath)) + logger.warn(marker, "Az '{}' állomány már létezik.", killDatePath); + else + Files.createFile(killDatePath); + } + + @SuppressWarnings("unchecked") + public static T decode(Path escortFile) { + T result = null; + try { + byte[] bytes = Files.readAllBytes(escortFile); + String content = new String(bytes); + result = (T) JSON.parse(content); + } catch (Exception e) { + logger.error("Decode error. System message is: ", e.getMessage()); + } + return result; + } + + public static void ensureUNCFolder(Path filePath) throws IOException { + File folder = filePath.toFile(); + if (!folder.exists() || !folder.isDirectory()) { + try { + Set perms = PosixFilePermissions.fromString("rwxrwxrwx"); + FileAttribute> attr = PosixFilePermissions.asFileAttribute(perms); + Files.createDirectories(filePath, attr); + } catch (Exception e) { + //logger.catching(e); + try { + Files.createDirectories(filePath); + } catch (Exception e1) { + logger.catching(e1); + throw e1; + } + } + } + } + + public static void ensureUNCFolder(String filePath, String folderName) throws IOException { + Path statusPath = Paths.get(filePath, folderName); + ensureUNCFolder(statusPath); + } + + public static boolean isCatchedFileExists(Path escortFile) { + Path catchedFilePath = Paths.get(escortFile.toString() + DOT_CATCHED); + return catchedFilePath.toFile().exists(); + } + + /*** + * A media eleresi utjan alapjan a .STATUS almappaban vizsgalja .catch fajl letezeset. + * + * @param mediaFile + * @return + */ + public static boolean isMediaCatched(Path mediaFile) { + Path catchedFile = createMediaCathFilePath(mediaFile); + return catchedFile.toFile().exists(); + } + + public static boolean isMetadataExists(String filePath, String fileName) throws IOException { + boolean result = false; + String metadataFileName = fileName + DOT_JSON; + Path metadataPath = Paths.get(filePath, STATUSFOLDER, metadataFileName); + result = metadataPath.toFile().exists(); + return result; + } + + public static void notifyRecipient(Path escortFile, Logger logger, Message msg) { + if (escortFile.toFile().exists()) { + try { + BasicDBObject downloadable = EscortFiles.decode(escortFile); + String recipientKey = "recipient"; + if (downloadable.containsKey(recipientKey)) { + String recipient = downloadable.getString(recipientKey); + logger.info(new MediaCubeMarker(recipient, "MediaCube rendszerüzenet"), msg); + } + } catch (Exception e) { + logger.catching(e); + } + + } + } + + public static void remove(Path file) { + try { + file.toFile().delete(); + } catch (Exception e) { + logger.error("Unable to delete {}", file.toAbsolutePath().toString()); + } + } + + public static void removeCatchedFile(Path escortFile) { + remove(Paths.get(escortFile.toString() + DOT_CATCHED)); + } + + /*** + * A media eleresi utjan alapjan a .STATUS almappabol torli a .catch fajlt. + * + * @param mediaFile + * @throws IOException + */ + public static void removeMediaCatch(Path mediaFile) { + Path catchedFile = createMediaCathFilePath(mediaFile); + remove(catchedFile); + } + + public static void setNEXIOKillDate(int killDateDays, String targetFileName, String nexioAgency, StoreUri targetUri) throws Exception { + OutputStream outStream = null; + try { + FTPClient targetFTP = ((FtpDirectoryLister) targetUri.getLister()).connect(); + Calendar killDate = CalendarUtils.createCalendar(new Date()); + killDate.add(Calendar.DAY_OF_YEAR, killDateDays); + if (targetFileName.toLowerCase().contains(".mxf")) + targetFileName = targetFileName.substring(0, targetFileName.lastIndexOf('.')); + byte[] killDateFile = EscortFiles.createNEXIOKillDateFile(targetFileName, killDate.getTime(), null, nexioAgency); + String xml = targetFileName + ".xml"; + outStream = targetFTP.storeFileStream(xml); + if (outStream == null) { + throw new NullPointerException("Can not open: " + targetFileName.substring(0, targetFileName.lastIndexOf('.')) + ".xml" + " Reply:" + + targetFTP.getReplyString()); + } + outStream.write(killDateFile); + outStream.flush(); + } catch (Exception e) { + throw e; + } finally { + if (outStream != null) + outStream.close(); + targetUri.cleanUp(); + } + } + + private static byte[] xmDocumentToString(Document xmlDocument) + throws TransformerFactoryConfigurationError, TransformerConfigurationException, TransformerException, IOException, UnsupportedEncodingException { + DOMSource domSource = new DOMSource(xmlDocument); + TransformerFactory tf = TransformerFactory.newInstance(); + Transformer transformer = tf.newTransformer(); + transformer.setOutputProperty(OutputKeys.METHOD, "xml"); + transformer.setOutputProperty(OutputKeys.ENCODING, "UTF-16"); + transformer.setOutputProperty(OutputKeys.INDENT, "yes"); + StringWriter sw = new StringWriter(); + StreamResult sr = new StreamResult(sw); + transformer.transform(domSource, sr); + String result = sw.toString(); + sw.close(); + return result.getBytes("UTF-16"); + } +} diff --git a/server/-product/production/AMC/jobs/steps/shared/PBQuery.java b/server/-product/production/AMC/jobs/steps/shared/PBQuery.java new file mode 100644 index 00000000..2b50e30d --- /dev/null +++ b/server/-product/production/AMC/jobs/steps/shared/PBQuery.java @@ -0,0 +1,413 @@ +package user.jobengine.server.steps.shared; + +import java.io.DataInputStream; +import java.io.FileNotFoundException; +import java.io.IOException; +import java.io.StringReader; +import java.net.URL; +import java.nio.file.CopyOption; +import java.nio.file.DirectoryStream; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.nio.file.StandardCopyOption; +import java.text.SimpleDateFormat; +import java.util.Arrays; +import java.util.Calendar; +import java.util.Collection; +import java.util.Date; +import java.util.HashMap; +import java.util.HashSet; +import java.util.Iterator; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.TreeMap; + +import javax.xml.parsers.DocumentBuilder; +import javax.xml.parsers.DocumentBuilderFactory; + +import org.apache.commons.lang.StringUtils; +import org.apache.http.HttpEntity; +import org.apache.http.HttpResponse; +import org.apache.http.auth.AuthScope; +import org.apache.http.auth.UsernamePasswordCredentials; +import org.apache.http.client.methods.HttpPost; +import org.apache.http.entity.StringEntity; +import org.apache.http.impl.client.DefaultHttpClient; +import org.apache.http.params.BasicHttpParams; +import org.apache.http.params.HttpConnectionParams; +import org.apache.http.params.HttpParams; +import org.apache.http.protocol.HTTP; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.apache.logging.log4j.Marker; +import org.apache.logging.log4j.MarkerManager; +import org.w3c.dom.Document; +import org.w3c.dom.NamedNodeMap; +import org.w3c.dom.Node; +import org.w3c.dom.NodeList; +import org.xml.sax.InputSource; + +import user.commons.configuration.SystemConfiguration; + +public class PBQuery { + public class MMMedia { + private final String name; + private final String usage; + + MMMedia(String name, String usage) { + this.name = name; + this.usage = usage; + + } + + public String getName() { + return name; + } + + public String getUsage() { + return usage; + } + } + + private static final Logger logger = LogManager.getLogger(); + private Marker subtitleMarker = MarkerManager.getMarker("MISSING-SUBTITLE"); + private static final int CONNECTION_TIMEOUT = 5000; + private static final int SOCKET_TIMEOUT = 5000; + public static final String GETMEDIAUSAGEBYUTRANGE = "/getMediaUsageByUTRange"; + public static final String GETCUSTOMVIEW = "/getCustomView"; + + private static final Path subtitleRoot = Paths.get("X:\\PB_ARCH"); + private static final Path missingSubsRoot = Paths.get("R:\\BeachPool"); + private Set subsDirectoriesList; + + private static Document toDocument(String xml) { + DocumentBuilderFactory factory = DocumentBuilderFactory.newInstance(); + DocumentBuilder builder = null; + try { + builder = factory.newDocumentBuilder(); + Document doc = builder.parse(new InputSource(new StringReader(xml))); + return doc; + } catch (Exception e) { + logger.error(e.getMessage()); + } + return null; + } + + private SimpleDateFormat sdf_range = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss:'00'"); + private String primaryEndPoint; + private String primaryUserName; + private String primaryPassword; + private String secondaryEndPoint; + private String secondaryUserName; + + private String secondaryPassword; + + private int rangeForwardHours; + + private String getAttribute(Node node, String name) { + String result = null; + if (node != null && node.hasAttributes()) { + NamedNodeMap attributes = node.getAttributes(); + if (attributes != null) { + Node mediaNameNode = attributes.getNamedItem(name); + if (mediaNameNode != null) + result = mediaNameNode.getNodeValue(); + } + } + return result; + } + + public Collection getPossibelMissingMaterialNames(List poolContent) throws Exception { + Map result = querySortableMedias(poolContent); + queryOtherMedias(poolContent, result); + + TreeMap sortedUsages = new TreeMap<>(); + for (MMMedia m : result.values()) { + // a TreeMap lehetove teszi: nincs duplikatum + a korabbi musorok elore kerulnek + sortedUsages.put(m.getUsage() + "." + m.getName(), m); + } + + return sortedUsages.values(); + + } + + private Path getSOAPTemplate(String viewName) throws FileNotFoundException { + String configDir = SystemConfiguration.getInstance().getConfig("settings/soap"); + logger.info("Loading template {} from {}", viewName, configDir); + return Paths.get(configDir, viewName); + } + + public void init(String primaryEndPoint, String primaryUserName, String primaryPassword, String secondaryEndPoint, + String secondaryUserName, String secondaryPassword, int rangeForwardHours) throws Exception { + this.primaryEndPoint = primaryEndPoint; + this.primaryUserName = primaryUserName; + this.primaryPassword = primaryPassword; + this.secondaryEndPoint = secondaryEndPoint; + this.secondaryUserName = secondaryUserName; + this.secondaryPassword = secondaryPassword; + this.rangeForwardHours = rangeForwardHours; + this.subsDirectoriesList = readSubtileDirectoriesList(subtitleRoot); + } + + private void putEarliest(Map result, String mediaName, String usageTime) { + if (result.containsKey(mediaName)) { + MMMedia m = result.get(mediaName); + // az aktualis a korabbi idopont + if (usageTime.compareTo(m.getUsage()) < 0) + result.put(mediaName, new MMMedia(mediaName, usageTime)); + } else + result.put(mediaName, new MMMedia(mediaName, usageTime)); + } + + public String query(String endPoint, String action, String soap, String user, String pwd) throws Exception { + + HttpParams httpParameters = new BasicHttpParams(); + HttpConnectionParams.setConnectionTimeout(httpParameters, CONNECTION_TIMEOUT); + HttpConnectionParams.setSoTimeout(httpParameters, SOCKET_TIMEOUT); + DefaultHttpClient httpclient = new DefaultHttpClient(httpParameters); + // http://10.170.100.61:18083 + + String result = null; + DataInputStream is = null; + try { + URL url = new URL(endPoint); + httpclient.getCredentialsProvider().setCredentials( + new AuthScope(url.getHost(), url.getPort(), null, "Digest"), + new UsernamePasswordCredentials(user, pwd)); + + HttpPost httppost = new HttpPost(url.toURI()); + httppost.setHeader("soapaction", action); + // Content-Type application/soap+xml; charset=utf-8; + // action="/getMediaUsageByUTRange" + httppost.setHeader("Content-Type", "text/xml; charset=utf-8"); + HttpEntity entity = new StringEntity(soap.toString(), HTTP.UTF_8); + httppost.setEntity(entity); + logger.info("Executing call {}", action); + HttpResponse response = httpclient.execute(httppost);// calling server + HttpEntity r_entity = response.getEntity(); // get response + // Header[] headers = response.getAllHeaders(); + // for (Header h : headers) { + // logger.info("Reponse Header", h.getName() + ": " + h.getValue()); + // } + if (r_entity != null) { + byte[] bytes = new byte[(int) r_entity.getContentLength()]; + if (r_entity.isStreaming()) { + is = new DataInputStream(r_entity.getContent()); + is.readFully(bytes); + } + result = new String(bytes); + } + } catch (Exception e) { + logger.error("Exception while connecting to endpoint {}. System message is: {}", e.getMessage()); + throw e; + } finally { + if (is != null) { + try { + is.close(); + } catch (IOException e) { + } + } + if (httpclient != null) { + httpclient.getConnectionManager().shutdown(); + } + } + + return result; + } + + public void queryOtherMedias(List poolContent, Map result) throws Exception { + String xml = null; + String soap = null; + + Path template = getSOAPTemplate(GETCUSTOMVIEW); + + if (!template.toFile().exists()) + throw new FileNotFoundException(template.toString()); + try { + soap = new String(Files.readAllBytes(template)); + xml = query(primaryEndPoint, GETCUSTOMVIEW, soap, primaryUserName, primaryPassword); + } catch (Exception e1) { + try { + xml = query(secondaryEndPoint, GETCUSTOMVIEW, soap, secondaryUserName, secondaryPassword); + } catch (Exception e2) { + } + } + + if (StringUtils.isBlank(xml)) { + logger.info("Query result is empty"); + return; + } + + Document document = toDocument(xml); + NodeList mediaNodes = document.getElementsByTagName("media"); + + if (mediaNodes == null || mediaNodes.getLength() == 0) + return; + + for (int i = 0; i < mediaNodes.getLength(); i++) { + Node mediaNode = mediaNodes.item(i); + String mediaName = getAttribute(mediaNode, "mediaName"); + if (StringUtils.isBlank(mediaName)) { + continue; + } + + String usageType = getAttribute(mediaNode, "UsageType"); + if (!"PrimaryVideo".equals(usageType) && !"Live".equals(usageType)) + logger.info(subtitleMarker, "queryOtherMedias: {}, {}", usageType, mediaName); + + if (poolContent.contains(mediaName)) + continue; + + String usageTime = getAttribute(mediaNode, "earliestUsageTime"); + putEarliest(result, mediaName, usageTime); + } + + logger.info("Document done, items {}", mediaNodes.getLength()); + + } + + public Map querySortableMedias(List poolContent) throws Exception { + Map result = new HashMap<>(); + + String xml = null; + String soap = null; + + Path template = getSOAPTemplate(GETMEDIAUSAGEBYUTRANGE); + + if (!template.toFile().exists()) + throw new FileNotFoundException(template.toString()); + + try { + long now = System.currentTimeMillis(); + Calendar cal = Calendar.getInstance(); + cal.setTimeInMillis(now); + cal.add(Calendar.HOUR_OF_DAY, rangeForwardHours); + + String fromDateStr = sdf_range.format(new Date(now)); + String toDateStr = sdf_range.format(cal.getTime()); + + soap = new String(Files.readAllBytes(template)); + soap = soap.replace("{start}", fromDateStr); + soap = soap.replace("{end}", toDateStr); + + xml = query(primaryEndPoint, GETMEDIAUSAGEBYUTRANGE, soap, primaryUserName, primaryPassword); + } catch (Exception e1) { + try { + xml = query(secondaryEndPoint, GETMEDIAUSAGEBYUTRANGE, soap, secondaryUserName, secondaryPassword); + } catch (Exception e2) { + } + } + + if (StringUtils.isBlank(xml)) { + logger.info("Query result is empty"); + return result; + } + + Document document = toDocument(xml); + NodeList mediaNodes = document.getElementsByTagName("mediaUsage"); + + if (mediaNodes == null || mediaNodes.getLength() == 0) + return result; + + for (int i = 0; i < mediaNodes.getLength(); i++) { + Node mediaNode = mediaNodes.item(i); + String mediaName = getAttribute(mediaNode, "mediaName"); + String mediaType = getAttribute(mediaNode, "mediaType"); + if (StringUtils.isBlank(mediaName)) + continue; + + if (StringUtils.equals("Subtitle", mediaType)) { + logger.info(this.subtitleMarker, "Missing subtitle: {}", mediaName); + + try { + this.copyMissingSubtitle(mediaName + ".stl"); + } catch (Exception e) { + logger.error(this.subtitleMarker, e.getMessage()); + } + + continue; + } + + if (!StringUtils.equals("Video", mediaType)) + continue; + + if (poolContent.contains(mediaName)) + continue; + + Node usageChild = mediaNode.getFirstChild(); + if (usageChild == null) { + logger.info("Skipping {}, no media usage defined", mediaName); + continue; + } + + NodeList usageNodes = usageChild.getChildNodes(); + if (usageNodes == null || usageNodes.getLength() == 0) { + logger.info("Skipping {}, no media usage defined", mediaName); + continue; + } + + for (int j = 0; j < usageNodes.getLength(); j++) { + Node mediaUsageNode = usageNodes.item(j); + String usageTime = getAttribute(mediaUsageNode, "earliestUsageTime"); + logger.info("{}, {}", mediaName, usageTime); + putEarliest(result, mediaName, usageTime); + } + + } + + return result; + } + + private Set readSubtileDirectoriesList(Path root) { + Set result = new HashSet<>(); + DirectoryStream stream = null; + try { + stream = Files.newDirectoryStream(Paths.get(root.toString())); + Iterator dirIterator = stream.iterator(); + + while (dirIterator.hasNext()) { + Path path = dirIterator.next(); + if (Files.isDirectory(path)) { + result.add(path); + logger.info(this.subtitleMarker, "Detected subtitle root {}", path); + } + } + } catch (Exception e) { + logger.error(this.subtitleMarker, e.getMessage()); + } finally { + try { + if (stream != null) + stream.close(); + } catch (IOException e) { + } + + } + return result; + } + + private void copyMissingSubtitle(String subtitleName) throws Exception { + boolean found = false; + for (Path subtitleDir : this.subsDirectoriesList) { + Path source = Paths.get(subtitleDir.toString(), new String[] { subtitleName }); + if (source.toFile().exists()) { + found = true; + logger.info(this.subtitleMarker, "Found missing {}", source); + Path target = Paths.get(missingSubsRoot.toString(), new String[] { subtitleName }); + if (target.toFile().exists()) + logger.info(this.subtitleMarker, "Missing subtitle {} already exists, overriding", target); + Files.copy(source, target, new CopyOption[] { StandardCopyOption.REPLACE_EXISTING }); + boolean isContentEquals = Arrays.equals(Files.readAllBytes(source), Files.readAllBytes(target)); + if (isContentEquals) { + logger.info(this.subtitleMarker, "Successfully copied {}", target); + break; + } + logger.error(this.subtitleMarker, "Source {} and target {} content are not equals", source, target); + break; + } + } + if (!found) + logger.warn(this.subtitleMarker, "Can't find missing {}", subtitleName); + } +} diff --git a/server/-product/production/AMC/jobs/templates/FILEZILLA_AVID-validate-and-restore.xml b/server/-product/production/AMC/jobs/templates/FILEZILLA_AVID-validate-and-restore.xml new file mode 100644 index 00000000..5395ce44 --- /dev/null +++ b/server/-product/production/AMC/jobs/templates/FILEZILLA_AVID-validate-and-restore.xml @@ -0,0 +1,95 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/server/-product/production/AMC/jobs/templates/FILEZILLA_PASARESTORE-validate-and-archive.xml b/server/-product/production/AMC/jobs/templates/FILEZILLA_PASARESTORE-validate-and-archive.xml new file mode 100644 index 00000000..7caa7e2c --- /dev/null +++ b/server/-product/production/AMC/jobs/templates/FILEZILLA_PASARESTORE-validate-and-archive.xml @@ -0,0 +1,94 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/server/-product/production/AMC/jobs/templates/FILEZILLA_PASARESTORE-validate-and-restore.xml b/server/-product/production/AMC/jobs/templates/FILEZILLA_PASARESTORE-validate-and-restore.xml new file mode 100644 index 00000000..da82bd6d --- /dev/null +++ b/server/-product/production/AMC/jobs/templates/FILEZILLA_PASARESTORE-validate-and-restore.xml @@ -0,0 +1,95 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/server/-product/production/AMC/jobs/templates/NEXIO1-validate-and-archive.xml b/server/-product/production/AMC/jobs/templates/NEXIO1-validate-and-archive.xml new file mode 100644 index 00000000..7426382a --- /dev/null +++ b/server/-product/production/AMC/jobs/templates/NEXIO1-validate-and-archive.xml @@ -0,0 +1,94 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/server/-product/production/AMC/jobs/templates/NEXIO1-validate-and-restore.xml b/server/-product/production/AMC/jobs/templates/NEXIO1-validate-and-restore.xml new file mode 100644 index 00000000..5b11ead5 --- /dev/null +++ b/server/-product/production/AMC/jobs/templates/NEXIO1-validate-and-restore.xml @@ -0,0 +1,95 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/server/-product/production/AMC/jobs/templates/NEXIO2-validate-and-archive.xml b/server/-product/production/AMC/jobs/templates/NEXIO2-validate-and-archive.xml new file mode 100644 index 00000000..005122a1 --- /dev/null +++ b/server/-product/production/AMC/jobs/templates/NEXIO2-validate-and-archive.xml @@ -0,0 +1,94 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/server/-product/production/AMC/jobs/templates/NEXIO2-validate-and-restore.xml b/server/-product/production/AMC/jobs/templates/NEXIO2-validate-and-restore.xml new file mode 100644 index 00000000..15b0e18b --- /dev/null +++ b/server/-product/production/AMC/jobs/templates/NEXIO2-validate-and-restore.xml @@ -0,0 +1,95 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/server/-product/production/AMC/jobs/templates/PASAPOOL-validate-and-restore.xml b/server/-product/production/AMC/jobs/templates/PASAPOOL-validate-and-restore.xml new file mode 100644 index 00000000..df9ebe1d --- /dev/null +++ b/server/-product/production/AMC/jobs/templates/PASAPOOL-validate-and-restore.xml @@ -0,0 +1,95 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/server/-product/production/AMC/jobs/templates/PB-archive-checker.xml b/server/-product/production/AMC/jobs/templates/PB-archive-checker.xml new file mode 100644 index 00000000..54d36065 --- /dev/null +++ b/server/-product/production/AMC/jobs/templates/PB-archive-checker.xml @@ -0,0 +1,26 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/server/-product/production/AMC/jobs/templates/PBARCHIVE-validate-and-archive.xml b/server/-product/production/AMC/jobs/templates/PBARCHIVE-validate-and-archive.xml new file mode 100644 index 00000000..7740b8e0 --- /dev/null +++ b/server/-product/production/AMC/jobs/templates/PBARCHIVE-validate-and-archive.xml @@ -0,0 +1,85 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/server/-product/production/AMC/jobs/templates/VOD-validate-and-restore.xml b/server/-product/production/AMC/jobs/templates/VOD-validate-and-restore.xml new file mode 100644 index 00000000..5cdcadb1 --- /dev/null +++ b/server/-product/production/AMC/jobs/templates/VOD-validate-and-restore.xml @@ -0,0 +1,89 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/server/-product/production/AMC/jobs/templates/cancelable.xml b/server/-product/production/AMC/jobs/templates/cancelable.xml new file mode 100644 index 00000000..fa29cc3f --- /dev/null +++ b/server/-product/production/AMC/jobs/templates/cancelable.xml @@ -0,0 +1,17 @@ + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/server/-product/production/AMC/jobs/templates/create-lowres-ondemand.xml b/server/-product/production/AMC/jobs/templates/create-lowres-ondemand.xml new file mode 100644 index 00000000..375beb2f --- /dev/null +++ b/server/-product/production/AMC/jobs/templates/create-lowres-ondemand.xml @@ -0,0 +1,106 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/server/-product/production/AMC/jobs/templates/fork-validate-and-archive.xml b/server/-product/production/AMC/jobs/templates/fork-validate-and-archive.xml new file mode 100644 index 00000000..5136c36d --- /dev/null +++ b/server/-product/production/AMC/jobs/templates/fork-validate-and-archive.xml @@ -0,0 +1,29 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/server/-product/production/AMC/jobs/templates/fork-validate-and-restore.xml b/server/-product/production/AMC/jobs/templates/fork-validate-and-restore.xml new file mode 100644 index 00000000..bddbab68 --- /dev/null +++ b/server/-product/production/AMC/jobs/templates/fork-validate-and-restore.xml @@ -0,0 +1,29 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/server/-product/production/AMC/jobs/templates/generic-archive-checker.xml b/server/-product/production/AMC/jobs/templates/generic-archive-checker.xml new file mode 100644 index 00000000..500369e8 --- /dev/null +++ b/server/-product/production/AMC/jobs/templates/generic-archive-checker.xml @@ -0,0 +1,34 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/server/-product/production/AMC/jobs/templates/harris1-missingmaterial-checker.xml b/server/-product/production/AMC/jobs/templates/harris1-missingmaterial-checker.xml new file mode 100644 index 00000000..5777fa2e --- /dev/null +++ b/server/-product/production/AMC/jobs/templates/harris1-missingmaterial-checker.xml @@ -0,0 +1,30 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/server/-product/production/AMC/jobs/templates/harris2-missingmaterial-checker.xml b/server/-product/production/AMC/jobs/templates/harris2-missingmaterial-checker.xml new file mode 100644 index 00000000..5777fa2e --- /dev/null +++ b/server/-product/production/AMC/jobs/templates/harris2-missingmaterial-checker.xml @@ -0,0 +1,30 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/server/-product/production/AMC/jobs/templates/nexio1-archive-checker.xml b/server/-product/production/AMC/jobs/templates/nexio1-archive-checker.xml new file mode 100644 index 00000000..991c0626 --- /dev/null +++ b/server/-product/production/AMC/jobs/templates/nexio1-archive-checker.xml @@ -0,0 +1,26 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/server/-product/production/AMC/jobs/templates/nexio2-archive-checker.xml b/server/-product/production/AMC/jobs/templates/nexio2-archive-checker.xml new file mode 100644 index 00000000..991c0626 --- /dev/null +++ b/server/-product/production/AMC/jobs/templates/nexio2-archive-checker.xml @@ -0,0 +1,26 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/server/-product/production/AMC/jobs/templates/peablebeach-missingmaterial-checker.xml b/server/-product/production/AMC/jobs/templates/peablebeach-missingmaterial-checker.xml new file mode 100644 index 00000000..b05b2ef8 --- /dev/null +++ b/server/-product/production/AMC/jobs/templates/peablebeach-missingmaterial-checker.xml @@ -0,0 +1,58 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/server/-product/production/AMC/jobs/templates/register-user-restore.xml b/server/-product/production/AMC/jobs/templates/register-user-restore.xml new file mode 100644 index 00000000..43a94d22 --- /dev/null +++ b/server/-product/production/AMC/jobs/templates/register-user-restore.xml @@ -0,0 +1,29 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/server/-product/production/AMC/jobs/templates/register-vod-restore.xml b/server/-product/production/AMC/jobs/templates/register-vod-restore.xml new file mode 100644 index 00000000..ac996108 --- /dev/null +++ b/server/-product/production/AMC/jobs/templates/register-vod-restore.xml @@ -0,0 +1,21 @@ + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/server/-product/production/AMC/jobs/templates/server-status-report.xml b/server/-product/production/AMC/jobs/templates/server-status-report.xml new file mode 100644 index 00000000..eec3da65 --- /dev/null +++ b/server/-product/production/AMC/jobs/templates/server-status-report.xml @@ -0,0 +1,6 @@ + + + + + + \ No newline at end of file diff --git a/server/-product/production/AMC/jobs/templates/sync-subtitles.xml b/server/-product/production/AMC/jobs/templates/sync-subtitles.xml new file mode 100644 index 00000000..475ab5d2 --- /dev/null +++ b/server/-product/production/AMC/jobs/templates/sync-subtitles.xml @@ -0,0 +1,40 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/server/-product/production/AMC/jobs/templates/test-fork-cancelable.xml b/server/-product/production/AMC/jobs/templates/test-fork-cancelable.xml new file mode 100644 index 00000000..8c88d84b --- /dev/null +++ b/server/-product/production/AMC/jobs/templates/test-fork-cancelable.xml @@ -0,0 +1,6 @@ + + + + + + \ No newline at end of file diff --git a/server/-product/production/AMC/jobs/templates/validate-dir-mxf.xml b/server/-product/production/AMC/jobs/templates/validate-dir-mxf.xml new file mode 100644 index 00000000..2a2972be --- /dev/null +++ b/server/-product/production/AMC/jobs/templates/validate-dir-mxf.xml @@ -0,0 +1,17 @@ + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/server/-product/production/AMC/settings/application.yaml b/server/-product/production/AMC/settings/application.yaml new file mode 100644 index 00000000..78d138a1 --- /dev/null +++ b/server/-product/production/AMC/settings/application.yaml @@ -0,0 +1,37 @@ +datasource: + mediacube: + url: jdbc:db2://192.168.0.15:50000/mc + user: db2admin + password: password + external-indexer: false + simple-search: true + login-timeout: 3 + pool-size: 10 + mediacube-nosql: + url: jdbc:db2://192.168.0.15:50000/mc + user: db2admin + password: password + schema: test + login-timeout: 3 +services: + ffmpeg: + execurable-location: /opt/ffmpeg/ffmpeg + mediacube: + proxy-root: /opt + nexio: + host: 10.10.1.55 + collection-name: nexioclips + use-mos-gateway: true + disabled: true +jobs: + validate-transfers: false + copy-buffer-size: 32768 + scheduled-execution-disabled: false +tsm: + randomize-archives: false + delimiter: \ + node-name: PASANODE + fs-name: NEXIO + alternate-fs-name: PASA + hl-name: \ + \ No newline at end of file diff --git a/server/-product/production/AMC/settings/dsm.opt b/server/-product/production/AMC/settings/dsm.opt new file mode 100644 index 00000000..952cfa11 --- /dev/null +++ b/server/-product/production/AMC/settings/dsm.opt @@ -0,0 +1,6 @@ +NODENAME pasanode +TCPSERVERADDRESS 192.168.0.9 +passwordaccess generate + +commmethod TCPIP +tcpport 1500 diff --git a/server/-product/production/AMC/settings/jetty.xml b/server/-product/production/AMC/settings/jetty.xml new file mode 100644 index 00000000..68c77169 --- /dev/null +++ b/server/-product/production/AMC/settings/jetty.xml @@ -0,0 +1,35 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/server/-product/production/AMC/settings/log4j2-210810.xml b/server/-product/production/AMC/settings/log4j2-210810.xml new file mode 100644 index 00000000..b3d6ea00 --- /dev/null +++ b/server/-product/production/AMC/settings/log4j2-210810.xml @@ -0,0 +1,202 @@ + + + + log + ${logPath}/mediacube.log + ${logPath}/$${date:yyyy-MM}/mediacube-%d{MM-dd-yyyy}-%i.log.gz + ${logPath}/markered-mediacube.log + ${logPath}/$${date:yyyy-MM}/markered-mediacube-%d{MM-dd-yyyy}-%i.log.gz + ${logPath}/mediacube-err.log + ${logPath}/$${date:yyyy-MM}/mediacube-err-%d{MM-dd-yyyy}-%i.log.gz + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/server/-product/production/AMC/settings/log4j2.xml b/server/-product/production/AMC/settings/log4j2.xml new file mode 100644 index 00000000..74fbfa7f --- /dev/null +++ b/server/-product/production/AMC/settings/log4j2.xml @@ -0,0 +1,209 @@ + + + + log + ${logPath}/mediacube.log + ${logPath}/$${date:yyyy-MM}/mediacube-%d{MM-dd-yyyy}-%i.log.gz + ${logPath}/markered-mediacube.log + ${logPath}/$${date:yyyy-MM}/markered-mediacube-%d{MM-dd-yyyy}-%i.log.gz + ${logPath}/mediacube-err.log + ${logPath}/$${date:yyyy-MM}/mediacube-err-%d{MM-dd-yyyy}-%i.log.gz + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/server/-product/production/AMC/settings/maestro.yaml b/server/-product/production/AMC/settings/maestro.yaml new file mode 100644 index 00000000..1d361571 --- /dev/null +++ b/server/-product/production/AMC/settings/maestro.yaml @@ -0,0 +1,38 @@ +sourceStoreUri: + name: Default + protocol: LOCAL + uri: "/mnt/NLE" + fileFilter: "*.mxf" + showDirectories: true +alternateSourceStoreUris: +- name: NLE1 + protocol: LOCAL + uri: "/mnt/NLE/NLE1" + fileFilter: "*.mxf" + showDirectories: true +- name: NLE2 + protocol: LOCAL + uri: "/mnt/NLE/NLE2" + fileFilter: "*.mxf" + showDirectories: true +- name: NLE3 + protocol: LOCAL + uri: "/mnt/NLE/NLE3" + fileFilter: "*.mxf" + showDirectories: true +- name: NLE4 + protocol: LOCAL + uri: "/mnt/NLE/NLE4" + fileFilter: "*.mxf" + showDirectories: true +- name: POLC + protocol: LOCAL + uri: "/mnt/POLC" + fileFilter: "*.mxf" + showDirectories: true +targets: +- name: FINISHED_SHOWS + killDateDays: 7 + storeUri: + protocol: LOCAL + uri: "/mnt/PROMISE/FINISHED_SHOWS" diff --git a/server/-product/production/AMC/settings/mediacube.yaml b/server/-product/production/AMC/settings/mediacube.yaml new file mode 100644 index 00000000..291a0ca1 --- /dev/null +++ b/server/-product/production/AMC/settings/mediacube.yaml @@ -0,0 +1,55 @@ +jobQueuePollInterval: 1000 +disableHelp: true +maestroDisabled: true +alternateRetrieveSelector: true +disableStatistics: true +disableEditor: false +targetRestoreFilters: +- FILEZILLA_AVID +- FILEZILLA_PASARESTORE +topTypeFilters: +- name: Hír bejátszó +- name: Hír nyers +- name: Visszarögzített +- name: Egyéb +bottomTypeFilters: +- name: Műsor +- name: Műsor nyers +- name: Promo +- name: Promo nyers +- name: Reklám +- name: Reklám nyers +authentication: + authEnabled: true + adHost: intra.amc.hu + adNonSecurePort: 3268 + adBaseDn: DC=intra,DC=amc,DC=hu + adAdminMap: + - INFORMATIKUSOK + - MUSZAKVEZETOK + adSubmitterMap: + - INFORMATIKUSOK + - MUSZAKVEZETOK + adEditorMap: + - INFORMATIKUSOK + - MUSZAKVEZETOK + localAccounts: + - user: user + password: 5F4DCC3B5AA765D61D8327DEB882CF99 + email: + - user: user1 + password: 5F4DCC3B5AA765D61D8327DEB882CF99 + email: + - user: lebony + password: 4E25B117B14D86D7DCECB4E433CF932C + email: + - user: root + password: 5F4DCC3B5AA765D61D8327DEB882CF99 + email: vasary@elgekko.net + localAdmins: + - root + - admin + localSubmitters: + - lebony + localEditors: + - editor -- 2.54.0