From: vasary.daniel Date: Tue, 10 Aug 2021 14:38:34 +0000 (+0000) Subject: git-tfs-id: [http://tfs.userrendszerhaz.hu:8080/tfs/DefaultCollection]$/MediaCube... X-Git-Url: http://git.useribm.hu/?a=commitdiff_plain;h=b1b48c407b48928a8c1a6571ef041be0b4f94bb0;p=mediacube.git git-tfs-id: [tfs.userrendszerhaz.hu:8080/tfs/DefaultCollection]$/MediaCube;C32302 --- diff --git a/server/-product/production/AMC/jobs/classes/user/jobengine/server/steps/EscortFiles.class b/server/-product/production/AMC/jobs/classes/user/jobengine/server/steps/EscortFiles.class deleted file mode 100644 index 82b58552..00000000 Binary files a/server/-product/production/AMC/jobs/classes/user/jobengine/server/steps/EscortFiles.class and /dev/null differ diff --git a/server/-product/production/AMC/jobs/classes/user/jobengine/server/steps/PBQuery$MMMedia.class b/server/-product/production/AMC/jobs/classes/user/jobengine/server/steps/PBQuery$MMMedia.class deleted file mode 100644 index eb6cb52d..00000000 Binary files a/server/-product/production/AMC/jobs/classes/user/jobengine/server/steps/PBQuery$MMMedia.class and /dev/null differ diff --git a/server/-product/production/AMC/jobs/classes/user/jobengine/server/steps/PBQuery.class b/server/-product/production/AMC/jobs/classes/user/jobengine/server/steps/PBQuery.class deleted file mode 100644 index e5a67790..00000000 Binary files a/server/-product/production/AMC/jobs/classes/user/jobengine/server/steps/PBQuery.class and /dev/null differ diff --git a/server/-product/production/AMC/jobs/executors.xml b/server/-product/production/AMC/jobs/executors.xml deleted file mode 100644 index 4e6b6822..00000000 --- a/server/-product/production/AMC/jobs/executors.xml +++ /dev/null @@ -1,32 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - \ No newline at end of file diff --git a/server/-product/production/AMC/jobs/schedules.json b/server/-product/production/AMC/jobs/schedules.json deleted file mode 100644 index 822c1905..00000000 --- a/server/-product/production/AMC/jobs/schedules.json +++ /dev/null @@ -1,136 +0,0 @@ -{"joblist":[ - { - "template": "sync-subtitles.xml", - "name" : "Feliratok szinkronizálása", - "parameters": [ - {"name": "sourceFolder", "value": "y:\\IBMS_Rundowns", "type": "java.lang.String"}, - { - "name": "filter", - "value": { - "fileName": ".*\\.(txt)$" - }, - "type": "com.ibm.nosql.json.api.BasicDBObject" - } - ] - }, - { - "template": "server-status-report.xml", - "name" : "MediaCube státusz", - "active": true, - "executeimmediate": false, - "cronexpression": "0 */30 * * * ?" - }, - { - "template": "nexio1-archive-checker.xml", - "name" : "NEXIO1 archiválás előkészítése", - "active": true, - "executeimmediate": false, - "cronexpression": "0 0 */1 * * ?", - "parameters": - [ - {"name": "sourceStoreName", "value": "NEXIO1", "type": "java.lang.String"}, - {"name": "escortStoreName", "value": "MEDIACUBE_DOWNLOADS", "type": "java.lang.String"}, - {"name": "lastModifiedHours", "value": 3, "type": "java.lang.Integer"} - ] - }, - { - "template": "nexio2-archive-checker.xml", - "name" : "NEXIO2 archiválás előkészítése", - "active": true, - "executeimmediate": false, - "cronexpression": "0 0 */1 * * ?", - "parameters": - [ - {"name": "sourceStoreName", "value": "NEXIO2", "type": "java.lang.String"}, - {"name": "escortStoreName", "value": "MEDIACUBE_DOWNLOADS", "type": "java.lang.String"}, - {"name": "lastModifiedHours", "value": 3, "type": "java.lang.Integer"} - ] - }, - { - "template": "fork-validate-and-archive.xml", - "name" : "Kötegelt validálás és archiválás", - "active": true, - "executeimmediate": false, - "cronexpression": "0 */30 * * * ?", - "parameters": - [ - {"name": "sourceStoreName", "value": "MEDIACUBE_DOWNLOADS", "type": "java.lang.String"}, - {"name": "template", "value": "validate-and-archive.xml", "type": "java.lang.String"}, - {"name": "expectedColorSpace", "value": "", "type": "java.lang.String"}, - {"name": "limit", "value": 100, "type": "java.lang.Integer"} - ] - }, - { - "template": "harris1-missingmaterial-checker.xml", - "name" : "HARRIS1 missing material lekérdezés", - "active": true, - "executeimmediate": false, - "cronexpression": "0 */5 * * * ?", - "parameters": - [ - {"name": "vicFiles", "value": ["z:\\NEXIO.VIC", "z:\\EDIT 9.VIC", "z:\\TRIM2.VIC"], "type": "com.ibm.nosql.json.api.BasicDBList"}, - {"name": "escortStoreName", "value": "MEDIACUBE_UPLOADS", "type": "java.lang.String"}, - {"name": "targetStoreName", "value": "NEXIO1", "type": "java.lang.String"}, - {"name": "targetProtocol", "value": "FTP", "type": "java.lang.String"} - ] - }, - { - "template": "harris2-missingmaterial-checker.xml", - "name" : "HARRIS2 missing material lekérdezés", - "active": true, - "executeimmediate": false, - "cronexpression": "0 */5 * * * ?", - "parameters": - [ - {"name": "vicFiles", "value": ["z:\\NEXIO2.VIC", "z:\\EDIT-4.VIC", "z:\\EDIT 10.VIC"], "type": "com.ibm.nosql.json.api.BasicDBList"}, - {"name": "escortStoreName", "value": "MEDIACUBE_UPLOADS", "type": "java.lang.String"}, - {"name": "targetStoreName", "value": "NEXIO2", "type": "java.lang.String"}, - {"name": "targetProtocol", "value": "FTP", "type": "java.lang.String"} - ] - }, - { - "template": "peablebeach-missingmaterial-checker.xml", - "name" : "PEABLEBEACH missing material lekérdezés", - "active": true, - "executeimmediate": false, - "cronexpression": "0 */5 * * * ?", - "parameters": - [ - {"name": "escortStoreName", "value": "MEDIACUBE_UPLOADS", "type": "java.lang.String"}, - {"name": "lookupStoreNames", "value": ["PEABLEBEACH", "PASAPOOL", "SELENIOPOOL", "SELENIOPOOL_16_9", "SELENIOPOOL_16_9_PROCESSING", "SELENIOPOOL_4_3", "SELENIOPOOL_4_3_PROCESSING"], "type": "com.ibm.nosql.json.api.BasicDBList"}, - {"name": "targetStoreName", "value": "PASAPOOL", "type": "java.lang.String"}, - {"name": "targetProtocol", "value": "FTP", "type": "java.lang.String"}, - {"name": "primaryEndPoint", "value": "http://10.170.100.61:18083", "type": "java.lang.String"}, - {"name": "primaryUserName", "value": "pbs", "type": "java.lang.String"}, - {"name": "primaryPassword", "value": "pbs", "type": "java.lang.String"}, - {"name": "secondaryEndPoint", "value": "http://10.170.100.62:18083", "type": "java.lang.String"}, - {"name": "secondaryUserName", "value": "pbs", "type": "java.lang.String"}, - {"name": "secondaryPassword", "value": "pbs", "type": "java.lang.String"}, - {"name": "rangeForwardHours", "value": 168, "type": "java.lang.Integer"} - ] - }, - { - "template": "fork-validate-and-restore.xml", - "name" : "Kötegelt validálás és visszatöltés", - "active": true, - "executeimmediate": false, - "cronexpression": "0 */1 * * * ?", - "parameters": - [ - {"name": "sourceStoreName", "value": "MEDIACUBE_UPLOADS", "type": "java.lang.String"}, - {"name": "template", "value": "validate-and-restore.xml", "type": "java.lang.String"}, - {"name": "expectedColorSpace", "value": "", "type": "java.lang.String"}, - {"name": "limit", "value": 0, "type": "java.lang.Integer"} - ] - }, - { - "template": "register-user-restore.xml", - "name" : "Felhasználói visszatöltés regisztráció", - "active": false, - "executeimmediate": false, - "parameters": - [ - {"name": "escortStoreName", "value": "MEDIACUBE_UPLOADS", "type": "java.lang.String"} - ] - } -]} diff --git a/server/-product/production/AMC/jobs/steps/CancelableStep.java b/server/-product/production/AMC/jobs/steps/CancelableStep.java deleted file mode 100644 index 40812594..00000000 --- a/server/-product/production/AMC/jobs/steps/CancelableStep.java +++ /dev/null @@ -1,58 +0,0 @@ -package user.jobengine.server.steps; - -import org.apache.commons.net.ftp.FTPClient; -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; - -import user.commons.StoreUri; -import user.commons.remotestore.FtpDirectoryLister; -import user.commons.remotestore.RemoteStoreProtocol; - -public class CancelableStep extends JobStep { - private static final Logger logger = LogManager.getLogger(); - int count = 10; - - @StepEntry - public Object[] execute(int param) throws Exception { - try { - // ftpTest(); - - for (int i = 0; i < count; i++) { - if (getJobRuntime().isWaitingCancel()) - break; - Thread.sleep(500); - int progress = (i + 1) * 100 / count; - setProgress(progress); - - } - } catch (Exception e) { - e.printStackTrace(); - throw e; - } - return null; - } - - void ftpTest() throws Exception { - StoreUri source = null; - StoreUri target = null; - try { - source = getManager().getStoreUri("MEDIACUBE_UPLOADS", RemoteStoreProtocol.LOCAL); - target = getManager().getStoreUri("NEXIO1", RemoteStoreProtocol.FTP); - source.addProgressListener(e -> setProgress(e.getProgress())); - logger.info("Uploading file PART_TEST.mxf"); - source.transferFrom(target, "PART_TEST.mxf", "PART_TEST.mxf"); - FTPClient client = ((FtpDirectoryLister) target.getLister()).connect(); - logger.info("Renaming file PART_TEST.mxf to RENAME_TEST.mxf"); - client.rename("PART_TEST.mxf", "RENAME_TEST.mxf"); - } catch (Exception e) { - logger.catching(e); - } finally { - if (target != null) - target.cleanUp(); - if (source != null) - source.cleanUp(); - } - - } - -} diff --git a/server/-product/production/AMC/jobs/steps/DirMXFValidatorStep.java b/server/-product/production/AMC/jobs/steps/DirMXFValidatorStep.java deleted file mode 100644 index f26d679b..00000000 --- a/server/-product/production/AMC/jobs/steps/DirMXFValidatorStep.java +++ /dev/null @@ -1,61 +0,0 @@ -package user.jobengine.server.steps; - -import java.nio.file.DirectoryStream; -import java.nio.file.Files; -import java.nio.file.Path; -import java.nio.file.Paths; - -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; - -import user.commons.mediatool.MediaInfo; -import user.jobengine.db.Media; - -public class DirMXFValidatorStep extends JobStep { - private static final Logger logger = LogManager.getLogger(); - - @StepEntry - public Object[] execute(String source) throws Exception { - Path path = Paths.get(source); - DirectoryStream directoryStream = Files.newDirectoryStream(path); - for (Path file : directoryStream) { - if (getJobRuntime().isWaitingCancel()) { - cancel(); - return null; - } - - if (file.toFile().isDirectory()) - continue; - - String fileName = file.getFileName().toString(); - if (fileName.contains(".")) - fileName = fileName.substring(0, fileName.lastIndexOf(".")); - - if (file.toFile().length() == 0) { - logger.error("Zero: {}", fileName); - continue; - } - - Media media = getManager().getMedia(fileName); - long frames = 0; - if (media == null) { - logger.warn("Unchecked yet: {}", fileName); - } else - frames = media.getLength(); - - MediaInfo mi = null; - - try { - mi = new MediaInfo(file); - mi.process(); - } catch (Exception e) { - logger.error(getSessionMarker(), e.getMessage()); - logger.error("Unreadable: {}", fileName); - } - - logger.info("Valid: {} MC farmes {} FF frames {}", fileName, frames, mi.getFrames()); - } - - return null; - } -} diff --git a/server/-product/production/AMC/jobs/steps/FILEZILLA_AVIDTransferToStep.java b/server/-product/production/AMC/jobs/steps/FILEZILLA_AVIDTransferToStep.java deleted file mode 100644 index 97747098..00000000 --- a/server/-product/production/AMC/jobs/steps/FILEZILLA_AVIDTransferToStep.java +++ /dev/null @@ -1,11 +0,0 @@ -package user.jobengine.server.steps; - -import user.commons.StoreUri; - -public class FILEZILLA_AVIDTransferToStep extends TransferStep { - @Override - @StepEntry - public Object[] execute(StoreUri sourceStoreUri, String sourceFileName, StoreUri targetStoreUri, String targetFileName) throws Exception { - return super.execute(sourceStoreUri, sourceFileName, targetStoreUri, targetFileName); - } -} diff --git a/server/-product/production/AMC/jobs/steps/FILEZILLA_PASARESTORETransferToStep.java b/server/-product/production/AMC/jobs/steps/FILEZILLA_PASARESTORETransferToStep.java deleted file mode 100644 index d69a6ed4..00000000 --- a/server/-product/production/AMC/jobs/steps/FILEZILLA_PASARESTORETransferToStep.java +++ /dev/null @@ -1,11 +0,0 @@ -package user.jobengine.server.steps; - -import user.commons.StoreUri; - -public class FILEZILLA_PASARESTORETransferToStep extends TransferStep { - @Override - @StepEntry - public Object[] execute(StoreUri sourceStoreUri, String sourceFileName, StoreUri targetStoreUri, String targetFileName) throws Exception { - return super.execute(sourceStoreUri, sourceFileName, targetStoreUri, targetFileName); - } -} diff --git a/server/-product/production/AMC/jobs/steps/FileCleanupStep.java b/server/-product/production/AMC/jobs/steps/FileCleanupStep.java deleted file mode 100644 index 505526cf..00000000 --- a/server/-product/production/AMC/jobs/steps/FileCleanupStep.java +++ /dev/null @@ -1,34 +0,0 @@ -package user.jobengine.server.steps; - -import java.nio.file.Path; -import java.nio.file.Paths; - -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; -import org.apache.logging.log4j.message.Message; -import org.apache.logging.log4j.message.ParameterizedMessage; - -import user.commons.StoreUri; - -public class FileCleanupStep extends JobStep { - private static final Logger logger = LogManager.getLogger(); - - @StepEntry - public Object[] execute(StoreUri sourceStoreUri, String fileName, String escortFile) throws Exception { - try { - - Message msg = new ParameterizedMessage("A {} fájl visszatöltése befejeződött.", fileName); - EscortFiles.notifyRecipient(Paths.get(escortFile), logger, msg); - - Path filePath = Paths.get(sourceStoreUri.toString(true), fileName); - EscortFiles.remove(filePath); - EscortFiles.remove(Paths.get(escortFile)); - EscortFiles.remove(Paths.get(escortFile + EscortFiles.DOT_CATCHED)); - } catch (Exception e) { - logger.error(e.getMessage()); - throw e; - } - return null; - } - -} diff --git a/server/-product/production/AMC/jobs/steps/FileSearchFilterOptions.java b/server/-product/production/AMC/jobs/steps/FileSearchFilterOptions.java deleted file mode 100644 index 436c3be0..00000000 --- a/server/-product/production/AMC/jobs/steps/FileSearchFilterOptions.java +++ /dev/null @@ -1,41 +0,0 @@ -package user.jobengine.server.steps; - -import java.nio.file.Path; -import java.util.regex.Matcher; -import java.util.regex.Pattern; - -import com.ibm.nosql.json.api.BasicDBObject; - -public class FileSearchFilterOptions { - - private BasicDBObject filter; - - public FileSearchFilterOptions(BasicDBObject filter) { - this.filter = filter; - } - - public boolean acceptFile(Path file) { - if (filter == null) - return true; - - if (filter.containsKey("fileName")) { - //.*\.(sh|ini|conf|vhost|xml|php)$ - String fileNamePattern = filter.getString("fileName"); - if (fileNamePattern == null || fileNamePattern.trim().length() == 0) - return true; - - Pattern pattern = Pattern.compile(fileNamePattern, Pattern.CASE_INSENSITIVE); - - Matcher matcher = pattern.matcher(file.getFileName().toString()); - if (matcher.find()) - return true; - - } - - return false; - } - - public boolean preAcceptDirectory(Path file) { - return true; - } -} diff --git a/server/-product/production/AMC/jobs/steps/FileValidatorStep.java b/server/-product/production/AMC/jobs/steps/FileValidatorStep.java deleted file mode 100644 index e8662e49..00000000 --- a/server/-product/production/AMC/jobs/steps/FileValidatorStep.java +++ /dev/null @@ -1,155 +0,0 @@ -package user.jobengine.server.steps; - -import java.io.FileNotFoundException; -import java.nio.file.Path; -import java.nio.file.Paths; - -import org.apache.commons.lang.StringUtils; -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; -import org.apache.logging.log4j.message.Message; -import org.apache.logging.log4j.message.ParameterizedMessage; - -import user.commons.DownloadableMedia; -import user.commons.StoreUri; -import user.commons.mediatool.MediaInfo; -import user.commons.remotestore.RemoteStoreProtocol; -import user.jobengine.db.Media; - -public class FileValidatorStep extends JobStep { - private static final String COLORSPACE = "COLORSPACE"; - private static final Logger logger = LogManager.getLogger(); - private static final String testSimulate = System.getProperty("test.simulate"); - - @StepEntry - public Object[] execute(StoreUri sourceStoreUri, String fileName, long expectedFrameNumber, long expectedSize, String exceptedColorSpace, String escortFile) - throws Exception { - if (!RemoteStoreProtocol.LOCAL.equals(sourceStoreUri.getProtocol())) - throw new Exception("Store URI protocol must be local."); - Path filePath = Paths.get(sourceStoreUri.toString(true), fileName); - if (!filePath.toFile().exists()) - throw new FileNotFoundException("File not exists: " + filePath.toString()); - - Path escortFilePath = Paths.get(escortFile); - - if (getJobRuntime().isWaitingCancel()) { - EscortFiles.remove(filePath); - //EscortFiles.remove(escortFilePath); - EscortFiles.remove(Paths.get(escortFile + EscortFiles.DOT_CATCHED)); - - Message msg = new ParameterizedMessage("A {} fájl visszatöltése kezelői beavatkozás miatt megszakadt.", fileName); - EscortFiles.notifyRecipient(escortFilePath, logger, msg); - - cancel(); - return null; - } - - DownloadableMedia downloadable = null; - try { - downloadable = EscortFiles.decode(escortFilePath); - } catch (Exception e) { - logger.error(getSessionMarker(), e.getMessage()); - } - - if (downloadable != null && expectedFrameNumber == 0) - storeCurrentFrames(filePath, downloadable); - - if (downloadable != null && downloadable.containsKey("skipValidation") && downloadable.getBoolean("skipValidation")) - return null; - - if (expectedFrameNumber > 0 || StringUtils.isNotBlank(exceptedColorSpace)) - validateMedia(filePath, expectedFrameNumber, exceptedColorSpace, escortFile); - - if (expectedSize > 0) - validateSize(filePath, expectedSize, escortFile); - - //A metadata persister miatt torlunk mindet teszt modban - /* - if (StringUtils.isNotBlank(testSimulate)) { - EscortFiles.remove(escortFilePath); - EscortFiles.remove(Paths.get(escortFile + EscortFiles.DOT_CATCHED)); - logger.info(getMarker(), "{} deleted to prevent real processing", escortFile); - } - */ - logger.info(getMarker(), "{} passed validation", fileName); - return null; - } - - private void storeCurrentFrames(Path filePath, DownloadableMedia downloadable) { - MediaInfo mi = null; - try { - mi = new MediaInfo(filePath); - mi.process(); - long frames = mi.getFrames(); - - if (downloadable.containsKey("mediaId")) { - Media media = getManager().getMedia(downloadable.getLong("mediaId")); - if (media != null) { - media.setLength(frames); - getManager().modify(media); - } - - } - - } catch (Exception e) { - logger.error(getSessionMarker(), e.getMessage()); - } - } - - private void validateMedia(Path filePath, long expectedFrameNumber, String exceptedColorSpace, String escortFile) throws Exception { - MediaInfo mi = null; - try { - mi = new MediaInfo(filePath); - mi.process(); - - if (expectedFrameNumber > 0) { - long frames = mi.getFrames(); - if (Math.abs(frames - expectedFrameNumber) > 3) { - Message msg = new ParameterizedMessage("Validációs hiba miatt a {} fájl másolása nem lehetséges.", filePath.getFileName()); - EscortFiles.notifyRecipient(Paths.get(escortFile), logger, msg); - - //200924 komment, mert a mar TSM-bol letoltott mozikat hagyjuk ott, a download irany ugyis felulir - //EscortFiles.remove(filePath); - //EscortFiles.remove(Paths.get(escortFile)); - - //200911 a kommentet kivettem, mert ezeket is torolni kell - EscortFiles.remove(Paths.get(escortFile + EscortFiles.DOT_CATCHED)); - throw new Exception("Length mismatch in " + filePath + ". Expected: " + expectedFrameNumber + ", found: " + frames); - } - if (frames != expectedFrameNumber) - logger.warn(getSessionMarker(), "A {} fájl másolása lehetséges, de a képkockaszám nem egyezik. Várt: {}, aktuális: {}", - expectedFrameNumber, frames); - } - - if (StringUtils.isNotBlank(exceptedColorSpace)) { - String colorSpace = mi.getColorSpace(); - if (colorSpace == null) { - EscortFiles.createFellow(escortFile, COLORSPACE); - throw new Exception("Can not determine color space in " + filePath); - } - - if (!colorSpace.equals(exceptedColorSpace)) { - EscortFiles.createFellow(escortFile, COLORSPACE); - throw new Exception("Expected color space is " + exceptedColorSpace + ", but found " + colorSpace); - } - } - - } catch (Exception e) { - logger.error(getSessionMarker(), e.getMessage()); - throw e; - } - } - - private void validateSize(Path filePath, long expectedSize, String escortFile) throws Exception { - try { - long length = filePath.toFile().length(); - if (length != expectedSize) - throw new Exception("Size mismatch. Expected: " + expectedSize + ", found: " + length); - - } catch (Exception e) { - logger.error(getSessionMarker(), e.getMessage()); - throw e; - } - } - -} diff --git a/server/-product/production/AMC/jobs/steps/ForkDownloadStep.java b/server/-product/production/AMC/jobs/steps/ForkDownloadStep.java deleted file mode 100644 index f9715161..00000000 --- a/server/-product/production/AMC/jobs/steps/ForkDownloadStep.java +++ /dev/null @@ -1,199 +0,0 @@ -package user.jobengine.server.steps; - -import java.io.IOException; -import java.nio.file.DirectoryStream; -import java.nio.file.Files; -import java.nio.file.Path; -import java.nio.file.Paths; -import java.util.ArrayList; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.Set; - -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; - -import user.commons.DownloadableMedia; -import user.commons.JobStatus; -import user.commons.MediaCubeMarker; -import user.commons.StoreUri; -import user.commons.remotestore.RemoteStoreProtocol; -import user.jobengine.db.Store; -import user.jobengine.server.IJobEngine; -import user.jobengine.server.IJobRuntime; - -public class ForkDownloadStep extends JobStep { - private static final Logger logger = LogManager.getLogger(); - - @StepEntry - public Object[] execute(String tempStoreName, String template, String expectedColorSpace, int limit) throws Exception { - -// sendStatusReport(); - //return null; - DirectoryStream directoryStream = null; - int count = limit; - int allCount = 0; - int currentCount = 0; - - try { - StoreUri tempStoreUri = getManager().getStoreUri(tempStoreName, RemoteStoreProtocol.LOCAL); - Path path = Paths.get(tempStoreUri.toString(true), ".STATUS"); - - List escortFiles = new ArrayList<>(); - directoryStream = Files.newDirectoryStream(path); - for (Path escortFile : directoryStream) { - if (getJobRuntime().isWaitingCancel()) { - cancel(); - return null; - } - escortFiles.add(escortFile); - } - allCount = escortFiles.size(); - escortFiles.sort((p1, p2) -> { - return Long.valueOf(p1.toFile().lastModified()).compareTo(p2.toFile().lastModified()); - }); - - if (allCount > 0) { - for (Path escortFile : escortFiles) { - if (getJobRuntime().isWaitingCancel()) { - cancel(); - break; - } - - if (processPathItem(tempStoreUri, template, escortFile)) { - count--; - if (count == 0) - break; - } - currentCount++; - int progress = currentCount * 100 / allCount; - setProgress(progress); - } - } - - setProgress(100); - } catch (Exception e) { - logger.error(getSessionMarker(), e.getMessage()); - throw e; - } finally { - if (directoryStream != null) { - try { - directoryStream.close(); - } catch (IOException e) { - } - } - } - - return null; - } - - private boolean processPathItem(StoreUri tempStoreUri, String template, Path escortFile) throws IOException { - if (escortFile.toFile().isDirectory()) - return false; - - //ami nem JSON azt nem kezeljuk - if (!escortFile.getFileName().toString().toLowerCase().endsWith(EscortFiles.DOT_JSON)) - return false; - - //aminek van COLORSPACE kiterjesztesu valtozata azt nem kezeljuk - // Path colorspaceFile = Paths.get(escortFile.toString() + ".colorspace"); - // if (colorspaceFile.toFile().exists()) - // return false; - - if (EscortFiles.isCatchedFileExists(escortFile)) - return false; - - String escortFileName = escortFile.getFileName().toString(); - //minimum 3 karakter: STORENAME.FILENAME.MXF.JSON - long dotCount = escortFileName.chars().filter(ch -> ch == '.').count(); - if (dotCount < 3) - return false; - - try { - DownloadableMedia downloadable = EscortFiles.decode(escortFile); - if (downloadable == null) { - logger.error(getSessionMarker(), "Can't encode {}", escortFile); - return false; - } - - StoreUri sourceStoreUri = getManager().getStoreUri(downloadable.getLong("sourceStoreUriId")); - StoreUri targetStoreUri = getManager().getStoreUri(downloadable.getLong("targetStoreUriId")); - - Store sourceStore = getManager().getStore(sourceStoreUri.getStoreId()); - String sourceStoreName = sourceStore.getName(); - template = sourceStoreName + "-" + template; - logger.info(getMarker(), "Starting {} for {}", template, escortFileName); - - String sourceFileName = downloadable.getString("fileName"); - Map parameters = new HashMap<>(); - parameters.put("sourceStoreUri", sourceStoreUri); - parameters.put("fileName", sourceFileName); - parameters.put("tempStoreUri", tempStoreUri); - parameters.put("targetStoreUri", targetStoreUri); - parameters.put("expectedFrameNumber", downloadable.getLong("frames")); - parameters.put("expectedSize", downloadable.getLong("size")); - parameters.put("expectedColorSpace", downloadable.getString("colorSpace")); - parameters.put("escortFile", escortFile.toAbsolutePath().toString()); - - int priority = 50; - - //megszakitasok miatt a mar lejott mozikat vegyuk elore 20.09.09 - if (targetExists(tempStoreUri, sourceFileName)) - priority = 51; - - //parameterek: IJobRuntime parent, String template, String name, int priority, Map parameters - IJobRuntime child = getEngine().submit(null, e -> { - if (e.getStatus().equals(JobStatus.CANCELED) || e.getStatus().equals(JobStatus.SUSPENDED)) - EscortFiles.removeCatchedFile(escortFile); - }, template, "Archiválás", priority, IJobEngine.DEFAULT_OWNER, parameters); - if (child == null) - throw new Exception("Submit error."); - - child.setDescription(sourceFileName); - - child.setRelated(downloadable.getString("title") + " / " + sourceStoreName); - - EscortFiles.createCatchedFile(escortFile); - - } catch (Exception e) { - logger.error(getSessionMarker(), e.getMessage()); - return false; - } - - return true; - } - - private void sendStatusReport() { - IJobEngine engine = getEngine(); - int jobCount = 0; - StringBuilder sb = new StringBuilder(); - Map jobs = engine.getJobs(); - if (jobs != null) { - Set keys = jobs.keySet(); - jobCount = keys.size(); - for (Long key : keys) { - IJobRuntime runtime = jobs.get(key); - if (JobStatus.SUSPENDED.equals(runtime.getStatus())) - sb.append(runtime.getRelated() + " SUSPENDED: " + runtime.getDescription() + "
"); - } - } - MediaCubeMarker marker = new MediaCubeMarker(); - marker.setSubject("AMC MediaCube feldolgozási sor " + jobCount); - logger.info(marker, sb.toString()); - } - - private boolean targetExists(StoreUri storeUri, String fileName) { - try { - Path targetFile = Paths.get(storeUri.toString(true), fileName); - if (targetFile.toFile().exists()) { - logger.info(getMarker(), "Retrieved file {} exists", fileName); - return true; - } - } catch (Exception e) { - logger.error(getMarker(), e.getMessage()); - } - logger.info(getMarker(), "Retrieved file {} not exists yet", fileName); - return false; - } -} diff --git a/server/-product/production/AMC/jobs/steps/ForkUploadStep.java b/server/-product/production/AMC/jobs/steps/ForkUploadStep.java deleted file mode 100644 index 67010d4c..00000000 --- a/server/-product/production/AMC/jobs/steps/ForkUploadStep.java +++ /dev/null @@ -1,214 +0,0 @@ -package user.jobengine.server.steps; - -import java.io.IOException; -import java.nio.file.DirectoryStream; -import java.nio.file.Files; -import java.nio.file.Path; -import java.nio.file.Paths; -import java.util.ArrayList; -import java.util.HashMap; -import java.util.List; -import java.util.Map; - -import org.apache.commons.lang.StringUtils; -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; - -import user.commons.DownloadableMedia; -import user.commons.JobStatus; -import user.commons.StoreUri; -import user.commons.remotestore.RemoteStoreProtocol; -import user.jobengine.db.Media; -import user.jobengine.db.MediaFile; -import user.jobengine.db.Store; -import user.jobengine.server.IJobEngine; -import user.jobengine.server.IJobRuntime; - -public class ForkUploadStep extends JobStep { - private static final Logger logger = LogManager.getLogger(); - - @StepEntry - public Object[] execute(String tempStoreName, String template, String expectedColorSpace, int limit) throws Exception { - DirectoryStream directoryStream = null; - int count = limit; - int allCount = 0; - int currentCount = 0; - - try { - StoreUri tempStoreUri = getManager().getStoreUri(tempStoreName, RemoteStoreProtocol.LOCAL); - Path path = Paths.get(tempStoreUri.toString(true), ".STATUS"); - - List escortFiles = new ArrayList<>(); - directoryStream = Files.newDirectoryStream(path); - for (Path escortFile : directoryStream) { - if (getJobRuntime().isWaitingCancel()) { - cancel(); - return null; - } - escortFiles.add(escortFile); - } - allCount = escortFiles.size(); - //a file neve az adasbakerules idopontjaval kezdodik - // escortFiles.sort((p1, p2) -> { - // return p1.compareTo(p2); - // }); - - //200911 megis inkabb a letrehozas idopontja legyen - escortFiles.sort((p1, p2) -> { - return Long.valueOf(p1.toFile().lastModified()).compareTo(p2.toFile().lastModified()); - }); - - if (allCount > 0) { - for (Path escortFile : escortFiles) { - if (getJobRuntime().isWaitingCancel()) { - cancel(); - break; - } - - if (processPathItem(tempStoreUri, template, escortFile)) { - count--; - if (count == 0) - break; - } - currentCount++; - int progress = currentCount * 100 / allCount; - setProgress(progress); - } - } - - setProgress(100); - } catch (Exception e) { - logger.error(getSessionMarker(), e.getMessage()); - throw e; - } finally { - if (directoryStream != null) { - try { - directoryStream.close(); - } catch (IOException e) { - } - } - } - - return null; - } - - private String getTSMFileName(long mediaId, StoreUri sourceStoreUri) { - String result = null; - Media media = getManager().getMedia(mediaId); - - if (media != null) { - List mediaFiles = media.getMediaFiles(); - for (MediaFile mediaFile : mediaFiles) { - if (mediaFile.getStoreId() == sourceStoreUri.getStoreId()) { - result = mediaFile.getRelativePath(); - break; - } - } - } - - return result; - } - - private boolean processPathItem(StoreUri tempStoreUri, String template, Path escortFile) throws IOException { - if (escortFile.toFile().isDirectory()) - return false; - - //ami nem JSON azt nem kezeljuk - if (!escortFile.getFileName().toString().toLowerCase().endsWith(EscortFiles.DOT_JSON)) - return false; - - //aminek van COLORSPACE kiterjesztésű - // Path colorspaceFile = Paths.get(escortFile.toString() + ".colorspace"); - // if (colorspaceFile.toFile().exists()) - // return false; - - if (EscortFiles.isCatchedFileExists(escortFile)) - return false; - - String escortFileName = escortFile.getFileName().toString(); - //minimum 3 karakter: STORENAME.FILENAME.MXF.JSON - long dotCount = escortFileName.chars().filter(ch -> ch == '.').count(); - if (dotCount < 3) - return false; - - try { - DownloadableMedia downloadable = EscortFiles.decode(escortFile); - - if (downloadable == null) { - logger.error(getSessionMarker(), "Can't decode {}", escortFile); - return false; - } - - StoreUri sourceStoreUri = getManager().getStoreUri("TSM", RemoteStoreProtocol.TSM); - - long mediaId = downloadable.getLong("mediaId"); - String tsmFileName = getTSMFileName(mediaId, sourceStoreUri); - if (StringUtils.isBlank(tsmFileName)) { - logger.error(getMarker(), "Media ID not exists anymore {}", mediaId); - return false; - } - - StoreUri targetStoreUri = getManager().getStoreUri(downloadable.getLong("targetStoreUriId")); - - Store targetStore = getManager().getStore(targetStoreUri.getStoreId()); - template = targetStore.getName() + "-" + template; - logger.info(getMarker(), "Starting {} for {}", template, escortFileName); - - String targetFileName = downloadable.getString("fileName"); - - Map parameters = new HashMap<>(); - parameters.put("sourceStoreUri", sourceStoreUri); - parameters.put("tsmFileName", tsmFileName); - parameters.put("fileName", targetFileName); - parameters.put("tempStoreUri", tempStoreUri); - parameters.put("targetStoreUri", targetStoreUri); - parameters.put("relativeTargetPath", downloadable.getString("relativeTargetPath")); - parameters.put("expectedFrameNumber", downloadable.getLong("frames")); - parameters.put("expectedSize", downloadable.getLong("size")); - parameters.put("expectedColorSpace", downloadable.getString("colorSpace")); - parameters.put("escortFile", escortFile.toAbsolutePath().toString()); - - //int priority = downloadable.containsKey("priority") ? downloadable.getInt("priority") : 0; - //mindenki egyforman induljon 20.09.07 - //a NEXIO egy kicsit fontosabb 20.09.08. megsem 20.09.09 - //int priority = escortFile.toString().contains("NEXIO") ? 51 : 50; - //megszakitasok miatt a mar lejott mozikat vegyuk elore 20.09.09 - - int priority = 50; - if (targetExists(tempStoreUri, targetFileName)) - priority = 51; - - //parameterek: IJobRuntime parent, String template, String name, int priority, Map parameters - IJobRuntime child = getEngine().submit(null, e -> { - if (e.getStatus().equals(JobStatus.CANCELED) || e.getStatus().equals(JobStatus.SUSPENDED)) - EscortFiles.removeCatchedFile(escortFile); - }, template, "Visszatöltés", priority, IJobEngine.DEFAULT_OWNER, parameters); - if (child == null) - throw new Exception("Submit error."); - - child.setDescription(escortFileName); - - child.setRelated(downloadable.getString("title") + " / " + targetStore.getName()); - EscortFiles.createCatchedFile(escortFile); - } catch (Exception e) { - logger.error(getSessionMarker(), e.getMessage()); - return false; - } - - return true; - } - - private boolean targetExists(StoreUri storeUri, String fileName) { - try { - Path targetFile = Paths.get(storeUri.toString(true), fileName); - if (targetFile.toFile().exists()) { - logger.info(getMarker(), "Retrieved file {} exists", fileName); - return true; - } - } catch (Exception e) { - logger.error(getMarker(), e.getMessage()); - } - logger.info(getMarker(), "Retrieved file {} not exists yet", fileName); - return false; - } -} diff --git a/server/-product/production/AMC/jobs/steps/GenericArchiveCheckerStep.java b/server/-product/production/AMC/jobs/steps/GenericArchiveCheckerStep.java deleted file mode 100644 index 1e6078ec..00000000 --- a/server/-product/production/AMC/jobs/steps/GenericArchiveCheckerStep.java +++ /dev/null @@ -1,93 +0,0 @@ -package user.jobengine.server.steps; - -import java.io.IOException; -import java.nio.file.Paths; -import java.sql.Timestamp; -import java.util.List; - -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; - -import user.commons.DownloadableMedia; -import user.commons.RemoteFile; -import user.commons.StoreUri; -import user.commons.remotestore.RemoteStoreProtocol; -import user.jobengine.db.Media; - -public class GenericArchiveCheckerStep extends JobStep { - private static final String DOT_MXF = ".MXF"; - private static final Logger logger = LogManager.getLogger(); - - private boolean check(Timestamp modified, Timestamp created, Media media) { - boolean create = media == null; - create |= create || !media.getModified().equals(modified); - create |= create || !media.getCreated().equals(created); - return create; - } - - @StepEntry - public Object[] execute(String sourceStoreName, String protocol, String wildCard, String escortStoreName, int limit) throws Exception { - StoreUri sourceStoreUri = null; - int count = limit; - - try { - sourceStoreUri = getManager().getStoreUri(sourceStoreName, Enum.valueOf(RemoteStoreProtocol.class, protocol)); - StoreUri escortSoreUri = getManager().getStoreUri(escortStoreName, RemoteStoreProtocol.LOCAL); - StoreUri targetStoreUri = getManager().getStoreUri("TSM", RemoteStoreProtocol.TSM); - - String outputPath = Paths.get(escortSoreUri.toString(true)).toString(); - - //az FTP store nem tamogatja az asszinkron listazast - List remoteFiles = sourceStoreUri.getRemoteFiles(wildCard); - for (RemoteFile remoteFile : remoteFiles) { - if (processClip(sourceStoreName, remoteFile, sourceStoreUri, targetStoreUri, outputPath)) - count--; - - if (count == 0) { - logger.info("Limit reached: {}", limit); - break; - } - - } - } catch (Exception e) { - throw e; - } finally { - if (sourceStoreUri != null) - sourceStoreUri.cleanUp(); - } - - return null; - } - - private boolean processClip(String storeName, RemoteFile remoteFile, StoreUri sourceStoreUri, StoreUri targetStoreUri, String outputPath) { - boolean result = false; - String title = remoteFile.getName(); - Timestamp modified = Timestamp.from(remoteFile.getModify().toInstant()); - Timestamp created = Timestamp.from(remoteFile.getCreated().toInstant()); - - //TODO a masolas alatt allo klippeket kiszurni - if (title.contains(".")) - title = title.substring(0, title.indexOf(".")); - long size = remoteFile.getSize(); - - Media media = getManager().getMedia(title); - - boolean createEscort = check(modified, created, media); - - if (createEscort) { - long mediaId = media == null ? 0 : media.getId(); - String fileName = title + DOT_MXF; - DownloadableMedia archive = DownloadableMedia.create(title, fileName, modified, created, 0, size, sourceStoreUri.getId(), targetStoreUri.getId(), - mediaId); - String escortFileName = storeName + "." + remoteFile.getName(); - try { - result = EscortFiles.createMetadataIfNotExists(outputPath, escortFileName, archive.toPrettyString("")); - } catch (IOException e) { - logger.error("Escort file creation error.", e); - } - } - - return result; - } - -} diff --git a/server/-product/production/AMC/jobs/steps/HarrisMissingMaterialCheckerStep.java b/server/-product/production/AMC/jobs/steps/HarrisMissingMaterialCheckerStep.java deleted file mode 100644 index 9f683503..00000000 --- a/server/-product/production/AMC/jobs/steps/HarrisMissingMaterialCheckerStep.java +++ /dev/null @@ -1,171 +0,0 @@ -package user.jobengine.server.steps; - -import java.io.IOException; -import java.nio.file.Path; -import java.nio.file.Paths; -import java.util.Arrays; -import java.util.List; - -import org.apache.commons.lang.StringUtils; -import org.apache.commons.net.ftp.FTPClient; -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; - -import com.ibm.nosql.json.api.BasicDBList; - -import user.commons.DownloadableMedia; -import user.commons.StoreUri; -import user.commons.harris.HarrisRecord; -import user.commons.harris.VICFileParser; -import user.commons.remotestore.FtpDirectoryLister; -import user.commons.remotestore.RemoteStoreProtocol; -import user.jobengine.db.Media; - -public class HarrisMissingMaterialCheckerStep extends JobStep { - private static final Logger logger = LogManager.getLogger(HarrisMissingMaterialCheckerStep.class); - private static final String lineFormat = System.getProperty("harris.vic.lineformat", - "A TTTTTTTT LLLLLLLLLLL MMMMMMMMMMMMMMMMMMMMMMM X DDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDD"); - private static final String validTypeCodes = System.getProperty("harris.vic.validtypecodes", "SPACE,E"); - private static final String appendExtension = System.getProperty("missingmaterial.appendextension", ".mxf"); - - @StepEntry - public Object[] execute(BasicDBList vicFiles, String escortStoreName, String targetStoreName, String targetProtocol) throws Exception { - StoreUri escortStoreUri = null; - StoreUri targetStoreUri = null; - try { - getJobRuntime().setRelated(targetStoreName + " restore"); - - escortStoreUri = getManager().getStoreUri(escortStoreName, RemoteStoreProtocol.LOCAL); - targetStoreUri = getManager().getStoreUri(targetStoreName, Enum.valueOf(RemoteStoreProtocol.class, targetProtocol)); - StoreUri sourceStoreUri = getManager().getStoreUri("TSM", RemoteStoreProtocol.TSM); - List records = null; - for (Object vicFile : vicFiles) { - if (getJobRuntime().isWaitingCancel()) { - cancel(); - break; - } - - Path vicFilePath = Paths.get(String.valueOf(vicFile)); - - if (!vicFilePath.toFile().exists()) { - logger.warn("File not exists: {}", vicFilePath); - continue; - } - - logger.info(getMarker(), "Processing {}", vicFilePath); - records = processVICFile(records, vicFilePath); - } - - if (records != null && !getJobRuntime().isWaitingCancel()) { - FTPClient client = ((FtpDirectoryLister) targetStoreUri.getLister()).connect(); - logger.info(getMarker(), "Reading target file list"); - List fileNames = Arrays.asList(client.listNames()); - logger.info(getMarker(), "Target file list reading completed, mediabase size is {}", fileNames.size()); - int i = 0; - for (HarrisRecord record : records) { - - if (getJobRuntime().isWaitingCancel()) { - cancel(); - break; - } - - boolean contains = fileNames.contains(record.getFileName() + ".lxf"); - if (contains) - logger.info(getMarker(), "File {} already exists on {}", record.getFileName(), targetStoreName); - else - processRecord(record, targetStoreName, sourceStoreUri, targetStoreUri, escortStoreUri); - i++; - int progress = i * 100 / records.size(); - setProgress(progress); - } - } - - if (records == null || records.size() == 0) - logger.info(getMarker(), "Nothing to do"); - else - logger.info(getMarker(), "Completed"); - - setProgress(100); - } catch (Exception e) { - logger.error(e.getMessage()); - throw e; - } finally { - if (escortStoreUri != null) - escortStoreUri.cleanUp(); - if (targetStoreUri != null) - targetStoreUri.cleanUp(); - } - - return null; - } - - private boolean processRecord(HarrisRecord record, String targetStoreName, StoreUri sourceStoreUri, StoreUri targetStoreUri, StoreUri escortStoreUri) - throws Exception { - - String fileName = record.getFileName(); - String outputPath = Paths.get(escortStoreUri.toString(true)).toString(); - - if (StringUtils.isNotBlank(appendExtension)) - fileName += appendExtension; - - String escortFileName = targetStoreName + "." + fileName; - - //ha mar letezik, nem toltjuk fel ujra (a vic fajlbol kezzel torlik valamikor...) - try { - if (EscortFiles.isMetadataExists(outputPath, escortFileName)) { - logger.debug(getMarker(), "Status file already exists {}", escortFileName); - return true; - } - } catch (IOException e1) { - logger.info(getMarker(), "Status file check error for {}", fileName); - return true; - } - - //nincs bent MC-ben - Media media = getManager().getMedia(record.getFileName()); - if (media == null) { - logger.error(getSessionMarker(), "File is not available in archive {}", record.getFileName()); - return true; - } - - // if (existsOnNexio(record, targetStoreName, targetStoreUri)) { - // logger.info(getMarker(), "File {} already exists on {}", record.getFileName(), targetStoreName); - // return false; - // } - - DownloadableMedia downloadable = DownloadableMedia.create(media.getTitle(), fileName, media.getModified(), media.getCreated(), media.getLength(), 0L, - sourceStoreUri.getId(), targetStoreUri.getId(), media.getId()); - - //alapbol magasabb (1-99) - downloadable.put("priority", 50); - downloadable.put("isNexio", true); - - try { - EscortFiles.createMetadata(outputPath, escortFileName, downloadable.toPrettyString("")); - logger.info(getMarker(), "Status file created {}", escortFileName); - } catch (Exception e) { - logger.error("Can't create escort file {}. System message is: {}", escortFileName, e.getMessage()); - } - return true; - } - - private List processVICFile(List records, Path vicFilePath) { - VICFileParser parser = null; - try { - parser = new VICFileParser(vicFilePath, lineFormat, validTypeCodes); - - List currentRecords = parser.getRecords(); - if (records == null) - records = currentRecords; - else { - if (currentRecords != null) - records.addAll(currentRecords); - } - - } catch (Exception e) { - logger.error(e.getMessage()); - } - return records; - } - -} diff --git a/server/-product/production/AMC/jobs/steps/MetadataPersisterStep.java b/server/-product/production/AMC/jobs/steps/MetadataPersisterStep.java deleted file mode 100644 index 43fd68c9..00000000 --- a/server/-product/production/AMC/jobs/steps/MetadataPersisterStep.java +++ /dev/null @@ -1,110 +0,0 @@ -package user.jobengine.server.steps; - -import java.nio.file.Path; -import java.nio.file.Paths; -import java.sql.Timestamp; -import java.time.Instant; - -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; - -import user.commons.DownloadableMedia; -import user.commons.StoreUri; -import user.jobengine.db.IItemManager; -import user.jobengine.db.Item; -import user.jobengine.db.Media; -import user.jobengine.db.MediaFile; - -public class MetadataPersisterStep extends JobStep { - private static final String GENERIC = "Generic"; - private static final Logger logger = LogManager.getLogger(); - - @StepEntry - public Object[] execute(StoreUri targetStoreUri, String escortFile) throws Exception { - try { - - Path escortFilePath = Paths.get(escortFile); - //ha nem ment at a letoltott file az ellenorzesen, akkor már torolve lett - if (escortFilePath.toFile().exists()) { - String escortFileName = escortFilePath.getFileName().toString(); - String sourceStoreName = escortFileName.substring(0, escortFileName.indexOf(".")); - DownloadableMedia downloadable = EscortFiles.decode(escortFilePath); - String title = downloadable.getString("title"); - String fileName = downloadable.getString("fileName"); - long mediaId = downloadable.containsKey("mediaId") ? downloadable.getLong("mediaId") : 0; - String fileTypeName = fileName.substring(fileName.lastIndexOf(".") + 1).toUpperCase(); - - IItemManager manager = getManager(); - - if (mediaId == 0) { - mediaId = tryGetMediaId(title, mediaId); - } - - //TODO ha van mediaId, akkor update!!!!!!!!! - - Timestamp modified = downloadable.getTimestamp("modified"); - Timestamp created = downloadable.getTimestamp("created"); - long frames = downloadable.getLong("frames"); - - if (mediaId == 0) { - String targetStoreName = getManager().getStore(targetStoreUri.getStoreId()).getName(); - //ez elszallhat, mert nincs filetype vagy store - MediaFile mediaFile = manager.createMediaFile(fileName, fileTypeName, targetStoreName); - //ez elszallhat, mert nincs itemtype - Item item = manager.createItem(GENERIC, title, sourceStoreName, null); - //ez elszallhat, mert nincs itemtype - Media media = manager.createMedia(GENERIC, title, sourceStoreName, null); - - //TODO megoldani egy tranzakcioban - item.add(); - media.setItemId(item.getId()); - media.setLength(frames); - media.setCreated(created); - media.setModified(modified); - media.setArchived(Timestamp.from(Instant.now())); - media.add(); - - //TODO - //mediaFile.setFileSize(fileSize); - mediaFile.setMediaId(media.getId()); - mediaFile.setLastModified(modified); - mediaFile.add(); - } else { - Media media = manager.getMedia(mediaId); - media.setLength(frames); - media.setCreated(created); - media.setModified(modified); - media.setArchived(Timestamp.from(Instant.now())); - manager.modify(media); - - //TODO - //mediaFile.setFileSize(fileSize); - MediaFile mediaFile = manager.getSystemMediaFile(media); - mediaFile.setRelativePath(fileName); - mediaFile.setLastModified(modified); - manager.modify(mediaFile); - } - - } else - logger.error(getMarker(), "Status file does not exists {}", escortFilePath); - } catch (Exception e) { - logger.error(getSessionMarker(), e.getMessage()); - throw e; - } - return null; - } - - private long tryGetMediaId(String title, long mediaId) { - Media media = null; - //duplikatum miatt - try { - media = getManager().getMedia(title); - if (media != null) - mediaId = media.getId(); - } catch (Exception e) { - logger.error(getMarker(), "Media with title {} is not available. System message is: {}", title, e.getMessage()); - } - return mediaId; - } - -} diff --git a/server/-product/production/AMC/jobs/steps/NEXIO1TransferFromStep.java b/server/-product/production/AMC/jobs/steps/NEXIO1TransferFromStep.java deleted file mode 100644 index 7963459f..00000000 --- a/server/-product/production/AMC/jobs/steps/NEXIO1TransferFromStep.java +++ /dev/null @@ -1,30 +0,0 @@ -package user.jobengine.server.steps; - -import java.nio.file.Path; -import java.nio.file.Paths; - -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; - -import user.commons.StoreUri; - -public class NEXIO1TransferFromStep extends TransferStep { - private static final Logger logger = LogManager.getLogger(); - - @Override - @StepEntry - public Object[] execute(StoreUri sourceStoreUri, String sourceFileName, StoreUri targetStoreUri, String targetFileName) throws Exception { - Object[] result = null; - try { - Path targetFile = Paths.get(targetStoreUri.toString(true), targetFileName); - if (targetFile.toFile().exists()) { - logger.info(getMarker(), "Using previously retrived file {}", targetFile); - } else { - result = super.execute(sourceStoreUri, sourceFileName, targetStoreUri, targetFileName); - } - } catch (Exception e) { - logger.error(getMarker(), e.getMessage()); - } - return result; - } -} diff --git a/server/-product/production/AMC/jobs/steps/NEXIO1TransferToStep.java b/server/-product/production/AMC/jobs/steps/NEXIO1TransferToStep.java deleted file mode 100644 index 5af53df1..00000000 --- a/server/-product/production/AMC/jobs/steps/NEXIO1TransferToStep.java +++ /dev/null @@ -1,54 +0,0 @@ -package user.jobengine.server.steps; - -import java.util.Arrays; -import java.util.List; - -import org.apache.commons.net.ftp.FTPClient; -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; - -import user.commons.StoreUri; -import user.commons.remotestore.FtpDirectoryLister; - -public class NEXIO1TransferToStep extends TransferStep { - private static final Logger logger = LogManager.getLogger(); - - @Override - @StepEntry - public Object[] execute(StoreUri sourceStoreUri, String sourceFileName, StoreUri targetStoreUri, String targetFileName) throws Exception { - // - // try { - // String title = targetFileName; - // if (title.contains(".")) - // title = title.substring(0, title.lastIndexOf(".")); - // RemoteFile remoteFile = targetStoreUri.getFileWithContent(title + ".xml"); - // if (remoteFile != null && remoteFile.getContent() != null) { - // logger.info("File already exists on NEXIO1 {}", targetFileName); - // return null; - // } - // } catch (Exception e) { - // logger.error(e.getMessage()); - // } finally { - // targetStoreUri.cleanUp(); - // } -/* - try { - FTPClient client = ((FtpDirectoryLister) targetStoreUri.getLister()).connect(); - logger.info(getMarker(), "Reading target file list"); - List fileNames = Arrays.asList(client.listNames()); - logger.info(getMarker(), "Target file list reading completed, mediabase size is {}", fileNames.size()); - String title = targetFileName.replace(".mxf", ".lxf"); - boolean contains = fileNames.contains(title); - if (contains) { - logger.info(getMarker(), "File already exists {} on {}", title, getManager().getStore(targetStoreUri.getStoreId()).getName()); - return null; - } - } catch (Exception e) { - logger.catching(e); - } finally { - targetStoreUri.cleanUp(); - } -*/ - return super.execute(sourceStoreUri, sourceFileName, targetStoreUri, targetFileName); - } -} diff --git a/server/-product/production/AMC/jobs/steps/NEXIO2TransferFromStep.java b/server/-product/production/AMC/jobs/steps/NEXIO2TransferFromStep.java deleted file mode 100644 index a698c6e8..00000000 --- a/server/-product/production/AMC/jobs/steps/NEXIO2TransferFromStep.java +++ /dev/null @@ -1,30 +0,0 @@ -package user.jobengine.server.steps; - -import java.nio.file.Path; -import java.nio.file.Paths; - -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; - -import user.commons.StoreUri; - -public class NEXIO2TransferFromStep extends TransferStep { - private static final Logger logger = LogManager.getLogger(); - - @Override - @StepEntry - public Object[] execute(StoreUri sourceStoreUri, String sourceFileName, StoreUri targetStoreUri, String targetFileName) throws Exception { - Object[] result = null; - try { - Path targetFile = Paths.get(targetStoreUri.toString(true), targetFileName); - if (targetFile.toFile().exists()) { - logger.info(getMarker(), "Using previously retrived file {}", targetFile); - } else { - result = super.execute(sourceStoreUri, sourceFileName, targetStoreUri, targetFileName); - } - } catch (Exception e) { - logger.error(getMarker(), e.getMessage()); - } - return result; - } -} diff --git a/server/-product/production/AMC/jobs/steps/NEXIO2TransferToStep.java b/server/-product/production/AMC/jobs/steps/NEXIO2TransferToStep.java deleted file mode 100644 index 20c81a68..00000000 --- a/server/-product/production/AMC/jobs/steps/NEXIO2TransferToStep.java +++ /dev/null @@ -1,53 +0,0 @@ -package user.jobengine.server.steps; - -import java.util.Arrays; -import java.util.List; - -import org.apache.commons.net.ftp.FTPClient; -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; - -import user.commons.StoreUri; -import user.commons.remotestore.FtpDirectoryLister; - -public class NEXIO2TransferToStep extends TransferStep { - private static final Logger logger = LogManager.getLogger(); - - @Override - @StepEntry - public Object[] execute(StoreUri sourceStoreUri, String sourceFileName, StoreUri targetStoreUri, String targetFileName) throws Exception { - // try { - // String title = targetFileName; - // if (title.contains(".")) - // title = title.substring(0, title.lastIndexOf(".")); - // RemoteFile remoteFile = targetStoreUri.getFileWithContent(title + ".xml"); - // if (remoteFile != null && remoteFile.getContent() != null) { - // logger.info("File already exists on NEXIO2 {}", targetFileName); - // return null; - // } - // } catch (Exception e) { - // logger.error(e.getMessage()); - // } finally { - // targetStoreUri.cleanUp(); - // } -/* - try { - FTPClient client = ((FtpDirectoryLister) targetStoreUri.getLister()).connect(); - logger.info(getMarker(), "Reading target file list"); - List fileNames = Arrays.asList(client.listNames()); - logger.info(getMarker(), "Target file list reading completed, mediabase size is {}", fileNames.size()); - String title = targetFileName.replace(".mxf", ".lxf"); - boolean contains = fileNames.contains(title); - if (contains) { - logger.info(getMarker(), "File already exists {} on {}", title, getManager().getStore(targetStoreUri.getStoreId()).getName()); - return null; - } - } catch (Exception e) { - logger.catching(e); - } finally { - targetStoreUri.cleanUp(); - } -*/ - return super.execute(sourceStoreUri, sourceFileName, targetStoreUri, targetFileName); - } -} diff --git a/server/-product/production/AMC/jobs/steps/NEXIOArchiveCheckerStep.java b/server/-product/production/AMC/jobs/steps/NEXIOArchiveCheckerStep.java deleted file mode 100644 index 8beafb23..00000000 --- a/server/-product/production/AMC/jobs/steps/NEXIOArchiveCheckerStep.java +++ /dev/null @@ -1,187 +0,0 @@ -package user.jobengine.server.steps; - -import java.io.IOException; -import java.nio.file.Paths; -import java.sql.Timestamp; -import java.text.SimpleDateFormat; -import java.time.Duration; -import java.time.Instant; -import java.util.Iterator; - -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; - -import user.commons.DownloadableMedia; -import user.commons.StoreUri; -import user.commons.nexio.api.Clip; -import user.commons.nexio.api.ClipNotFoundException; -import user.commons.nexio.api.Controller; -import user.commons.nexio.api.Mediabase; -import user.commons.nexio.server.protocol.ProtocolException; -import user.commons.remotestore.RemoteStoreProtocol; -import user.jobengine.db.Media; -import user.jobengine.db.MediaFile; - -public class NEXIOArchiveCheckerStep extends JobStep { - private static final String DOT_MXF = ".mxf"; - private static final Logger logger = LogManager.getLogger(); - private SimpleDateFormat df = new SimpleDateFormat("yyyyMMdd-HHmmss"); - - private boolean check(Timestamp modified, Timestamp created, int duration, Media media, int videoFormat, int lastModifiedHours) { - String title = media.getTitle(); - - //formatum - if (!(videoFormat == 3 || videoFormat == 19)) { - logger.debug(getMarker(), "{} unsupported video format", videoFormat); - return false; - } - - if (!checkModifiedByTime(title, modified, lastModifiedHours)) - return false; - - //utolso modositas - MediaFile mediaFile = getManager().getSystemMediaFile(media); - boolean isModified = true; - if (mediaFile == null) { - logger.info(getMarker(), "{} system media file not available", title); - return false; - } - - String mcMod = df.format(mediaFile.getLastModified()); - String nexioMod = df.format(modified); - isModified = mcMod.compareTo(nexioMod) < 0; - if (isModified) { - logger.info(getMarker(), "{} modification time: MC {} < NEXIO {}", title, mcMod, nexioMod); - return true; - } - - boolean isLengthChanged = media.getLength() > 0 && Math.abs(media.getLength() - duration) > 3; - if (isLengthChanged) { - logger.info(getMarker(), "{} length: MC {} != NEXIO {}", title, media.getLength(), duration); - return true; - } - - return false; - } - - private boolean checkModifiedByTime(String title, Timestamp modified, int lastModifiedHours) { - //utolso modositas a mosthoz kepest - //The result of this method can be a negative period if the end is before the start. - long modifiedHours = Duration.between(modified.toInstant(), Instant.now()).toHours(); - if (modifiedHours < 0) { - logger.info(getMarker(), "{} modification time greather than now", title); - return false; - } - if (modifiedHours < lastModifiedHours) { - logger.debug(getMarker(), "{} modification time is too cloose for now", title); - return false; - } - - return true; - } - - @StepEntry - public Object[] execute(String sourceStoreName, String escortStoreName, int lastModifiedHours) throws Exception { - Controller controller = null; - - try { - getJobRuntime().setRelated(sourceStoreName + " archive"); - - StoreUri nexioStoreUri = getManager().getStoreUri(sourceStoreName, RemoteStoreProtocol.NEXIO); - StoreUri sourceStoreUri = getManager().getStoreUri(sourceStoreName, RemoteStoreProtocol.FTP); - StoreUri targetStoreUri = getManager().getStoreUri("TSM", RemoteStoreProtocol.TSM); - StoreUri escortStoreUri = getManager().getStoreUri(escortStoreName, RemoteStoreProtocol.LOCAL); - String outputPath = Paths.get(escortStoreUri.toString(true)).toString(); - - logger.info(getMarker(), "Connecting mediabase {}", nexioStoreUri.toString()); - if (nexioStoreUri.getPortNumber() == 0) - controller = new Controller(nexioStoreUri.getUri(), nexioStoreUri.getPortNumber()); - else - controller = new Controller(nexioStoreUri.getUri()); - controller.connect(); - Mediabase mediabase = controller.getMediabase(); - - Iterator clips = mediabase.getClips(); - logger.info(getMarker(), "Processing clips"); - - processClips(sourceStoreName, sourceStoreUri, targetStoreUri, outputPath, clips, lastModifiedHours); - } catch (Exception e) { - logger.error(getSessionMarker(), e.getMessage() == null ? "Unknown error" : e.getMessage()); - throw e; - } finally { - setProgress(100); - if (controller != null) - controller.disconnect(); - } - - return null; - } - - private void processClips(String storeName, StoreUri sourceStoreUri, StoreUri targetStoreUri, String outputPath, Iterator clips, - int lastModifiedHours) throws ClipNotFoundException, IOException, ProtocolException { - int processed = 0; - while (clips.hasNext()) { - - if (getJobRuntime().isWaitingCancel()) { - cancel(); - break; - } - - Clip clip = clips.next(); - - try { - String title = null; - Timestamp created = null; - Timestamp modified = null; - int frames = 0; - int videoFormat = 0; - frames = clip.getDuration(); - - //ures clip - if (frames == 1) - continue; - - title = clip.getXid().get(); - created = Timestamp.from(clip.getRecordDateTimestamp().toInstant()); - modified = Timestamp.from(clip.getModifiedTimestamp().toInstant()); - videoFormat = clip.getVideoFormat(); - - Media media = getManager().getMedia(title); - logger.debug(getMarker(), "Checking {}", title); - boolean createEscort = false; - - long mediaId = 0; - if (media == null) { - if (checkModifiedByTime(title, modified, lastModifiedHours)) { - createEscort = true; - logger.info(getMarker(), "{} not archived yet", title); - } - } else { - mediaId = media.getId(); - createEscort = check(modified, created, frames, media, videoFormat, lastModifiedHours); - } - - if (createEscort && !title.contains("*")) { - String fileName = title + DOT_MXF; - DownloadableMedia downloadable = DownloadableMedia.create(title, fileName, modified, created, frames, 0, sourceStoreUri.getId(), - targetStoreUri.getId(), mediaId); - String escortFileName = storeName + "." + downloadable.getString("fileName"); - if (EscortFiles.createMetadataIfNotExists(outputPath, escortFileName, downloadable.toPrettyString(""))) { - logger.info(getMarker(), "Archive status file created for {}", fileName); - } else { - logger.info(getMarker(), "Archive status file already exists for {}", fileName); - } - - } - - processed++; - if (processed % 100 == 0) - logger.info(getMarker(), "Processed {} records", processed); - } catch (Exception e) { - logger.error(getSessionMarker(), e.getMessage()); - continue; - } - } - } - -} diff --git a/server/-product/production/AMC/jobs/steps/NEXIOMetadataPersisterStep.java b/server/-product/production/AMC/jobs/steps/NEXIOMetadataPersisterStep.java deleted file mode 100644 index ec9565f1..00000000 --- a/server/-product/production/AMC/jobs/steps/NEXIOMetadataPersisterStep.java +++ /dev/null @@ -1,109 +0,0 @@ -package user.jobengine.server.steps; - -import java.nio.file.Path; -import java.nio.file.Paths; -import java.sql.Timestamp; -import java.text.SimpleDateFormat; - -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; -import org.w3c.dom.Document; -import org.w3c.dom.Node; -import org.w3c.dom.NodeList; - -import user.commons.DownloadableMedia; -import user.commons.RemoteFile; -import user.commons.StoreUri; -import user.jobengine.db.Media; -import user.jobengine.db.MediaFile; - -public class NEXIOMetadataPersisterStep extends JobStep { - private static final Logger logger = LogManager.getLogger(); - private static final String DOT_XML = ".xml"; - - @StepEntry - public Object[] execute(StoreUri targetStoreUri, String escortFile) throws Exception { - try { - Path escortFilePath = Paths.get(escortFile); - if (!escortFilePath.toFile().exists()) { - logger.error(getMarker(), "Status file does not exists {}", escortFilePath); - return null; - } - - DownloadableMedia downloadable = EscortFiles.decode(escortFilePath); - if (!downloadable.containsKey("isNexio") || !downloadable.getBoolean("isNexio")) - return null; - - if (!downloadable.containsKey("mediaId")) - throw new NullPointerException("No media ID found for " + downloadable.getString("title")); - - Media media = getManager().getMedia(downloadable.getLong("mediaId")); - MediaFile mediaFile = getManager().getSystemMediaFile(media); - if (mediaFile == null) - throw new NullPointerException("No TSM media file found for " + downloadable.getString("title")); - - RemoteFile remoteFile = targetStoreUri.getFileWithContent(downloadable.getString("title") + ".xml"); - - //TODO mediafile lastmodified = feltoltott file nexio xml lastmodified - Timestamp lastModified = getLastModified(remoteFile); - logger.info("Setting last modification time on {} to {}", downloadable.getString("title"), lastModified); - mediaFile.setLastModified(lastModified); - getManager().modify(mediaFile); - - // if (!RemoteStoreProtocol.FTP.equals(targetStoreUri.getProtocol())) - // return null; - - // - // String title = downloadable.getString("title"); - // String fileName = downloadable.getString("fileName"); - // Date created = downloadable.getDate("created"); - // Date modified = downloadable.getDate("modified"); - // byte[] metadata = EscortFiles.createNEXIODatesMeta(title, created, modified); - // uploadMeta(targetStoreUri, fileName, metadata); - } catch (Exception e) { - logger.error(getSessionMarker(), e.getMessage()); - throw e; - } finally { - if (targetStoreUri != null) - targetStoreUri.cleanUp(); - } - return null; - } - - private Timestamp getLastModified(RemoteFile remoteFile) throws Exception { - Timestamp result = null; - Document document = EscortFiles.createNEXIOMeta(remoteFile.getContent()); - Node firstChild = document.getFirstChild(); - NodeList list = firstChild.getChildNodes(); - for (int i = 0; i < list.getLength(); i++) { - Node item = list.item(i); - if (item.getNodeName().equals("ModifiedTimeStamp")) { - SimpleDateFormat df = new SimpleDateFormat("MM-dd-yyyy (HH:mm:ss)"); - result = Timestamp.from(df.parse(item.getTextContent()).toInstant()); - break; - } - } - return result; - } - - // private void uploadMeta(StoreUri storeUri, String fileName, byte[] metadata) throws Exception { - // logger.info(getMarker(), "Uploading metadata for {}", fileName); - // OutputStream outStream = null; - // try { - // FTPClient sourceFtp = ((FtpDirectoryLister) storeUri.getLister()).connect(); - // outStream = sourceFtp.storeFileStream(fileName + DOT_XML); - // if (outStream == null) - // throw new NullPointerException("Can not create: " + fileName + DOT_XML + " on FTP. Reply is:" + sourceFtp.getReplyString()); - // outStream.write(metadata); - // outStream.flush(); - // } catch (Exception e) { - // logger.catching(e); - // throw e; - // } finally { - // if (outStream != null) - // outStream.close(); - // storeUri.cleanUp(); - // } - // } - -} diff --git a/server/-product/production/AMC/jobs/steps/PASAPOOLTransferToStep.java b/server/-product/production/AMC/jobs/steps/PASAPOOLTransferToStep.java deleted file mode 100644 index 0c69ee3a..00000000 --- a/server/-product/production/AMC/jobs/steps/PASAPOOLTransferToStep.java +++ /dev/null @@ -1,102 +0,0 @@ -package user.jobengine.server.steps; - -import java.nio.file.Path; -import java.nio.file.Paths; - -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; -import org.apache.logging.log4j.Marker; -import org.apache.logging.log4j.MarkerManager; - -import user.commons.RemoteFile; -import user.commons.StoreUri; -import user.commons.mediaarea.MediaArea; -import user.commons.strings.FileSizeUtils; - -public class PASAPOOLTransferToStep extends TransferStep { - private static final Logger logger = LogManager.getLogger(); - private static final String CODEC_PROFILE_HIGH = "HIGH"; - private static final String CODEC_PROFILE_MAIN = "MAIN"; - private static final String CODEC_PROFILE_422 = "4:2:2"; - private static final String DISPLAY_ASPECT_16_9 = "16:9"; - private static final String DISPLAY_ASPECT_4_3 = "4:3"; - private StoreUri sourceStoreUri; - private String sourceFileName; - private Marker marker = MarkerManager.getMarker("MEDIAPROFILE"); - - @Override - @StepEntry - public Object[] execute(StoreUri sourceStoreUri, String sourceFileName, StoreUri targetStoreUri, String targetFileName) throws Exception { - this.sourceStoreUri = sourceStoreUri; - this.sourceFileName = sourceFileName; - long start = System.currentTimeMillis(); - Object[] result = super.execute(sourceStoreUri, sourceFileName, targetStoreUri, targetFileName); - RemoteFile remoteFile = sourceStoreUri.getRemoteFile(targetFileName); - if (remoteFile != null && remoteFile.getSize() > 0) { - long diff = (System.currentTimeMillis() - start) / 1000; - if (diff > 0) { - long bytesSpeed = remoteFile.getSize() / diff; - String speed = FileSizeUtils.sizeAsString(bytesSpeed); - logger.info(getMarker(), "{} size is {}, upload speed was {}/sec", targetFileName, FileSizeUtils.sizeAsString(remoteFile.getSize()), speed); - } - } - return result; - } - - @Override - protected StoreUri getTargetStoreUri(StoreUri targetStoreUri) { - StoreUri result = targetStoreUri; - try { - Path filePath = Paths.get(sourceStoreUri.toString(true), sourceFileName); - MediaArea mi = new MediaArea(filePath); - mi.process(); - - if (isHD(mi)) { - logger.info(marker, "{} is MOVED_HD", sourceFileName); - result = getManager().getStoreUri("PEABLEBEACH", targetStoreUri.getProtocol()); - return result; - } - - if (isSD_HIGH_16_9(mi)) { - logger.info(marker, "{} is CONVSD16_9", sourceFileName); - result = getManager().getStoreUri("SELENIOPOOL_16_9", targetStoreUri.getProtocol()); - return result; - } - - if (isSD_HIGH_4_3(mi)) { - logger.info(marker, "{} is CONVSD4_3", sourceFileName); - result = getManager().getStoreUri("SELENIOPOOL_4_3", targetStoreUri.getProtocol()); - return result; - } - - if (isSD_MAIN_422(mi)) { - logger.info(marker, "{} is MOVED_SD", sourceFileName); - result = getManager().getStoreUri("PEABLEBEACH", targetStoreUri.getProtocol()); - return result; - } - } catch (Exception e) { - logger.error(getSessionMarker(), e.getMessage()); - } - return result; - } - - private boolean isHD(MediaArea mi) { - return mi.getHeight() == 1080; - } - - private boolean isSD_HIGH_16_9(MediaArea mi) { - String formatProfileName = mi.getFormatProfileName(); - return mi.getHeight() < 650 && formatProfileName.contains(CODEC_PROFILE_HIGH) && DISPLAY_ASPECT_16_9.equals(mi.getDisplayAspect()); - } - - private boolean isSD_HIGH_4_3(MediaArea mi) { - String formatProfileName = mi.getFormatProfileName(); - return mi.getHeight() < 650 && formatProfileName.contains(CODEC_PROFILE_HIGH) && DISPLAY_ASPECT_4_3.equals(mi.getDisplayAspect()); - } - - private boolean isSD_MAIN_422(MediaArea mi) { - String formatProfileName = mi.getFormatProfileName(); - return mi.getHeight() < 650 && (formatProfileName.contains(CODEC_PROFILE_MAIN) || formatProfileName.contains(CODEC_PROFILE_422)); - } - -} diff --git a/server/-product/production/AMC/jobs/steps/PBARCHIVETransferFromStep.java b/server/-product/production/AMC/jobs/steps/PBARCHIVETransferFromStep.java deleted file mode 100644 index d47c2b79..00000000 --- a/server/-product/production/AMC/jobs/steps/PBARCHIVETransferFromStep.java +++ /dev/null @@ -1,21 +0,0 @@ -package user.jobengine.server.steps; - -import java.nio.file.Path; -import java.nio.file.Paths; - -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; - -import user.commons.StoreUri; - -public class PBARCHIVETransferFromStep extends TransferStep { - private static final Logger logger = LogManager.getLogger(); - - @Override - @StepEntry - public Object[] execute(StoreUri sourceStoreUri, String sourceFileName, StoreUri targetStoreUri, String targetFileName) throws Exception { - Object[] result = null; - result = super.execute(sourceStoreUri, sourceFileName, targetStoreUri, targetFileName); - return result; - } -} diff --git a/server/-product/production/AMC/jobs/steps/PBArchiveCheckerStep.java b/server/-product/production/AMC/jobs/steps/PBArchiveCheckerStep.java deleted file mode 100644 index 8beafb23..00000000 --- a/server/-product/production/AMC/jobs/steps/PBArchiveCheckerStep.java +++ /dev/null @@ -1,187 +0,0 @@ -package user.jobengine.server.steps; - -import java.io.IOException; -import java.nio.file.Paths; -import java.sql.Timestamp; -import java.text.SimpleDateFormat; -import java.time.Duration; -import java.time.Instant; -import java.util.Iterator; - -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; - -import user.commons.DownloadableMedia; -import user.commons.StoreUri; -import user.commons.nexio.api.Clip; -import user.commons.nexio.api.ClipNotFoundException; -import user.commons.nexio.api.Controller; -import user.commons.nexio.api.Mediabase; -import user.commons.nexio.server.protocol.ProtocolException; -import user.commons.remotestore.RemoteStoreProtocol; -import user.jobengine.db.Media; -import user.jobengine.db.MediaFile; - -public class NEXIOArchiveCheckerStep extends JobStep { - private static final String DOT_MXF = ".mxf"; - private static final Logger logger = LogManager.getLogger(); - private SimpleDateFormat df = new SimpleDateFormat("yyyyMMdd-HHmmss"); - - private boolean check(Timestamp modified, Timestamp created, int duration, Media media, int videoFormat, int lastModifiedHours) { - String title = media.getTitle(); - - //formatum - if (!(videoFormat == 3 || videoFormat == 19)) { - logger.debug(getMarker(), "{} unsupported video format", videoFormat); - return false; - } - - if (!checkModifiedByTime(title, modified, lastModifiedHours)) - return false; - - //utolso modositas - MediaFile mediaFile = getManager().getSystemMediaFile(media); - boolean isModified = true; - if (mediaFile == null) { - logger.info(getMarker(), "{} system media file not available", title); - return false; - } - - String mcMod = df.format(mediaFile.getLastModified()); - String nexioMod = df.format(modified); - isModified = mcMod.compareTo(nexioMod) < 0; - if (isModified) { - logger.info(getMarker(), "{} modification time: MC {} < NEXIO {}", title, mcMod, nexioMod); - return true; - } - - boolean isLengthChanged = media.getLength() > 0 && Math.abs(media.getLength() - duration) > 3; - if (isLengthChanged) { - logger.info(getMarker(), "{} length: MC {} != NEXIO {}", title, media.getLength(), duration); - return true; - } - - return false; - } - - private boolean checkModifiedByTime(String title, Timestamp modified, int lastModifiedHours) { - //utolso modositas a mosthoz kepest - //The result of this method can be a negative period if the end is before the start. - long modifiedHours = Duration.between(modified.toInstant(), Instant.now()).toHours(); - if (modifiedHours < 0) { - logger.info(getMarker(), "{} modification time greather than now", title); - return false; - } - if (modifiedHours < lastModifiedHours) { - logger.debug(getMarker(), "{} modification time is too cloose for now", title); - return false; - } - - return true; - } - - @StepEntry - public Object[] execute(String sourceStoreName, String escortStoreName, int lastModifiedHours) throws Exception { - Controller controller = null; - - try { - getJobRuntime().setRelated(sourceStoreName + " archive"); - - StoreUri nexioStoreUri = getManager().getStoreUri(sourceStoreName, RemoteStoreProtocol.NEXIO); - StoreUri sourceStoreUri = getManager().getStoreUri(sourceStoreName, RemoteStoreProtocol.FTP); - StoreUri targetStoreUri = getManager().getStoreUri("TSM", RemoteStoreProtocol.TSM); - StoreUri escortStoreUri = getManager().getStoreUri(escortStoreName, RemoteStoreProtocol.LOCAL); - String outputPath = Paths.get(escortStoreUri.toString(true)).toString(); - - logger.info(getMarker(), "Connecting mediabase {}", nexioStoreUri.toString()); - if (nexioStoreUri.getPortNumber() == 0) - controller = new Controller(nexioStoreUri.getUri(), nexioStoreUri.getPortNumber()); - else - controller = new Controller(nexioStoreUri.getUri()); - controller.connect(); - Mediabase mediabase = controller.getMediabase(); - - Iterator clips = mediabase.getClips(); - logger.info(getMarker(), "Processing clips"); - - processClips(sourceStoreName, sourceStoreUri, targetStoreUri, outputPath, clips, lastModifiedHours); - } catch (Exception e) { - logger.error(getSessionMarker(), e.getMessage() == null ? "Unknown error" : e.getMessage()); - throw e; - } finally { - setProgress(100); - if (controller != null) - controller.disconnect(); - } - - return null; - } - - private void processClips(String storeName, StoreUri sourceStoreUri, StoreUri targetStoreUri, String outputPath, Iterator clips, - int lastModifiedHours) throws ClipNotFoundException, IOException, ProtocolException { - int processed = 0; - while (clips.hasNext()) { - - if (getJobRuntime().isWaitingCancel()) { - cancel(); - break; - } - - Clip clip = clips.next(); - - try { - String title = null; - Timestamp created = null; - Timestamp modified = null; - int frames = 0; - int videoFormat = 0; - frames = clip.getDuration(); - - //ures clip - if (frames == 1) - continue; - - title = clip.getXid().get(); - created = Timestamp.from(clip.getRecordDateTimestamp().toInstant()); - modified = Timestamp.from(clip.getModifiedTimestamp().toInstant()); - videoFormat = clip.getVideoFormat(); - - Media media = getManager().getMedia(title); - logger.debug(getMarker(), "Checking {}", title); - boolean createEscort = false; - - long mediaId = 0; - if (media == null) { - if (checkModifiedByTime(title, modified, lastModifiedHours)) { - createEscort = true; - logger.info(getMarker(), "{} not archived yet", title); - } - } else { - mediaId = media.getId(); - createEscort = check(modified, created, frames, media, videoFormat, lastModifiedHours); - } - - if (createEscort && !title.contains("*")) { - String fileName = title + DOT_MXF; - DownloadableMedia downloadable = DownloadableMedia.create(title, fileName, modified, created, frames, 0, sourceStoreUri.getId(), - targetStoreUri.getId(), mediaId); - String escortFileName = storeName + "." + downloadable.getString("fileName"); - if (EscortFiles.createMetadataIfNotExists(outputPath, escortFileName, downloadable.toPrettyString(""))) { - logger.info(getMarker(), "Archive status file created for {}", fileName); - } else { - logger.info(getMarker(), "Archive status file already exists for {}", fileName); - } - - } - - processed++; - if (processed % 100 == 0) - logger.info(getMarker(), "Processed {} records", processed); - } catch (Exception e) { - logger.error(getSessionMarker(), e.getMessage()); - continue; - } - } - } - -} diff --git a/server/-product/production/AMC/jobs/steps/PBQuery.java b/server/-product/production/AMC/jobs/steps/PBQuery.java deleted file mode 100644 index 458b1375..00000000 --- a/server/-product/production/AMC/jobs/steps/PBQuery.java +++ /dev/null @@ -1,326 +0,0 @@ -package user.jobengine.server.steps; - -import java.io.DataInputStream; -import java.io.FileNotFoundException; -import java.io.IOException; -import java.io.StringReader; -import java.net.URL; -import java.nio.file.Files; -import java.nio.file.Path; -import java.nio.file.Paths; -import java.text.SimpleDateFormat; -import java.util.Calendar; -import java.util.Collection; -import java.util.Date; -import java.util.HashMap; -import java.util.List; -import java.util.Map; -import java.util.TreeMap; - -import javax.xml.parsers.DocumentBuilder; -import javax.xml.parsers.DocumentBuilderFactory; - -import org.apache.commons.lang.StringUtils; -import org.apache.http.HttpEntity; -import org.apache.http.HttpResponse; -import org.apache.http.auth.AuthScope; -import org.apache.http.auth.UsernamePasswordCredentials; -import org.apache.http.client.methods.HttpPost; -import org.apache.http.entity.StringEntity; -import org.apache.http.impl.client.DefaultHttpClient; -import org.apache.http.params.BasicHttpParams; -import org.apache.http.params.HttpConnectionParams; -import org.apache.http.params.HttpParams; -import org.apache.http.protocol.HTTP; -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; -import org.w3c.dom.Document; -import org.w3c.dom.NamedNodeMap; -import org.w3c.dom.Node; -import org.w3c.dom.NodeList; -import org.xml.sax.InputSource; - -public class PBQuery { - public class MMMedia { - private final String name; - private final String usage; - - MMMedia(String name, String usage) { - this.name = name; - this.usage = usage; - - } - - public String getName() { - return name; - } - - public String getUsage() { - return usage; - } - } - - private static final Logger logger = LogManager.getLogger(); - private static final int CONNECTION_TIMEOUT = 5000; - private static final int SOCKET_TIMEOUT = 5000; - public static final String GETMEDIAUSAGEBYUTRANGE = "/getMediaUsageByUTRange"; - public static final String GETCUSTOMVIEW = "/getCustomView"; - - public static final String SOAP_TEMPLATE_ROOT = System.getProperty("jobengine.soap.template.root", "/opt/mediacube/configuration/soap"); - - private static Document toDocument(String xml) { - DocumentBuilderFactory factory = DocumentBuilderFactory.newInstance(); - DocumentBuilder builder = null; - try { - builder = factory.newDocumentBuilder(); - Document doc = builder.parse(new InputSource(new StringReader(xml))); - return doc; - } catch (Exception e) { - logger.error(e.getMessage()); - } - return null; - } - - private SimpleDateFormat sdf_range = new SimpleDateFormat("yyyy-MM-dd'T'HH:mm:ss:'00'"); - private String primaryEndPoint; - private String primaryUserName; - private String primaryPassword; - private String secondaryEndPoint; - private String secondaryUserName; - - private String secondaryPassword; - - private int rangeForwardHours; - - private String getAttribute(Node node, String name) { - String result = null; - if (node != null && node.hasAttributes()) { - NamedNodeMap attributes = node.getAttributes(); - if (attributes != null) { - Node mediaNameNode = attributes.getNamedItem(name); - if (mediaNameNode != null) - result = mediaNameNode.getNodeValue(); - } - } - return result; - } - - public Collection getPossibelMissingMaterialNames(List poolContent) throws Exception { - Map result = querySortableMedias(poolContent); - queryOtherMedias(poolContent, result); - - TreeMap sortedUsages = new TreeMap<>(); - for (MMMedia m : result.values()) { - //a TreeMap lehetove teszi: nincs duplikatum + a korabbi musorok elore kerulnek - sortedUsages.put(m.getUsage() + "." + m.getName(), m); - } - - return sortedUsages.values(); - - } - - public void init(String primaryEndPoint, String primaryUserName, String primaryPassword, String secondaryEndPoint, String secondaryUserName, - String secondaryPassword, int rangeForwardHours) throws Exception { - this.primaryEndPoint = primaryEndPoint; - this.primaryUserName = primaryUserName; - this.primaryPassword = primaryPassword; - this.secondaryEndPoint = secondaryEndPoint; - this.secondaryUserName = secondaryUserName; - this.secondaryPassword = secondaryPassword; - this.rangeForwardHours = rangeForwardHours; - } - - private void putEarliest(Map result, String mediaName, String usageTime) { - if (result.containsKey(mediaName)) { - MMMedia m = result.get(mediaName); - //az aktualis a korabbi idopont - if (usageTime.compareTo(m.getUsage()) < 0) - result.put(mediaName, new MMMedia(mediaName, usageTime)); - } else - result.put(mediaName, new MMMedia(mediaName, usageTime)); - } - - public String query(String endPoint, String action, String soap, String user, String pwd) throws Exception { - - HttpParams httpParameters = new BasicHttpParams(); - HttpConnectionParams.setConnectionTimeout(httpParameters, CONNECTION_TIMEOUT); - HttpConnectionParams.setSoTimeout(httpParameters, SOCKET_TIMEOUT); - DefaultHttpClient httpclient = new DefaultHttpClient(httpParameters); - //http://10.170.100.61:18083 - - String result = null; - DataInputStream is = null; - try { - URL url = new URL(endPoint); - httpclient.getCredentialsProvider().setCredentials(new AuthScope(url.getHost(), url.getPort(), null, "Digest"), - new UsernamePasswordCredentials(user, pwd)); - - HttpPost httppost = new HttpPost(url.toURI()); - httppost.setHeader("soapaction", action); - //Content-Type application/soap+xml; charset=utf-8; action="/getMediaUsageByUTRange" - httppost.setHeader("Content-Type", "text/xml; charset=utf-8"); - HttpEntity entity = new StringEntity(soap.toString(), HTTP.UTF_8); - httppost.setEntity(entity); - logger.info("Executing call {}", action); - HttpResponse response = httpclient.execute(httppost);// calling server - HttpEntity r_entity = response.getEntity(); //get response - // Header[] headers = response.getAllHeaders(); - // for (Header h : headers) { - // logger.info("Reponse Header", h.getName() + ": " + h.getValue()); - // } - if (r_entity != null) { - byte[] bytes = new byte[(int) r_entity.getContentLength()]; - if (r_entity.isStreaming()) { - is = new DataInputStream(r_entity.getContent()); - is.readFully(bytes); - } - result = new String(bytes); - } - } catch (Exception e) { - logger.error("Exception while connecting to endpoint {}. System message is: {}", e.getMessage()); - throw e; - } finally { - if (is != null) { - try { - is.close(); - } catch (IOException e) { - } - } - if (httpclient != null) { - httpclient.getConnectionManager().shutdown(); - } - } - - return result; - } - - public void queryOtherMedias(List poolContent, Map result) throws Exception { - String xml = null; - String soap = null; - Path template = Paths.get(SOAP_TEMPLATE_ROOT, GETCUSTOMVIEW); - - if (!template.toFile().exists()) - throw new FileNotFoundException(template.toString()); - try { - soap = new String(Files.readAllBytes(template)); - xml = query(primaryEndPoint, GETCUSTOMVIEW, soap, primaryUserName, primaryPassword); - } catch (Exception e1) { - try { - xml = query(secondaryEndPoint, GETCUSTOMVIEW, soap, secondaryUserName, secondaryPassword); - } catch (Exception e2) { - } - } - - if (StringUtils.isBlank(xml)) { - logger.info("Query result is empty"); - return; - } - - Document document = toDocument(xml); - NodeList mediaNodes = document.getElementsByTagName("media"); - - if (mediaNodes == null || mediaNodes.getLength() == 0) - return; - - for (int i = 0; i < mediaNodes.getLength(); i++) { - Node mediaNode = mediaNodes.item(i); - String mediaName = getAttribute(mediaNode, "mediaName"); - if (StringUtils.isBlank(mediaName)) { - logger.info("Missing mediaName"); - continue; - } - - if (poolContent.contains(mediaName)) - continue; - - String usageTime = getAttribute(mediaNode, "earliestUsageTime"); - putEarliest(result, mediaName, usageTime); - } - - logger.info("Document done, items {}", mediaNodes.getLength()); - - } - - public Map querySortableMedias(List poolContent) throws Exception { - Map result = new HashMap<>(); - - String xml = null; - String soap = null; - - Path template = Paths.get(SOAP_TEMPLATE_ROOT, GETMEDIAUSAGEBYUTRANGE); - if (!template.toFile().exists()) - throw new FileNotFoundException(template.toString()); - - try { - long now = System.currentTimeMillis(); - Calendar cal = Calendar.getInstance(); - cal.setTimeInMillis(now); - cal.add(Calendar.HOUR_OF_DAY, rangeForwardHours); - - String fromDateStr = sdf_range.format(new Date(now)); - String toDateStr = sdf_range.format(cal.getTime()); - - soap = new String(Files.readAllBytes(template)); - soap = soap.replace("{start}", fromDateStr); - soap = soap.replace("{end}", toDateStr); - - xml = query(primaryEndPoint, GETMEDIAUSAGEBYUTRANGE, soap, primaryUserName, primaryPassword); - } catch (Exception e1) { - try { - xml = query(secondaryEndPoint, GETMEDIAUSAGEBYUTRANGE, soap, secondaryUserName, secondaryPassword); - } catch (Exception e2) { - } - } - - if (StringUtils.isBlank(xml)) { - logger.info("Query result is empty"); - return result; - } - - Document document = toDocument(xml); - NodeList mediaNodes = document.getElementsByTagName("mediaUsage"); - - if (mediaNodes == null || mediaNodes.getLength() == 0) - return result; - - for (int i = 0; i < mediaNodes.getLength(); i++) { - Node mediaNode = mediaNodes.item(i); - String mediaName = getAttribute(mediaNode, "mediaName"); - String mediaType = getAttribute(mediaNode, "mediaType"); - if (StringUtils.isBlank(mediaName)) { - logger.info("Missing mediaName"); - continue; - } - - if (!StringUtils.equals("Video", mediaType)) { - continue; - } - - if (poolContent.contains(mediaName)) - continue; - - Node usageChild = mediaNode.getFirstChild(); - if (usageChild == null) { - logger.info("Skipping {}, no media usage defined", mediaName); - continue; - } - - NodeList usageNodes = usageChild.getChildNodes(); - if (usageNodes == null || usageNodes.getLength() == 0) { - logger.info("Skipping {}, no media usage defined", mediaName); - continue; - } - - for (int j = 0; j < usageNodes.getLength(); j++) { - Node mediaUsageNode = usageNodes.item(j); - String usageTime = getAttribute(mediaUsageNode, "earliestUsageTime"); - logger.info("{}, {}", mediaName, usageTime); - putEarliest(result, mediaName, usageTime); - } - - } - - return result; - } - -} diff --git a/server/-product/production/AMC/jobs/steps/PathItemsCollectorStep.java b/server/-product/production/AMC/jobs/steps/PathItemsCollectorStep.java deleted file mode 100644 index 725bca4f..00000000 --- a/server/-product/production/AMC/jobs/steps/PathItemsCollectorStep.java +++ /dev/null @@ -1,73 +0,0 @@ -package user.jobengine.server.steps; - -import java.io.IOException; -import java.nio.file.FileVisitResult; -import java.nio.file.FileVisitor; -import java.nio.file.Files; -import java.nio.file.Path; -import java.nio.file.Paths; -import java.nio.file.SimpleFileVisitor; -import java.nio.file.attribute.BasicFileAttributes; -import java.util.ArrayList; -import java.util.List; - -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; - -import com.ibm.nosql.json.api.BasicDBObject; - -/* DO NOT REMOVE! -import user.jobengine.server.steps.FileSearchFilterOptions; -*/ -public class PathItemsCollectorStep extends JobStep { - private static final Logger logger = LogManager.getLogger(); - - protected FileSearchFilterOptions createFileFilter(BasicDBObject filter) { - return new FileSearchFilterOptions(filter); - } - - @StepEntry - public Object[] execute(String sourceFolder, BasicDBObject filter) throws Exception { - Path sourcePath = Paths.get(sourceFolder); - - List files = new ArrayList<>(); - - //teljes rekurzivitas szuressel - FileSearchFilterOptions filterOptions = createFileFilter(filter); - FileVisitor visitor = new SimpleFileVisitor() { - @Override - public FileVisitResult preVisitDirectory(Path dir, BasicFileAttributes attrs) throws IOException { - if (!filterOptions.preAcceptDirectory(dir)) - return FileVisitResult.SKIP_SUBTREE; - return FileVisitResult.CONTINUE; - } - - @Override - public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) throws IOException { - if (filterOptions.acceptFile(file)) - files.add(file.toString()); - logger.info(file); - return FileVisitResult.CONTINUE; - } - - @Override - public FileVisitResult visitFileFailed(Path file, IOException exc) throws IOException { - return FileVisitResult.CONTINUE; - } - - }; - - try { - Files.walkFileTree(sourcePath, visitor); - } catch (Exception e) { - logger.error(getSessionMarker(), "Hiba a '{}' mappa feldolgozásában. A rendszer hibaüzenete: {}", sourcePath, e.getMessage()); - logger.catching(e); - throw e; - } finally { - } - - logger.info("Found {} files", files.size()); - - return new Object[] { files }; - } -} diff --git a/server/-product/production/AMC/jobs/steps/PeableBeachMissingMaterialCheckerStep.java b/server/-product/production/AMC/jobs/steps/PeableBeachMissingMaterialCheckerStep.java deleted file mode 100644 index 3b1a1fa0..00000000 --- a/server/-product/production/AMC/jobs/steps/PeableBeachMissingMaterialCheckerStep.java +++ /dev/null @@ -1,225 +0,0 @@ -package user.jobengine.server.steps; - -import java.nio.file.DirectoryStream; -import java.nio.file.Files; -import java.nio.file.Path; -import java.nio.file.Paths; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.Collection; -import java.util.LinkedHashSet; -import java.util.List; - -import org.apache.commons.lang.StringUtils; -import org.apache.commons.net.ftp.FTPClient; -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; - -import com.ibm.nosql.json.api.BasicDBList; - -import user.commons.DownloadableMedia; -import user.commons.StoreUri; -import user.commons.remotestore.FtpDirectoryLister; -import user.commons.remotestore.RemoteStoreProtocol; -import user.jobengine.db.Media; -import user.jobengine.server.steps.PBQuery.MMMedia; - -public class PeableBeachMissingMaterialCheckerStep extends JobStep { - private static final Logger logger = LogManager.getLogger(); - private static final String appendExtension = System.getProperty("missingmaterial.appendextension", ".mxf"); - - @StepEntry - public Object[] execute(String escortStoreName, BasicDBList lookupStoreNames, String targetStoreName, String targetProtocol, String primaryEndPoint, - String primaryUserName, String primaryPassword, String secondaryEndPoint, String secondaryUserName, String secondaryPassword, int rangeForwardHours) - throws Exception { - StoreUri escortStoreUri = null; - try { - getJobRuntime().setRelated(targetStoreName + " restore"); - - escortStoreUri = getManager().getStoreUri(escortStoreName, RemoteStoreProtocol.LOCAL); - StoreUri sourceStoreUri = getManager().getStoreUri("TSM", RemoteStoreProtocol.TSM); - StoreUri targetStoreUri = getManager().getStoreUri(targetStoreName, Enum.valueOf(RemoteStoreProtocol.class, targetProtocol)); - - LinkedHashSet poolContent = new LinkedHashSet<>(); - - int progress = 0; - int count = lookupStoreNames.size(); - int processed = 0; - - for (Object lookup : lookupStoreNames) { - if (getJobRuntime().isWaitingCancel()) { - cancel(); - return null; - } - String lookupStoreName = String.valueOf(lookup); - - StoreUri lookupStoreUri = getManager().getStoreUri(lookupStoreName, Enum.valueOf(RemoteStoreProtocol.class, targetProtocol)); - logger.info(getMarker(), "Checking pool {}", lookupStoreName); - List lookupContent = getPoolContent(lookupStoreName, lookupStoreUri); - poolContent.addAll(lookupContent); - processed++; - progress = processed * 50 / count; - setProgress(progress); - } - /* - if (1 == 1) { - - logger.info(getMarker(), "Kikapcsolva"); - return null; - } - */ - - PBQuery query = new PBQuery(); - query.init(primaryEndPoint, primaryUserName, primaryPassword, secondaryEndPoint, secondaryUserName, secondaryPassword, rangeForwardHours); - Collection medias = query.getPossibelMissingMaterialNames(new ArrayList<>(poolContent)); - logger.info(getMarker(), "API returned {} items", medias == null ? 0 : medias.size()); - - if (medias != null && medias.size() > 0) { - count = medias.size(); - for (MMMedia media : medias) { - logger.info(getMarker(), "API response contains {}", media.getName()); - } - - for (MMMedia media : medias) { - if (getJobRuntime().isWaitingCancel()) { - cancel(); - break; - } - processRecord(media, sourceStoreUri, targetStoreName, targetStoreUri, escortStoreUri); - processed++; - progress = 50 + (processed * 50 / count); - setProgress(progress); - } - - } - setProgress(100); - } catch (Exception e) { - logger.error(getSessionMarker(), e.getMessage()); - throw e; - } finally { - if (escortStoreUri != null) - escortStoreUri.cleanUp(); - } - - return null; - } - - private List getPoolContent(String storeName, StoreUri targetStoreUri) throws Exception { - List poolContent = new ArrayList<>(); - - FtpDirectoryLister lister = (FtpDirectoryLister) targetStoreUri.getLister(); - FTPClient client = lister.connect(); - logger.info(getMarker(), "Working dir is {}", client.printWorkingDirectory()); - String[] listNames = client.listNames(); - targetStoreUri.cleanUp(); - - if (listNames != null) { - List remoteFileNames = Arrays.asList(listNames); - logger.info(getMarker(), "Remote file list size :{}", remoteFileNames.size()); - for (String name : remoteFileNames) { - try { - if (name.toLowerCase().endsWith(".mxf")) { - int pos = name.lastIndexOf("."); - if (pos > -1) { - name = name.substring(0, pos); - //logger.info(getMarker(), "Adding {}", name); - poolContent.add(name); - } - } - } catch (Exception e) { - logger.error(e.getMessage()); - } - - } - } - - // List remoteFiles = targetStoreUri.getRemoteFiles(); - // if (remoteFiles != null) { - // logger.info(getMarker(), "Remote file list size :{}", remoteFiles.size()); - // for (RemoteFile rf : remoteFiles) { - // try { - // String name = rf.getName(); - // if (name.toLowerCase().endsWith(".mxf")) { - // int pos = name.lastIndexOf("."); - // if (pos > -1) { - // name = name.substring(0, pos); - // //logger.info(getMarker(), "Adding {}", name); - // poolContent.add(name); - // } - // } - // } catch (Exception e) { - // logger.error(e.getMessage()); - // } - // - // } - // } - logger.info(getMarker(), "Pool {} contains {} items", storeName, poolContent == null ? 0 : poolContent.size()); - return poolContent; - } - - //csak visszaterunk - private String normalizeName(String mediaName) { - String result = mediaName; - if (StringUtils.isNotBlank(appendExtension)) - result += appendExtension; - // if (StringUtils.isNotBlank(mediaName)) { - // result = mediaName.trim().toLowerCase(); - // } - return result; - } - - private void processRecord(MMMedia mmedia, StoreUri sourceStoreUri, String targetStoreName, StoreUri targetStoreUri, StoreUri escortStoreUri) { - //TODO kisbetu/nagybetu problema kezelese - - Media media = null; - //duplikatum miatt - try { - media = getManager().getMedia(mmedia.getName()); - } catch (Exception e) { - logger.error(getMarker(), "Can't identify missing material {}. System message is: {}", mmedia.getName(), e.getMessage()); - return; - } - - String fileName = normalizeName(mmedia.getName()); - - if (media == null) { - logger.warn(getMarker(), "File {} not archived yet", mmedia.getName()); - return; - } - - DownloadableMedia downloadable = DownloadableMedia.create(mmedia.getName(), fileName, media.getModified(), media.getCreated(), media.getLength(), 0L, - sourceStoreUri.getId(), targetStoreUri.getId(), media.getId()); - downloadable.put("priority", 50); - String usage = mmedia.getUsage(); - usage = usage.replace("-", "").replace(":", ""); - String escortFileName = usage + "." + targetStoreName + "." + fileName; - Path outputPath = null; - try { - outputPath = Paths.get(escortStoreUri.toString(true)); - boolean exsists = statusFileExists(outputPath, fileName); - if (exsists) - return; - EscortFiles.createMetadata(outputPath.toString(), escortFileName, downloadable.toPrettyString("")); - logger.info(getMarker(), "Status file created {}", escortFileName); - } catch (Exception e) { - logger.error(getSessionMarker(), "Can't create status file {}", Paths.get(outputPath.toString(), escortFileName)); - } - - } - - private boolean statusFileExists(Path outputPath, String fileName) { - boolean exsists = false; - Path statusPath = Paths.get(outputPath.toString(), EscortFiles.STATUSFOLDER); - try (DirectoryStream p = Files.newDirectoryStream(statusPath, "*" + fileName + EscortFiles.DOT_JSON)) { - if (p.iterator().hasNext()) { - logger.info(getMarker(), "Status file for {} already exists", fileName); - exsists = true; - } - - } catch (Exception e1) { - logger.error(getSessionMarker(), e1.getMessage()); - } - return exsists; - } - -} diff --git a/server/-product/production/AMC/jobs/steps/RegisterUserRestoreStep.java b/server/-product/production/AMC/jobs/steps/RegisterUserRestoreStep.java deleted file mode 100644 index ba23e3f6..00000000 --- a/server/-product/production/AMC/jobs/steps/RegisterUserRestoreStep.java +++ /dev/null @@ -1,71 +0,0 @@ -package user.jobengine.server.steps; - -import java.nio.file.Paths; -import java.util.List; - -import org.apache.commons.lang.StringUtils; -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; - -import user.commons.DownloadableMedia; -import user.commons.StoreUri; -import user.commons.remotestore.RemoteStoreProtocol; -import user.jobengine.db.ArchivedMedia; -import user.jobengine.db.Media; - -public class RegisterUserRestoreStep extends JobStep { - private static final Logger logger = LogManager.getLogger(); - private static final String appendExtension = System.getProperty("missingmaterial.appendextension", ".mxf"); - - @StepEntry - public Object[] execute(String escortStoreName, String targetStoreName, List basket, String recipient) throws Exception { - StoreUri escortStoreUri = null; - try { - escortStoreUri = getManager().getStoreUri(escortStoreName, RemoteStoreProtocol.LOCAL); - StoreUri targetStoreUri = getManager().getStore(targetStoreName).getTargetStoreUri(); - StoreUri sourceStoreUri = getManager().getStoreUri("TSM", RemoteStoreProtocol.TSM); - int processed = 0; - for (ArchivedMedia media : basket) { - processRecord(media, targetStoreName, sourceStoreUri, targetStoreUri, escortStoreUri, recipient); - processed++; - int progress = processed * 100 / basket.size(); - setProgress(progress); - } - } catch (Exception e) { - logger.error(getSessionMarker(), e.getMessage()); - throw e; - } finally { - if (escortStoreUri != null) - escortStoreUri.cleanUp(); - } - - return null; - } - - private void processRecord(ArchivedMedia archivedMedia, String targetStoreName, StoreUri sourceStoreUri, StoreUri targetStoreUri, StoreUri escortStoreUri, - String recipient) { - Media media = archivedMedia.getMedia(); - - String fileName = media.getTitle(); - if (StringUtils.isNotBlank(appendExtension)) - fileName += appendExtension; - - DownloadableMedia downloadable = DownloadableMedia.create(media.getTitle(), fileName, media.getModified(), media.getCreated(), media.getLength(), 0L, - sourceStoreUri.getId(), targetStoreUri.getId(), media.getId()); - downloadable.put("recipient", recipient); - downloadable.put("skipValidation", true); - String escortFileName = targetStoreName + "." + downloadable.getString("fileName"); - String outputPath = null; - try { - outputPath = Paths.get(escortStoreUri.toString(true)).toString(); - if (EscortFiles.createMetadataIfNotExists(outputPath, escortFileName, downloadable.toPrettyString(""))) - logger.info(getSessionMarker(), "Status file created {}", escortFileName); - else - logger.info(getSessionMarker(), "Status file already exists {}", escortFileName); - } catch (Exception e) { - logger.error("Can't create escort file {}", Paths.get(outputPath.toString(), escortFileName)); - } - - } - -} diff --git a/server/-product/production/AMC/jobs/steps/RegisterVODRestoreStep.java b/server/-product/production/AMC/jobs/steps/RegisterVODRestoreStep.java deleted file mode 100644 index a7cb11f6..00000000 --- a/server/-product/production/AMC/jobs/steps/RegisterVODRestoreStep.java +++ /dev/null @@ -1,79 +0,0 @@ -package user.jobengine.server.steps; - -import java.nio.file.Paths; - -import org.apache.commons.lang.StringUtils; -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; - -import com.ibm.nosql.json.api.BasicDBList; -import com.ibm.nosql.json.api.BasicDBObject; - -import user.commons.DownloadableMedia; -import user.commons.StoreUri; -import user.commons.remotestore.RemoteStoreProtocol; -import user.jobengine.db.Media; - -public class RegisterVODRestoreStep extends JobStep { - private static final Logger logger = LogManager.getLogger(); - private static final String appendExtension = System.getProperty("missingmaterial.appendextension", ".mxf"); - - @StepEntry - public Object[] execute(String targetStoreName, BasicDBList basket) throws Exception { - try { - StoreUri sourceStoreUri = getManager().getStoreUri("TSM", RemoteStoreProtocol.TSM); - - StoreUri targetStoreUri = getManager().getStore(targetStoreName).getTargetStoreUri(); - StoreUri escortStoreUri = getManager().getStoreUri("MEDIACUBE_UPLOADS", RemoteStoreProtocol.LOCAL); - int processed = 0; - for (int i = 0; i < basket.size(); i++) { - BasicDBObject item = (BasicDBObject) basket.get(i); - //logger.info(item.toPrettyString()); - processRecord(targetStoreName, sourceStoreUri, targetStoreUri, escortStoreUri, item); - processed++; - int progress = processed * 100 / basket.size(); - setProgress(progress); - } - } catch (Exception e) { - logger.error(getSessionMarker(), e.getMessage()); - throw e; - } finally { - } - - return null; - } - - private void processRecord(String targetStoreName, StoreUri sourceStoreUri, StoreUri targetStoreUri, StoreUri escortStoreUri, BasicDBObject item) { - String mediaTitle = item.getString("title"); - String relativeTargetPath = item.getString("path"); - - Media media = getManager().getMedia(mediaTitle); - - if (media == null) { - logger.error("Media {} is not available", mediaTitle); - return; - } - - String fileName = media.getTitle(); - if (StringUtils.isNotBlank(appendExtension)) - fileName += appendExtension; - - DownloadableMedia downloadable = DownloadableMedia.create(media.getTitle(), fileName, media.getModified(), media.getCreated(), media.getLength(), 0L, - sourceStoreUri.getId(), targetStoreUri.getId(), media.getId()); - //downloadable.put("skipValidation", true); - downloadable.put("relativeTargetPath", relativeTargetPath); - String escortFileName = targetStoreName + "." + downloadable.getString("fileName"); - String outputPath = null; - try { - outputPath = Paths.get(escortStoreUri.toString(true)).toString(); - if (EscortFiles.createMetadataIfNotExists(outputPath, escortFileName, downloadable.toPrettyString(""))) - logger.info(getSessionMarker(), "Status file created {}", escortFileName); - else - logger.info(getSessionMarker(), "Status file already exists {}", escortFileName); - } catch (Exception e) { - logger.error("Can't create escort file {}", Paths.get(outputPath.toString(), escortFileName)); - } - - } - -} diff --git a/server/-product/production/AMC/jobs/steps/RundownFilesCollectorStep.java b/server/-product/production/AMC/jobs/steps/RundownFilesCollectorStep.java deleted file mode 100644 index 8b104d9b..00000000 --- a/server/-product/production/AMC/jobs/steps/RundownFilesCollectorStep.java +++ /dev/null @@ -1,42 +0,0 @@ -package user.jobengine.server.steps; - -import java.nio.file.Path; -import java.text.SimpleDateFormat; -import java.util.Calendar; - -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; - -import com.ibm.nosql.json.api.BasicDBObject; - -/* DO NOT REMOVE! -import user.jobengine.server.steps.FileSearchFilterOptions; -*/ -public class RundownFilesCollectorStep extends PathItemsCollectorStep { - private static final Logger logger = LogManager.getLogger(); - - @Override - protected FileSearchFilterOptions createFileFilter(BasicDBObject filter) { - SimpleDateFormat df = new SimpleDateFormat("yyyy/MM"); - Calendar thisMonthCal = Calendar.getInstance(); - Calendar nextMonthCal = Calendar.getInstance(); - nextMonthCal.add(Calendar.MONTH, 1); - - FileSearchFilterOptions result = new FileSearchFilterOptions(filter) { - @Override - public boolean acceptFile(Path file) { - if (file.toString().contains(df.format(thisMonthCal.getTime())) || file.toString().contains(df.format(nextMonthCal.getTime()))) - return super.acceptFile(file); - - return false; - } - }; - return result; - } - - @Override - @StepEntry - public Object[] execute(String sourceFolder, BasicDBObject filter) throws Exception { - return super.execute(sourceFolder, filter); - } -} diff --git a/server/-product/production/AMC/jobs/steps/ServerStatusReportStep.java b/server/-product/production/AMC/jobs/steps/ServerStatusReportStep.java deleted file mode 100644 index 787a07e8..00000000 --- a/server/-product/production/AMC/jobs/steps/ServerStatusReportStep.java +++ /dev/null @@ -1,39 +0,0 @@ -package user.jobengine.server.steps; - -import java.time.Instant; -import java.util.Map; -import java.util.Set; - -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; - -import user.commons.JobStatus; -import user.commons.MediaCubeMarker; -import user.jobengine.server.IJobEngine; -import user.jobengine.server.IJobRuntime; - -public class ServerStatusReportStep extends JobStep { - private static final Logger logger = LogManager.getLogger(); - - @StepEntry - public Object[] execute() throws Exception { - IJobEngine engine = getEngine(); - int jobCount = 0; - StringBuilder sb = new StringBuilder(); - Map jobs = engine.getJobs(); - if (jobs != null) { - Set keys = jobs.keySet(); - jobCount = keys.size(); - for (Long key : keys) { - IJobRuntime runtime = jobs.get(key); - if (JobStatus.SUSPENDED.equals(runtime.getStatus())) - sb.append(runtime.getRelated() + " felfüggesztve: " + runtime.getDescription() + "
"); - } - } - MediaCubeMarker marker = new MediaCubeMarker(); - - marker.setSubject(String.format("%s [%d db] AMC MediaCube feldolgozás", Instant.now(), jobCount)); - logger.info(marker, sb.toString()); - return null; - } -} diff --git a/server/-product/production/AMC/jobs/steps/SyncSubtitlesStep.java b/server/-product/production/AMC/jobs/steps/SyncSubtitlesStep.java deleted file mode 100644 index 39c338c3..00000000 --- a/server/-product/production/AMC/jobs/steps/SyncSubtitlesStep.java +++ /dev/null @@ -1,94 +0,0 @@ -package user.jobengine.server.steps; - -import java.nio.file.Path; -import java.nio.file.Paths; - -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; - -import user.commons.StoreUri; -import user.commons.remotestore.RemoteStoreProtocol; -import user.jobengine.db.Store; - -public class TransferStep extends JobStep { - private static final String DOT_PART = ".part"; - private static final Logger logger = LogManager.getLogger(); - private static final boolean simulateTransferToTSM = Boolean.parseBoolean(System.getProperty("test.simulate.transfer.tsm", "false")); - protected StoreUri targetStoreUri; - - @StepEntry - public Object[] execute(StoreUri sourceStoreUri, String sourceFileName, StoreUri targetStoreUri, String targetFileName) throws Exception { - try { - this.targetStoreUri = targetStoreUri; - getJobRuntime().setCancelable(false); - - Store sourceStore = getManager().getStore(sourceStoreUri.getStoreId()); - Store targetStore = getManager().getStore(targetStoreUri.getStoreId()); - - String description = String.format("%s -> %s : %s", sourceStore.getName(), targetStore.getName(), sourceFileName); - getJobRuntime().setDescription(description); - - //logger.info(getSessionMarker(), String.valueOf(isSimulateTSMWrite(sourceStoreUri, targetStoreUri))); - if (isSimulateTSMWrite(sourceStoreUri, targetStoreUri)) { - logger.info(getMarker(), "Skipping real TSM write of {} from {} to {}", sourceFileName, sourceStoreUri, targetStoreUri); - return null; - } - - sourceStoreUri.addProgressListener(e -> setProgress(e.getProgress())); - - this.targetStoreUri = getTargetStoreUri(); - tryCopy(sourceStoreUri, sourceFileName, targetStoreUri, targetFileName); - } catch (Exception e) { - logger.error(getMarker(), "Error in transfer of {} when copying from {} to {}.", sourceFileName, sourceStoreUri, targetStoreUri); - throw e; - // logger.error(getMarker(), "Error in transfer of {} when copying from {} to {}. Retrying after 3 seconds.", sourceFileName, sourceStoreUri, - // targetStoreUri); - - // try { - // Thread.sleep(3000); - // tryCopy(sourceStoreUri, sourceFileName, targetStoreUri, targetFileName); - // } catch (Exception e1) { - // logger.error(getSessionMarker(), "Error in transfer of {} when copying from {} to {}. System message is: {}", sourceFileName, sourceStoreUri, - // targetStoreUri, e1.getMessage()); - // throw e1; - // } - } finally { - if (sourceStoreUri != null) - sourceStoreUri.cleanUp(); - if (targetStoreUri != null) - targetStoreUri.cleanUp(); - } - return null; - } - - protected StoreUri getTargetStoreUri() { - return targetStoreUri; - } - - protected String getTmpExtension() { - return DOT_PART; - } - - private boolean isSimulateTSMWrite(StoreUri sourceStoreUri, StoreUri targetStoreUri) { - return simulateTransferToTSM && (RemoteStoreProtocol.TSM.equals(targetStoreUri.getProtocol())); - } - - private void tryCopy(StoreUri sourceStoreUri, String sourceFileName, StoreUri targetStoreUri, String targetFileName) throws Exception { - String currentTargetFileName = targetFileName; - - boolean renameAfterCopy = false; - if (getTmpExtension() != null && RemoteStoreProtocol.LOCAL.equals(targetStoreUri.getProtocol())) { - currentTargetFileName += getTmpExtension(); - renameAfterCopy = true; - } - - sourceStoreUri.transferFrom(targetStoreUri, sourceFileName, currentTargetFileName); - logger.info(getMarker(), "Transfer of {} completed from {} to {}", sourceFileName, sourceStoreUri, targetStoreUri); - - if (renameAfterCopy) { - Path tmpTargetFile = Paths.get(targetStoreUri.toString(true), currentTargetFileName); - Path targetFile = Paths.get(targetStoreUri.toString(true), targetFileName); - tmpTargetFile.toFile().renameTo(targetFile.toFile()); - } - } -} diff --git a/server/-product/production/AMC/jobs/steps/TSMTransferFromStep.java b/server/-product/production/AMC/jobs/steps/TSMTransferFromStep.java deleted file mode 100644 index ab1b6bbf..00000000 --- a/server/-product/production/AMC/jobs/steps/TSMTransferFromStep.java +++ /dev/null @@ -1,27 +0,0 @@ -package user.jobengine.server.steps; - -import java.nio.file.Path; -import java.nio.file.Paths; - -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; - -import user.commons.StoreUri; - -public class TSMTransferFromStep extends TransferStep { - private static final Logger logger = LogManager.getLogger(); - - @Override - @StepEntry - public Object[] execute(StoreUri sourceStoreUri, String sourceFileName, StoreUri targetStoreUri, String targetFileName) throws Exception { - Object[] result = null; - Path targetFile = Paths.get(targetStoreUri.toString(true), targetFileName); - if (targetFile.toFile().exists()) { - logger.info(getMarker(), "Using previously retrived file {}", targetFile); - } else { - result = super.execute(sourceStoreUri, sourceFileName, targetStoreUri, targetFileName); - } - return result; - } - -} diff --git a/server/-product/production/AMC/jobs/steps/TSMTransferToStep.java b/server/-product/production/AMC/jobs/steps/TSMTransferToStep.java deleted file mode 100644 index efff055e..00000000 --- a/server/-product/production/AMC/jobs/steps/TSMTransferToStep.java +++ /dev/null @@ -1,11 +0,0 @@ -package user.jobengine.server.steps; - -import user.commons.StoreUri; - -public class TSMTransferToStep extends TransferStep { - @Override - @StepEntry - public Object[] execute(StoreUri sourceStoreUri, String sourceFileName, StoreUri targetStoreUri, String targetFileName) throws Exception { - return super.execute(sourceStoreUri, sourceFileName, targetStoreUri, targetFileName); - } -} diff --git a/server/-product/production/AMC/jobs/steps/TestForkCancelableStep.java b/server/-product/production/AMC/jobs/steps/TestForkCancelableStep.java deleted file mode 100644 index fc434565..00000000 --- a/server/-product/production/AMC/jobs/steps/TestForkCancelableStep.java +++ /dev/null @@ -1,37 +0,0 @@ -package user.jobengine.server.steps; - -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; - -import user.commons.IJob; -import user.commons.JobStatus; -import user.commons.ListUtils; -import user.jobengine.server.IJobEngine; -import user.jobengine.server.IJobRuntime; - -public class TestForkCancelableStep extends JobStep { - private static final String CHILD_TITLE = "Párhuzamosított alfolyamat"; - // private static final String CHILD_TEMPLATE = "fake-concurrent.xml"; - private static final String CHILD_TEMPLATE = "cancelable.xml"; - private static final Logger logger = LogManager.getLogger(); - int count = 1; - - @StepEntry - public Object[] execute(IJobEngine jobEngine, IJobRuntime jobRuntime) throws Exception { - //jobRuntime.forkPrepare(); - for (int i = 0; i < count; i++) { - //jobEngine.submit(jobRuntime, null, CHILD_TEMPLATE, CHILD_TITLE, ListUtils.asMap("itemID", i)); - // IJobRuntime runtime = jobEngine.submit(jobRuntime, null, CHILD_TEMPLATE, "JOB " + i, ListUtils.asMap("param", i)); - IJobRuntime runtime = getEngine().submit(null, e -> { - if (e.getStatus().equals(JobStatus.CANCELED) || e.getStatus().equals(JobStatus.SUSPENDED)) - logger.info("Cleanup occured because status is {} {}", e.getStatus(), ((IJob) e.getSource()).getId()); - }, CHILD_TEMPLATE, "JOB " + i, 0, IJobEngine.DEFAULT_OWNER, ListUtils.asMap("param", i)); - - runtime.setRelated("TEST" + runtime.getId()); - } - - // jobRuntime.forkWaitComplete(); - logger.info("Done"); - return null; - } -} diff --git a/server/-product/production/AMC/jobs/steps/TransferStep.java b/server/-product/production/AMC/jobs/steps/TransferStep.java deleted file mode 100644 index 67cc025c..00000000 --- a/server/-product/production/AMC/jobs/steps/TransferStep.java +++ /dev/null @@ -1,100 +0,0 @@ -package user.jobengine.server.steps; - -import java.nio.file.Path; -import java.nio.file.Paths; - -import org.apache.commons.net.ftp.FTPClient; -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; - -import user.commons.StoreUri; -import user.commons.remotestore.FtpDirectoryLister; -import user.commons.remotestore.RemoteStoreProtocol; -import user.jobengine.db.Store; - -public class TransferStep extends JobStep { - private static final String DOT_PART = ".part"; - private static final Logger logger = LogManager.getLogger(); - - @StepEntry - public Object[] execute(StoreUri sourceStoreUri, String sourceFileName, StoreUri targetStoreUri, String targetFileName) throws Exception { - try { - getJobRuntime().setCancelable(false); - - Store sourceStore = getManager().getStore(sourceStoreUri.getStoreId()); - Store targetStore = getManager().getStore(targetStoreUri.getStoreId()); - - String description = String.format("%s -> %s : %s", sourceStore.getName(), targetStore.getName(), sourceFileName); - getJobRuntime().setDescription(description); - - sourceStoreUri.addProgressListener(e -> setProgress(e.getProgress())); - - StoreUri currentTargetStoreUri = getTargetStoreUri(targetStoreUri); - tryCopy(sourceStoreUri, sourceFileName, currentTargetStoreUri, targetFileName); - } catch (Exception e) { - logger.error(getMarker(), "Error in transfer of {} when copying from {} to {}.", sourceFileName, sourceStoreUri, targetStoreUri); - throw e; - } finally { - if (sourceStoreUri != null) - sourceStoreUri.cleanUp(); - if (targetStoreUri != null) - targetStoreUri.cleanUp(); - } - return null; - } - - protected StoreUri getTargetStoreUri(StoreUri targetStoreUri) { - return targetStoreUri; - } - - protected String getTmpExtension() { - return DOT_PART; - } - - private void tryCopy(StoreUri sourceStoreUri, String sourceFileName, StoreUri targetStoreUri, String targetFileName) throws Exception { - String currentTargetFileName = targetFileName; - - boolean renameAfterCopy = false; - boolean renameAfterFTP = false; - if (getTmpExtension() != null) { - if (RemoteStoreProtocol.LOCAL.equals(targetStoreUri.getProtocol())) { - currentTargetFileName += getTmpExtension(); - renameAfterCopy = true; - } - Store targetStore = getManager().getStore(targetStoreUri.getStoreId()); - - if (RemoteStoreProtocol.FTP.equals(targetStoreUri.getProtocol()) && !"NEXIO1".equals(targetStore.getName()) - && !"NEXIO2".equals(targetStore.getName())) { - currentTargetFileName += getTmpExtension(); - renameAfterFTP = true; - } - } - - sourceStoreUri.transferFrom(targetStoreUri, sourceFileName, currentTargetFileName); - - logger.info(getMarker(), "Transfer of {} completed from {} to {}", sourceFileName, sourceStoreUri, currentTargetFileName); - - if (renameAfterCopy) { - Path tmpTargetFile = Paths.get(targetStoreUri.toString(true), currentTargetFileName); - Path targetFile = Paths.get(targetStoreUri.toString(true), targetFileName); - try { - logger.info(getMarker(), "Renaming LOCAL file from {} to {} on {}", currentTargetFileName, targetFileName, sourceStoreUri); - tmpTargetFile.toFile().renameTo(targetFile.toFile()); - } catch (Exception e) { - logger.error(getMarker(), e.getMessage()); - } - } - if (renameAfterFTP) { - try { - FtpDirectoryLister lister = (FtpDirectoryLister) targetStoreUri.getLister(); - FTPClient client = lister.connect(); - logger.info(getMarker(), "Renaming FTP file from {} to {} on {}", currentTargetFileName, targetFileName, sourceStoreUri); - client.rename(currentTargetFileName, targetFileName); - } catch (Exception e) { - logger.error(getMarker(), e.getMessage()); - } finally { - targetStoreUri.cleanUp(); - } - } - } -} diff --git a/server/-product/production/AMC/jobs/steps/TransferToFTPStep.java b/server/-product/production/AMC/jobs/steps/TransferToFTPStep.java deleted file mode 100644 index b42d5d53..00000000 --- a/server/-product/production/AMC/jobs/steps/TransferToFTPStep.java +++ /dev/null @@ -1,11 +0,0 @@ -package user.jobengine.server.steps; - -import user.commons.StoreUri; - -public class TransferToFTPStep extends TransferStep { - @Override - @StepEntry - public Object[] execute(StoreUri sourceStoreUri, String sourceFileName, StoreUri targetStoreUri, String targetFileName) throws Exception { - return super.execute(sourceStoreUri, sourceFileName, targetStoreUri, targetFileName); - } -} diff --git a/server/-product/production/AMC/jobs/steps/VODTransferToStep.java b/server/-product/production/AMC/jobs/steps/VODTransferToStep.java deleted file mode 100644 index d17eaa1e..00000000 --- a/server/-product/production/AMC/jobs/steps/VODTransferToStep.java +++ /dev/null @@ -1,22 +0,0 @@ -package user.jobengine.server.steps; - -import user.commons.StoreUri; - -public class VODTransferToStep extends TransferStep { - private String relativeTargetPath; - - @StepEntry - public Object[] execute(StoreUri sourceStoreUri, String sourceFileName, StoreUri targetStoreUri, String relativeTargetPath, String targetFileName) - throws Exception { - this.relativeTargetPath = relativeTargetPath; - return super.execute(sourceStoreUri, sourceFileName, targetStoreUri, targetFileName); - } - - @Override - protected StoreUri getTargetStoreUri(StoreUri targetStoreUri) { - StoreUri result = getManager().getStoreUri(targetStoreUri.getId()); - result.setRootPath(relativeTargetPath); - return result; - } - -} diff --git a/server/-product/production/AMC/jobs/templates/FILEZILLA_AVID-validate-and-restore.xml b/server/-product/production/AMC/jobs/templates/FILEZILLA_AVID-validate-and-restore.xml deleted file mode 100644 index 5395ce44..00000000 --- a/server/-product/production/AMC/jobs/templates/FILEZILLA_AVID-validate-and-restore.xml +++ /dev/null @@ -1,95 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - \ No newline at end of file diff --git a/server/-product/production/AMC/jobs/templates/FILEZILLA_PASARESTORE-validate-and-archive.xml b/server/-product/production/AMC/jobs/templates/FILEZILLA_PASARESTORE-validate-and-archive.xml deleted file mode 100644 index 7caa7e2c..00000000 --- a/server/-product/production/AMC/jobs/templates/FILEZILLA_PASARESTORE-validate-and-archive.xml +++ /dev/null @@ -1,94 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - \ No newline at end of file diff --git a/server/-product/production/AMC/jobs/templates/FILEZILLA_PASARESTORE-validate-and-restore.xml b/server/-product/production/AMC/jobs/templates/FILEZILLA_PASARESTORE-validate-and-restore.xml deleted file mode 100644 index da82bd6d..00000000 --- a/server/-product/production/AMC/jobs/templates/FILEZILLA_PASARESTORE-validate-and-restore.xml +++ /dev/null @@ -1,95 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - \ No newline at end of file diff --git a/server/-product/production/AMC/jobs/templates/NEXIO1-validate-and-archive.xml b/server/-product/production/AMC/jobs/templates/NEXIO1-validate-and-archive.xml deleted file mode 100644 index 7426382a..00000000 --- a/server/-product/production/AMC/jobs/templates/NEXIO1-validate-and-archive.xml +++ /dev/null @@ -1,94 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - \ No newline at end of file diff --git a/server/-product/production/AMC/jobs/templates/NEXIO1-validate-and-restore.xml b/server/-product/production/AMC/jobs/templates/NEXIO1-validate-and-restore.xml deleted file mode 100644 index 5b11ead5..00000000 --- a/server/-product/production/AMC/jobs/templates/NEXIO1-validate-and-restore.xml +++ /dev/null @@ -1,95 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - \ No newline at end of file diff --git a/server/-product/production/AMC/jobs/templates/NEXIO2-validate-and-archive.xml b/server/-product/production/AMC/jobs/templates/NEXIO2-validate-and-archive.xml deleted file mode 100644 index 005122a1..00000000 --- a/server/-product/production/AMC/jobs/templates/NEXIO2-validate-and-archive.xml +++ /dev/null @@ -1,94 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - \ No newline at end of file diff --git a/server/-product/production/AMC/jobs/templates/NEXIO2-validate-and-restore.xml b/server/-product/production/AMC/jobs/templates/NEXIO2-validate-and-restore.xml deleted file mode 100644 index 15b0e18b..00000000 --- a/server/-product/production/AMC/jobs/templates/NEXIO2-validate-and-restore.xml +++ /dev/null @@ -1,95 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - \ No newline at end of file diff --git a/server/-product/production/AMC/jobs/templates/PASAPOOL-validate-and-restore.xml b/server/-product/production/AMC/jobs/templates/PASAPOOL-validate-and-restore.xml deleted file mode 100644 index df9ebe1d..00000000 --- a/server/-product/production/AMC/jobs/templates/PASAPOOL-validate-and-restore.xml +++ /dev/null @@ -1,95 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - \ No newline at end of file diff --git a/server/-product/production/AMC/jobs/templates/PB-archive-checker.xml b/server/-product/production/AMC/jobs/templates/PB-archive-checker.xml deleted file mode 100644 index 54d36065..00000000 --- a/server/-product/production/AMC/jobs/templates/PB-archive-checker.xml +++ /dev/null @@ -1,26 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - \ No newline at end of file diff --git a/server/-product/production/AMC/jobs/templates/PBARCHIVE-validate-and-archive.xml b/server/-product/production/AMC/jobs/templates/PBARCHIVE-validate-and-archive.xml deleted file mode 100644 index 2b4a7d71..00000000 --- a/server/-product/production/AMC/jobs/templates/PBARCHIVE-validate-and-archive.xml +++ /dev/null @@ -1,94 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - \ No newline at end of file diff --git a/server/-product/production/AMC/jobs/templates/VOD-validate-and-restore.xml b/server/-product/production/AMC/jobs/templates/VOD-validate-and-restore.xml deleted file mode 100644 index 5cdcadb1..00000000 --- a/server/-product/production/AMC/jobs/templates/VOD-validate-and-restore.xml +++ /dev/null @@ -1,89 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - \ No newline at end of file diff --git a/server/-product/production/AMC/jobs/templates/cancelable.xml b/server/-product/production/AMC/jobs/templates/cancelable.xml deleted file mode 100644 index fa29cc3f..00000000 --- a/server/-product/production/AMC/jobs/templates/cancelable.xml +++ /dev/null @@ -1,17 +0,0 @@ - - - - - - - - - - - - - - - - - \ No newline at end of file diff --git a/server/-product/production/AMC/jobs/templates/create-lowres-ondemand.xml b/server/-product/production/AMC/jobs/templates/create-lowres-ondemand.xml deleted file mode 100644 index 375beb2f..00000000 --- a/server/-product/production/AMC/jobs/templates/create-lowres-ondemand.xml +++ /dev/null @@ -1,106 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - \ No newline at end of file diff --git a/server/-product/production/AMC/jobs/templates/fork-validate-and-archive.xml b/server/-product/production/AMC/jobs/templates/fork-validate-and-archive.xml deleted file mode 100644 index 5136c36d..00000000 --- a/server/-product/production/AMC/jobs/templates/fork-validate-and-archive.xml +++ /dev/null @@ -1,29 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - \ No newline at end of file diff --git a/server/-product/production/AMC/jobs/templates/fork-validate-and-restore.xml b/server/-product/production/AMC/jobs/templates/fork-validate-and-restore.xml deleted file mode 100644 index bddbab68..00000000 --- a/server/-product/production/AMC/jobs/templates/fork-validate-and-restore.xml +++ /dev/null @@ -1,29 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - \ No newline at end of file diff --git a/server/-product/production/AMC/jobs/templates/generic-archive-checker.xml b/server/-product/production/AMC/jobs/templates/generic-archive-checker.xml deleted file mode 100644 index 500369e8..00000000 --- a/server/-product/production/AMC/jobs/templates/generic-archive-checker.xml +++ /dev/null @@ -1,34 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - \ No newline at end of file diff --git a/server/-product/production/AMC/jobs/templates/harris1-missingmaterial-checker.xml b/server/-product/production/AMC/jobs/templates/harris1-missingmaterial-checker.xml deleted file mode 100644 index 5777fa2e..00000000 --- a/server/-product/production/AMC/jobs/templates/harris1-missingmaterial-checker.xml +++ /dev/null @@ -1,30 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - \ No newline at end of file diff --git a/server/-product/production/AMC/jobs/templates/harris2-missingmaterial-checker.xml b/server/-product/production/AMC/jobs/templates/harris2-missingmaterial-checker.xml deleted file mode 100644 index 5777fa2e..00000000 --- a/server/-product/production/AMC/jobs/templates/harris2-missingmaterial-checker.xml +++ /dev/null @@ -1,30 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - \ No newline at end of file diff --git a/server/-product/production/AMC/jobs/templates/nexio1-archive-checker.xml b/server/-product/production/AMC/jobs/templates/nexio1-archive-checker.xml deleted file mode 100644 index 991c0626..00000000 --- a/server/-product/production/AMC/jobs/templates/nexio1-archive-checker.xml +++ /dev/null @@ -1,26 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - \ No newline at end of file diff --git a/server/-product/production/AMC/jobs/templates/nexio2-archive-checker.xml b/server/-product/production/AMC/jobs/templates/nexio2-archive-checker.xml deleted file mode 100644 index 991c0626..00000000 --- a/server/-product/production/AMC/jobs/templates/nexio2-archive-checker.xml +++ /dev/null @@ -1,26 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - \ No newline at end of file diff --git a/server/-product/production/AMC/jobs/templates/peablebeach-missingmaterial-checker.xml b/server/-product/production/AMC/jobs/templates/peablebeach-missingmaterial-checker.xml deleted file mode 100644 index b05b2ef8..00000000 --- a/server/-product/production/AMC/jobs/templates/peablebeach-missingmaterial-checker.xml +++ /dev/null @@ -1,58 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - \ No newline at end of file diff --git a/server/-product/production/AMC/jobs/templates/register-user-restore.xml b/server/-product/production/AMC/jobs/templates/register-user-restore.xml deleted file mode 100644 index 43a94d22..00000000 --- a/server/-product/production/AMC/jobs/templates/register-user-restore.xml +++ /dev/null @@ -1,29 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - \ No newline at end of file diff --git a/server/-product/production/AMC/jobs/templates/register-vod-restore.xml b/server/-product/production/AMC/jobs/templates/register-vod-restore.xml deleted file mode 100644 index ac996108..00000000 --- a/server/-product/production/AMC/jobs/templates/register-vod-restore.xml +++ /dev/null @@ -1,21 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - \ No newline at end of file diff --git a/server/-product/production/AMC/jobs/templates/server-status-report.xml b/server/-product/production/AMC/jobs/templates/server-status-report.xml deleted file mode 100644 index eec3da65..00000000 --- a/server/-product/production/AMC/jobs/templates/server-status-report.xml +++ /dev/null @@ -1,6 +0,0 @@ - - - - - - \ No newline at end of file diff --git a/server/-product/production/AMC/jobs/templates/sync-subtitles.xml b/server/-product/production/AMC/jobs/templates/sync-subtitles.xml deleted file mode 100644 index 49c35cb9..00000000 --- a/server/-product/production/AMC/jobs/templates/sync-subtitles.xml +++ /dev/null @@ -1,21 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - \ No newline at end of file diff --git a/server/-product/production/AMC/jobs/templates/test-fork-cancelable.xml b/server/-product/production/AMC/jobs/templates/test-fork-cancelable.xml deleted file mode 100644 index 8c88d84b..00000000 --- a/server/-product/production/AMC/jobs/templates/test-fork-cancelable.xml +++ /dev/null @@ -1,6 +0,0 @@ - - - - - - \ No newline at end of file diff --git a/server/-product/production/AMC/jobs/templates/validate-dir-mxf.xml b/server/-product/production/AMC/jobs/templates/validate-dir-mxf.xml deleted file mode 100644 index 2a2972be..00000000 --- a/server/-product/production/AMC/jobs/templates/validate-dir-mxf.xml +++ /dev/null @@ -1,17 +0,0 @@ - - - - - - - - - - - - - - - - - \ No newline at end of file diff --git a/server/-product/production/AMC/settings/application.yaml b/server/-product/production/AMC/settings/application.yaml deleted file mode 100644 index 12ff38b7..00000000 --- a/server/-product/production/AMC/settings/application.yaml +++ /dev/null @@ -1,34 +0,0 @@ -datasource: - mediacube: - url: jdbc:db2://192.168.0.16:50000/mc - user: db2admin - password: password - external-indexer: false - simple-search: true - login-timeout: 3 - pool-size: 10 - mediacube-nosql: - url: jdbc:db2://192.168.0.16:50000/mc - user: db2admin - password: password - schema: test - login-timeout: 3 -services: - ffmpeg: - execurable-location: /opt/ffmpeg/ffmpeg - mediacube: - proxy-root: /opt - nexio: - disabled: true -jobs: - validate-transfers: false - copy-buffer-size: 32768 - scheduled-execution-disabled: true -tsm: - randomize-archives: false - delimiter: \ - node-name: PASANODE - fs-name: NEXIO - alternate-fs-name: PASA - hl-name: \ - \ No newline at end of file diff --git a/server/-product/production/AMC/settings/jetty.xml b/server/-product/production/AMC/settings/jetty.xml deleted file mode 100644 index 5e47cacd..00000000 --- a/server/-product/production/AMC/settings/jetty.xml +++ /dev/null @@ -1,45 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - \ No newline at end of file diff --git a/server/-product/production/AMC/settings/log4j2.xml b/server/-product/production/AMC/settings/log4j2.xml deleted file mode 100644 index 3681b48f..00000000 --- a/server/-product/production/AMC/settings/log4j2.xml +++ /dev/null @@ -1,174 +0,0 @@ - - - - log - ${logPath}/mediacube.log - ${logPath}/$${date:yyyy-MM}/mediacube-%d{MM-dd-yyyy}-%i.log.gz - ${logPath}/markered-mediacube.log - ${logPath}/$${date:yyyy-MM}/markered-mediacube-%d{MM-dd-yyyy}-%i.log.gz - ${logPath}/mediacube-err.log - ${logPath}/$${date:yyyy-MM}/mediacube-err-%d{MM-dd-yyyy}-%i.log.gz - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - diff --git a/server/-product/production/AMC/settings/maestro.yaml b/server/-product/production/AMC/settings/maestro.yaml deleted file mode 100644 index 1d361571..00000000 --- a/server/-product/production/AMC/settings/maestro.yaml +++ /dev/null @@ -1,38 +0,0 @@ -sourceStoreUri: - name: Default - protocol: LOCAL - uri: "/mnt/NLE" - fileFilter: "*.mxf" - showDirectories: true -alternateSourceStoreUris: -- name: NLE1 - protocol: LOCAL - uri: "/mnt/NLE/NLE1" - fileFilter: "*.mxf" - showDirectories: true -- name: NLE2 - protocol: LOCAL - uri: "/mnt/NLE/NLE2" - fileFilter: "*.mxf" - showDirectories: true -- name: NLE3 - protocol: LOCAL - uri: "/mnt/NLE/NLE3" - fileFilter: "*.mxf" - showDirectories: true -- name: NLE4 - protocol: LOCAL - uri: "/mnt/NLE/NLE4" - fileFilter: "*.mxf" - showDirectories: true -- name: POLC - protocol: LOCAL - uri: "/mnt/POLC" - fileFilter: "*.mxf" - showDirectories: true -targets: -- name: FINISHED_SHOWS - killDateDays: 7 - storeUri: - protocol: LOCAL - uri: "/mnt/PROMISE/FINISHED_SHOWS" diff --git a/server/-product/production/AMC/settings/mediacube.yaml b/server/-product/production/AMC/settings/mediacube.yaml deleted file mode 100644 index fcf8e5c7..00000000 --- a/server/-product/production/AMC/settings/mediacube.yaml +++ /dev/null @@ -1,57 +0,0 @@ -jobQueuePollInterval: 1000 -disableHelp: true -maestroDisabled: false -alternateRetrieveSelector: false -disableStatistics: true -disableEditor: false -targetRestoreFilters: -- FILEZILLA_AVID -- FILEZILLA_PASARESTORE -topTypeFilters: -- name: Hír bejátszó -- name: Hír nyers -- name: Visszarögzített -- name: Egyéb -bottomTypeFilters: -- name: Műsor -- name: Műsor nyers -- name: Promo -- name: Promo nyers -- name: Reklám -- name: Reklám nyers -authentication: - authEnabled: true - defaultUser: root - defaultPassword: password - adHost: intra.amc.hu - adNonSecurePort: 3268 - adBaseDn: DC=intra,DC=amc,DC=hu - adAdminMap: - - INFORMATIKUSOK - - MUSZAKVEZETOK - adSubmitterMap: - - INFORMATIKUSOK - - MUSZAKVEZETOK - adEditorMap: - - INFORMATIKUSOK - - MUSZAKVEZETOK - localAccounts: - - user: user - password: 5F4DCC3B5AA765D61D8327DEB882CF99 - email: - - user: user1 - password: 5F4DCC3B5AA765D61D8327DEB882CF99 - email: - - user: lebony - password: 4E25B117B14D86D7DCECB4E433CF932C - email: - - user: root - password: 5F4DCC3B5AA765D61D8327DEB882CF99 - email: vasary@elgekko.net - localAdmins: - - root - - admin - localSubmitters: - - lebony - localEditors: - - editor