From: vasary.daniel Date: Mon, 16 Nov 2020 15:35:20 +0000 (+0000) Subject: git-tfs-id: [http://tfs.userrendszerhaz.hu:8080/tfs/DefaultCollection]$/MediaCube... X-Git-Url: http://git.useribm.hu/?a=commitdiff_plain;h=50ad328108524b07839296a5f054d7cc948bc1c6;p=mediacube.git git-tfs-id: [tfs.userrendszerhaz.hu:8080/tfs/DefaultCollection]$/MediaCube;C32054 --- diff --git a/client/Maestro/Configuration/configuration.json b/client/Maestro/Configuration/configuration.json index 49934f32..32edff1a 100644 --- a/client/Maestro/Configuration/configuration.json +++ b/client/Maestro/Configuration/configuration.json @@ -11,12 +11,12 @@ }, "source": { "$type": "UNCSource", - "filter": "avi,wav,mxf,mts", - "useAlternateCopy": true, + "filter": "avi,wav,mxf,mts,mov", + "useAlternateCopy": false, "autoTarget": "AUTOTARGET", "autoTags": [ "AUTOTAG" ], "local": { - "address": "file://c:/_XDCAM/Clip", + "address": "file://c:/_video/x", "userName": "mediacube", "password": "Dn8t4gfHcK98o8hyPgLDhr5SgSji4JCxsfpMJsODikUp3nXgrM0UNCi45lLAK8ZOnmEneO44P9qpJ4QDqhctN6MxZodjJgdZTyoZKmSa+ECzEzLr/wPYNgxVaXrVotEy", "timeout": 1000 @@ -24,42 +24,24 @@ }, "metadatas": [ { - "$type": "OctopusMetadata", + "$type": "MediaCubeMetadata", "server": { - "address": "http://10.10.1.27/services/rest/octopus", - "timeout": 5000 - } - }, - { - "$type": "TrafficMetadata", - "server": { - "address": "Data Source=10.10.1.45;Initial Catalog=PA_Echo;Persist Security Info=True;", - "userName": "MAM", - "password": "7RKZYBzumKjL40SJwuwiFCvX57xuCN8zay6OttUm2wbrgImyYZBHyZTUUYrXX31Ge2Uwew07HYsqh2uzdJeDBDwcVntxaHg3nIpv9Dyq/odVoiC4tUF/K+lgvKWANcrZ", + "address": "http://localhost:8888/services/rest/jobengine", + "userName": "mediacube", + "password": "Dn8t4gfHcK98o8hyPgLDhr5SgSji4JCxsfpMJsODikUp3nXgrM0UNCi45lLAK8ZOnmEneO44P9qpJ4QDqhctN6MxZodjJgdZTyoZKmSa+ECzEzLr/wPYNgxVaXrVotEy", "timeout": 1000 }, - "version": 0, - "redefineSegments": true - }, - { - "$type": "MediaCubeMetadata", - "server": { - "address": "http://localhost:8888/services/rest/jobengine", - "userName": "mediacube", - "password": "Dn8t4gfHcK98o8hyPgLDhr5SgSji4JCxsfpMJsODikUp3nXgrM0UNCi45lLAK8ZOnmEneO44P9qpJ4QDqhctN6MxZodjJgdZTyoZKmSa+ECzEzLr/wPYNgxVaXrVotEy", - "timeout": 1000 - }, - "wsserver": { - "address": "ws://10.10.1.27/services/wsapi", - "timeout": 1000 - }, - "jobTemplate": "retrieve-material.xml", - "archiveFolder": "file://10.10.1.105/TESZT/TC/ARCHIVE", - "restoreFolder": "file://10.10.1.105/BRAAVOS/ARCHIVE_RESTORE", - "restoreNamePattern": "%s_%GUID%", - "serverRestoreFolder": "/mnt/ISILON/ARCHIVE_RESTORE", - "killDateDays": 1 - } + "wsserver": { + "address": "ws://10.10.1.27/services/wsapi", + "timeout": 1000 + }, + "jobTemplate": "retrieve-material.xml", + "archiveFolder": "file://10.10.1.105/TESZT/TC/ARCHIVE", + "restoreFolder": "file://10.10.1.105/BRAAVOS/ARCHIVE_RESTORE", + "restoreNamePattern": "%s_%GUID%", + "serverRestoreFolder": "/mnt/ISILON/ARCHIVE_RESTORE", + "killDateDays": 1 + } ], "targets": [ { @@ -106,7 +88,7 @@ "killDateDays": 0, "saveArchiveMetadata": true, "remote": { - "address": "file://c:/opt1", + "address": "file://c:/_video/y", "userName": "mediacube", "password": "Dn8t4gfHcK98o8hyPgLDhr5SgSji4JCxsfpMJsODikUp3nXgrM0UNCi45lLAK8ZOnmEneO44P9qpJ4QDqhctN6MxZodjJgdZTyoZKmSa+ECzEzLr/wPYNgxVaXrVotEy", "timeout": 1000 diff --git a/client/MaestroShared/Commons/CustomFileCopier.cs b/client/MaestroShared/Commons/CustomFileCopier.cs index 45a0767f..690159b5 100644 --- a/client/MaestroShared/Commons/CustomFileCopier.cs +++ b/client/MaestroShared/Commons/CustomFileCopier.cs @@ -42,6 +42,9 @@ namespace MaestroShared.Commons { break; } } + + //kell ? + dest.Flush(); } } diff --git a/server/-configuration/log4j2.xml b/server/-configuration/log4j2.xml index 7153ef31..6d1a7865 100644 --- a/server/-configuration/log4j2.xml +++ b/server/-configuration/log4j2.xml @@ -20,7 +20,7 @@ + smtpPort="465" smtpProtocol="smtps" smtpDebug="false" smtpUsername="mediacubeserver@gmail.com" smtpPassword="salabakter" ignoreExceptions="false" bufferSize="1"> @@ -54,7 +54,7 @@ - + diff --git a/server/-configuration/scheduledjobs.json b/server/-configuration/scheduledjobs.json index 3f606aa9..8a4caaa7 100644 --- a/server/-configuration/scheduledjobs.json +++ b/server/-configuration/scheduledjobs.json @@ -22,6 +22,7 @@ ] }, { + "name" : "Megszakítható teszt folyamat", "template": "cancelable.xml", "parameters": [ {"name": "param", "value": 1, "type": "java.lang.Integer"} ] }, diff --git a/server/user.jobengine.executors/config/config-worker.xml b/server/user.jobengine.executors/config/config-worker.xml index 97f24d87..438def00 100644 --- a/server/user.jobengine.executors/config/config-worker.xml +++ b/server/user.jobengine.executors/config/config-worker.xml @@ -1,5 +1,6 @@ + diff --git a/server/user.jobengine.executors/jobtemplates/cancelable.xml b/server/user.jobengine.executors/jobtemplates/cancelable.xml index fa29cc3f..2be289b3 100644 --- a/server/user.jobengine.executors/jobtemplates/cancelable.xml +++ b/server/user.jobengine.executors/jobtemplates/cancelable.xml @@ -1,5 +1,5 @@ - + diff --git a/server/user.jobengine.executors/jobtemplates/sync-subtitles.xml b/server/user.jobengine.executors/jobtemplates/sync-subtitles.xml new file mode 100644 index 00000000..37e26483 --- /dev/null +++ b/server/user.jobengine.executors/jobtemplates/sync-subtitles.xml @@ -0,0 +1,21 @@ + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/server/user.jobengine.executors/src/user/jobengine/server/steps/CancelableStep.java b/server/user.jobengine.executors/src/user/jobengine/server/steps/CancelableStep.java index ea5d7bb9..0f21e2ef 100644 --- a/server/user.jobengine.executors/src/user/jobengine/server/steps/CancelableStep.java +++ b/server/user.jobengine.executors/src/user/jobengine/server/steps/CancelableStep.java @@ -1,29 +1,46 @@ package user.jobengine.server.steps; +import java.nio.file.Paths; +import java.util.Arrays; +import java.util.List; + import org.apache.commons.net.ftp.FTPClient; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; +import org.apache.logging.log4j.MarkerManager; +import user.commons.MediaCubeMarker; +import user.commons.RemoteFile; import user.commons.StoreUri; +import user.commons.mediatool.MediaInfo; import user.commons.remotestore.FtpDirectoryLister; +import user.commons.remotestore.IProgressEventListener; +import user.commons.remotestore.IStatusEventListener; +import user.commons.remotestore.ProgressEvent; import user.commons.remotestore.RemoteStoreProtocol; +import user.commons.remotestore.StatusEvent; public class CancelableStep extends JobStep { private static final Logger logger = LogManager.getLogger(); int count = 10; @StepEntry - public Object[] execute(int param) throws Exception { + public Object[] execute(int z) throws Exception { try { + // retrieveTest(); // ftpTest(); + ((MediaCubeMarker) getJobRuntime().getFinishMarker()).setTo("hering.tamas70@gmail.com"); for (int i = 0; i < count; i++) { if (getJobRuntime().isWaitingCancel()) break; - Thread.sleep(500); + Thread.sleep(100); int progress = (i + 1) * 100 / count; setProgress(progress); + logger.info(getSessionMarker(), "Current {}", i); } + + logger.info(getSessionMarker(), "Finished"); } catch (Exception e) { e.printStackTrace(); throw e; @@ -54,4 +71,48 @@ public class CancelableStep extends JobStep { } + void retrieveTest() { + String fn = null; + try { + List files = Arrays.asList("tLkaTXxW-M19-033419-1.MOV", "8RkqDrZF-M19-033418-1.MOV", "VnNflPem-M19-033415-1.MOV", + "OMMnXmXJ-M19-033416-1.MOV", "XKssDE0h-M19-033414-1-1.MOV", "o1PxGST2-M20-033486-3.MOV", "2aeKEvqG-M20-034175-1.MOV", + "eSlsY8zT-M20-034176-1.MOV", "sRhREmVR-M20-034177-1.MOV", "d3J3Tk21-M20-034177-1-1.MOV", "pTZDiXtO-M20-034178-1.MOV", + "7fdHizJG-M20-034179-1.MOV", "JGrkQ81d-M20-034156-1.MOV", "4K96kKNR-M20-034157-1.MOV", "MoTdzwAC-M20-034158-1.MOV", + "73uQdsE8-M20-034159-1.MOV", "TzcGEbpq-M20-034180-1.MOV", "G9mwxqMR-M20-034181-1.MOV", "C8UsIr45-M20-034182-1.MOV", + "n3bM1gns-M20-034161-1.MOV", "xYfCbTz4-M20-034160-1.MOV", "f0rkowuI-M20-034184-1.MOV", "HDAVOaZn-M20-034185-1.MOV", + "INxFHTy9-M20-034183-1.MOV", "cTiYIIa2-M20-035223-1.MOV", "YSiugJ2e-M20-035403-1.MOV", "Z4mhB0Hl-M20-035224-1.MOV", + "cBx8vwYP-M20-035402-1.MOV", "WNAunGsh-M20-035401-1.MOV", "4bJHH7dF-M20-035400-1.MOV", "whBVXEyl-M20-035217-1.MOV", + "pZJvUP4d-M20-035220-1.MOV", "fOp1jsww-M20-035407-1.MOV", "VIXrcyMD-M20-035406-1.MOV", "AkHIZveh-M20-035221-1.MOV", + "MEihdEMW-M20-035219-1.MOV", "H56Um2nU-M20-035405-1.MOV", "XZ7nysN8-M20-035222-1.MOV", "oCm9rtI2-M20-035218-1.MOV", + "fqkbYhxZ-M19-033363-1.mov", "aBaPHRqa-M19-033413-1.mov"); + + for (String fileName : files) { + StoreUri sourceUri = getManager().getStoreUri("TSM", RemoteStoreProtocol.TSM); + StoreUri targetUri = getManager().getStoreUri("Low-res", RemoteStoreProtocol.LOCAL); + targetUri.setRootPath("/mediacube/data"); + sourceUri.addProgressListener(new IProgressEventListener() { + @Override + public void progressChanged(ProgressEvent evt) { + //logger.info(evt.getProgress()); + } + }); + sourceUri.addStatusListener(new IStatusEventListener() { + @Override + public void statusChanged(StatusEvent evt) { + evt.setCancel(!canContinue()); + } + }); + fn = fileName.substring(9); + RemoteFile result = sourceUri.transferFrom(targetUri, fileName, fn); + + MediaInfo mi = new MediaInfo(Paths.get(targetUri.toString(true), fn)); + mi.process(); + logger.info(MarkerManager.getMarker("MOVTEST"), "Az {} állomány viszatöltve, hossz {}, audio {}", fn, mi.getFrames(), mi.getAudioStreams()); + } + } catch (Exception e) { + logger.info(MarkerManager.getMarker("MOVTEST"), "Az {} állomány viszatöltése sikertelen. {}", fn, e.getMessage()); + } + + } + } diff --git a/server/user.jobengine.executors/src/user/jobengine/server/steps/DirMXFValidatorStep.java b/server/user.jobengine.executors/src/user/jobengine/server/steps/DirMXFValidatorStep.java index 104806e4..bfb608c6 100644 --- a/server/user.jobengine.executors/src/user/jobengine/server/steps/DirMXFValidatorStep.java +++ b/server/user.jobengine.executors/src/user/jobengine/server/steps/DirMXFValidatorStep.java @@ -8,7 +8,10 @@ import java.nio.file.Paths; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; +import user.commons.RemoteFile; +import user.commons.StoreUri; import user.commons.mediatool.MediaInfo; +import user.commons.remotestore.RemoteStoreProtocol; import user.jobengine.db.Media; public class DirMXFValidatorStep extends JobStep { @@ -16,6 +19,14 @@ public class DirMXFValidatorStep extends JobStep { @StepEntry public Object[] execute(String source) throws Exception { + + logger.info("get"); + StoreUri storeUri = getManager().getStoreUri("NEXIO2", RemoteStoreProtocol.FTP); + RemoteFile f = storeUri.getRemoteFile("DANITESZT02.mxf"); + logger.info("get done {}", f); + if (f != null) + return null; + Path path = Paths.get(source); DirectoryStream directoryStream = Files.newDirectoryStream(path); for (Path file : directoryStream) { @@ -32,17 +43,16 @@ public class DirMXFValidatorStep extends JobStep { fileName = fileName.substring(0, fileName.lastIndexOf(".")); if (file.toFile().length() == 0) { - logger.error("Zero: {}", fileName); + logger.error("Zero length: {}", fileName); continue; } Media media = getManager().getMedia(fileName); + long frames = 0; if (media == null) { - logger.error("Unchecked: {}", fileName); - continue; - } - - long frames = media.getLength(); + logger.warn("Unchecked yet: {}", fileName); + } else + frames = media.getLength(); MediaInfo mi = null; @@ -54,14 +64,7 @@ public class DirMXFValidatorStep extends JobStep { logger.error("Unreadable: {}", fileName); } - if (frames == 0) { - logger.info("Ma valid: {}", fileName); - continue; - } - - if (Math.abs(frames - mi.getFrames()) < 4) { - logger.info("Valid: {} {} {}", fileName, frames, mi.getFrames()); - } + logger.info("{}: MC farmes {}, FF frames {}", fileName, frames, mi.getFrames()); } return null; diff --git a/server/user.jobengine.executors/src/user/jobengine/server/steps/SyncSubtitlesStep.java b/server/user.jobengine.executors/src/user/jobengine/server/steps/SyncSubtitlesStep.java new file mode 100644 index 00000000..83b14dbc --- /dev/null +++ b/server/user.jobengine.executors/src/user/jobengine/server/steps/SyncSubtitlesStep.java @@ -0,0 +1,91 @@ +package user.jobengine.server.steps; + +import java.io.IOException; +import java.io.RandomAccessFile; +import java.nio.channels.FileLock; +import java.nio.channels.OverlappingFileLockException; +import java.nio.file.FileVisitResult; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.nio.file.SimpleFileVisitor; +import java.nio.file.attribute.BasicFileAttributes; +import java.util.ArrayList; +import java.util.List; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + +import user.commons.StoreUri; +import user.commons.remotestore.RemoteStoreProtocol; + +public class SyncSubtitlesStep extends JobStep { + private static final String EXT_STL = ".stl"; + private static final Logger logger = LogManager.getLogger(); + int count = 10; + + @StepEntry + public Object[] execute(String sourceStoreName, String targetStoreName) throws Exception { + try { + StoreUri sourceStoreUri = getManager().getStoreUri(sourceStoreName, RemoteStoreProtocol.LOCAL); + StoreUri targetStoreUri = getManager().getStoreUri(targetStoreName, RemoteStoreProtocol.LOCAL); + Path sourcePath = Paths.get(sourceStoreUri.toString(true)); + Path targetPath = Paths.get(targetStoreUri.toString(true)); + List files = new ArrayList<>(); + + Files.walkFileTree(sourcePath, new SimpleFileVisitor() { + @Override + public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) throws IOException { + if (file.getFileName().toString().toLowerCase().endsWith(EXT_STL)) { + files.add(file); + } + if (getJobRuntime().isWaitingCancel()) + return FileVisitResult.TERMINATE; + return super.visitFile(file, attrs); + } + }); + + if (getJobRuntime().isWaitingCancel()) + return null; + + int currentCount = 0; + for (Path file : files) { + Path targetFile = Paths.get(targetPath.toString(), file.getFileName().toString()); + + boolean delete = false; + try (RandomAccessFile reader = new RandomAccessFile(file.toString(), "rw")) { + FileLock lock = reader.getChannel().lock(); + byte[] content = new byte[(int) file.toFile().length()]; + reader.read(content); + Files.write(targetFile, content); + lock.release(); + delete = true; + } catch (OverlappingFileLockException e) { + logger.warn(getSessionMarker(), "File is locked {}" + file); + } catch (Exception e) { + logger.warn(getSessionMarker(), e.getMessage()); + } + + if (delete && file.toFile().length() == targetFile.toFile().length()) { + try { + Files.delete(file); + } catch (Exception e) { + logger.error(getSessionMarker(), e.getMessage()); + } + } + + currentCount++; + int progress = currentCount * 100 / files.size(); + setProgress(progress); + if (getJobRuntime().isWaitingCancel()) + break; + } + + } catch (Exception e) { + logger.error(getSessionMarker(), e.getMessage()); + throw e; + } + return null; + } + +} diff --git a/server/user.jobengine.osgi.commons/src/user/commons/MediaCubeFinishMarker.java b/server/user.jobengine.osgi.commons/src/user/commons/MediaCubeFinishMarker.java index afe5f677..a0bee642 100644 --- a/server/user.jobengine.osgi.commons/src/user/commons/MediaCubeFinishMarker.java +++ b/server/user.jobengine.osgi.commons/src/user/commons/MediaCubeFinishMarker.java @@ -7,6 +7,7 @@ public class MediaCubeFinishMarker extends MediaCubeMarker { setSessionID(parent.getSessionID()); setSessionName(parent.getSessionName()); setTo(parent.getTo()); + setUseSessionLog(parent.isUseSessionLog()); } public MediaCubeFinishMarker(MediaCubeMarker parent, String to) { diff --git a/server/user.jobengine.osgi.commons/src/user/commons/mediatool/MediaInfo.java b/server/user.jobengine.osgi.commons/src/user/commons/mediatool/MediaInfo.java index 2baef985..7a60acfa 100644 --- a/server/user.jobengine.osgi.commons/src/user/commons/mediatool/MediaInfo.java +++ b/server/user.jobengine.osgi.commons/src/user/commons/mediatool/MediaInfo.java @@ -61,6 +61,7 @@ public class MediaInfo { demuxer.open(filePath.toString(), null, false, true, null, null); bitRate = demuxer.getBitRate(); + int numStreams = demuxer.getNumStreams(); frames = 0; int videoStreamId = -1; diff --git a/server/user.jobengine.osgi.server/test/user/jobengine/server/IT/AMCTests.java b/server/user.jobengine.osgi.server/test/user/jobengine/server/IT/AMCTests.java index b74a6555..72726c4e 100644 --- a/server/user.jobengine.osgi.server/test/user/jobengine/server/IT/AMCTests.java +++ b/server/user.jobengine.osgi.server/test/user/jobengine/server/IT/AMCTests.java @@ -4,7 +4,9 @@ import static org.junit.Assert.assertEquals; import static org.junit.Assert.assertNotNull; import java.io.FileInputStream; +import java.io.RandomAccessFile; import java.net.URL; +import java.nio.channels.FileLock; import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.Paths; @@ -61,111 +63,94 @@ public class AMCTests { } @Test - public void testNexioArchiveChecker() throws Exception { - //Tesztelni a HirTV-n lehet + public void testForkValidateAndArchive() throws Exception { + //Tesztelni egy helyi FTP-n lehet + Path statusPath = Paths.get(output, ".STATUS"); + Files.createDirectories(statusPath); - jobEngine = new JobEngine(); - jobEngine.startup(); - jobEngine.bindItemManagerService(manager); + StoreUri sourceStoreUri = manager.ensureUri("Local FTP", "localhost", false, false, RemoteStoreProtocol.FTP); + StoreUri targetStoreUri = manager.ensureUri("TSM", "xxx", true, false, RemoteStoreProtocol.TSM); + DownloadableMedia downloadable = new DownloadableMedia(); + String fileName = "HOSSZ_TEST.MXF"; + long frames = 24; + Path escortFile = Paths.get(statusPath.toString(), "Local FTP.HOSSZ_TEST.MXF.json"); - String sourceStoreName = "NEXIO"; - manager.ensureUri(sourceStoreName, System.getProperty("nexio.host"), false, false, RemoteStoreProtocol.NEXIO); + downloadable.put("fileName", fileName); + downloadable.put("frames", frames); + downloadable.put("size", 0); + downloadable.put("storeUriId", sourceStoreUri.getId()); - Files.createDirectories(Paths.get(output)); + Files.write(escortFile, downloadable.toPrettyString("").getBytes()); - Map jobParams = new HashMap<>(); - jobParams.put("sourceStoreName", sourceStoreName); - jobParams.put("escortStoreName", "LOCAL"); - jobParams.put("limit", 1); + String childTemplate = "validate-and-archive.xml"; CountDownLatch finishLatch = new CountDownLatch(1); - IJobRuntime jobRuntime = jobEngine.submit(null, e -> { - if (e.isRuntimeTerminated()) - finishLatch.countDown(); - }, "nexio-archive-checker.xml", "NEXIO archive check", jobParams); - if (jobRuntime != null) - finishLatch.await(); - - FileUtils.deleteDirectory(Paths.get(output).toFile()); - jobEngine.shutdown(); - - assertNotNull(jobRuntime); - assertEquals(JobStatus.FINISHED, jobRuntime.getStatus()); - - } - - @Test - public void testGenericArchiveChecker() throws Exception { - //Tesztelni egy helyi FTP-n lehet + jobEngine = new JobEngine() { + @Override + public IJobRuntime submit(String template, String name, Map parameters) throws JobEngineException { + if (childTemplate.equals(template)) { + assertEquals(sourceStoreUri.getId(), ((StoreUri) parameters.get("sourceStoreUri")).getId()); + assertEquals(tempUri.getId(), ((StoreUri) parameters.get("tempStoreUri")).getId()); + assertEquals(targetStoreUri.getId(), ((StoreUri) parameters.get("targetStoreUri")).getId()); + assertEquals(fileName, parameters.get("sourceFileName")); + assertEquals(frames, parameters.get("expectedFrameNumber")); + assertEquals(0L, parameters.get("expectedSize")); + assertEquals(escortFile.toAbsolutePath().toString(), parameters.get("escortFile")); + finishLatch.countDown(); + return new JobRuntime(); + } + return super.submit(template, name, parameters); + } - jobEngine = new JobEngine(); + }; jobEngine.startup(); jobEngine.bindItemManagerService(manager); - String sourceStoreName = "Local FTP"; - StoreUri storeUri = manager.ensureUri(sourceStoreName, "localhost", false, false, RemoteStoreProtocol.FTP); - if (!"dani".equals(storeUri.getUserName())) { - storeUri.setUserName("dani"); - storeUri.setPassword("dani"); - storeUri.modify(); - } - - Files.createDirectories(Paths.get(output)); - Map jobParams = new HashMap<>(); - jobParams.put("sourceStoreName", sourceStoreName); - jobParams.put("protocol", "FTP"); - jobParams.put("wildCard", ".mxf|.mov"); - jobParams.put("escortStoreName", "LOCAL"); + jobParams.put("sourceStoreName", "LOCAL"); + jobParams.put("targetStoreName", "TSM"); + jobParams.put("targetProtocol", "TSM"); + jobParams.put("template", childTemplate); jobParams.put("limit", 1); - CountDownLatch finishLatch = new CountDownLatch(1); IJobRuntime jobRuntime = jobEngine.submit(null, e -> { - if (e.isRuntimeTerminated()) - finishLatch.countDown(); - }, "generic-archive-checker.xml", "Generic archive check", jobParams); + }, "fork-validate-and-archive.xml", "Fork validate and archive", jobParams); if (jobRuntime != null) finishLatch.await(); FileUtils.deleteDirectory(Paths.get(output).toFile()); jobEngine.shutdown(); - - assertNotNull(jobRuntime); - assertEquals(JobStatus.FINISHED, jobRuntime.getStatus()); + //assertEquals(jobRuntime.getStatus()) } @Test - public void testForkValidateAndArchive() throws Exception { - //Tesztelni egy helyi FTP-n lehet + public void testForkValidateAndRestore() throws Exception { Path statusPath = Paths.get(output, ".STATUS"); Files.createDirectories(statusPath); - StoreUri sourceStoreUri = manager.ensureUri("Local FTP", "localhost", false, false, RemoteStoreProtocol.FTP); - StoreUri targetStoreUri = manager.ensureUri("TSM", "xxx", true, false, RemoteStoreProtocol.TSM); + StoreUri videoStoreUri = manager.ensureUri("TEST SOURCE", "/_video", false, false, RemoteStoreProtocol.LOCAL); + StoreUri targetStoreUri = manager.ensureUri("TEST TARGET", output, false, false, RemoteStoreProtocol.LOCAL); DownloadableMedia downloadable = new DownloadableMedia(); String fileName = "HOSSZ_TEST.MXF"; long frames = 24; - Path escortFile = Paths.get(statusPath.toString(), "Local FTP.HOSSZ_TEST.MXF.json"); + Path escortFile = Paths.get(statusPath.toString(), "NEXIO.HOSSZ_TEST.MXF.json"); downloadable.put("fileName", fileName); downloadable.put("frames", frames); downloadable.put("size", 0); - downloadable.put("storeUriId", sourceStoreUri.getId()); - + downloadable.put("storeUriId", videoStoreUri.getId()); Files.write(escortFile, downloadable.toPrettyString("").getBytes()); - String childTemplate = "validate-and-archive.xml"; - CountDownLatch finishLatch = new CountDownLatch(1); - + String childTemplate = "validate-and-restore.xml"; jobEngine = new JobEngine() { @Override public IJobRuntime submit(String template, String name, Map parameters) throws JobEngineException { if (childTemplate.equals(template)) { - assertEquals(sourceStoreUri.getId(), ((StoreUri) parameters.get("sourceStoreUri")).getId()); + assertEquals(videoStoreUri.getId(), ((StoreUri) parameters.get("sourceStoreUri")).getId()); assertEquals(tempUri.getId(), ((StoreUri) parameters.get("tempStoreUri")).getId()); assertEquals(targetStoreUri.getId(), ((StoreUri) parameters.get("targetStoreUri")).getId()); assertEquals(fileName, parameters.get("sourceFileName")); @@ -184,13 +169,15 @@ public class AMCTests { Map jobParams = new HashMap<>(); jobParams.put("sourceStoreName", "LOCAL"); - jobParams.put("targetStoreName", "TSM"); - jobParams.put("targetProtocol", "TSM"); + jobParams.put("targetStoreName", "TEST TARGET"); + jobParams.put("targetProtocol", "LOCAL"); jobParams.put("template", childTemplate); jobParams.put("limit", 1); IJobRuntime jobRuntime = jobEngine.submit(null, e -> { - }, "fork-validate-and-archive.xml", "Fork validate and archive", jobParams); + // if (e.isRuntimeTerminated()) + // finishLatch.countDown(); + }, "fork-validate-and-restore.xml", "Fork validate and archive", jobParams); if (jobRuntime != null) finishLatch.await(); @@ -202,55 +189,80 @@ public class AMCTests { } @Test - public void testValidateAndArchive() throws Exception { - Path statusPath = Paths.get(output, ".STATUS"); - Files.createDirectories(statusPath); - String fileName = "HOSSZ_TEST.MXF"; - Path escortFile = Paths.get(statusPath.toString(), "Local FTP." + fileName + ".json"); + public void testGenericArchiveChecker() throws Exception { + //Tesztelni egy helyi FTP-n lehet - StoreUri storeUri = manager.ensureUri("Local FTP", "localhost", false, false, RemoteStoreProtocol.FTP); + jobEngine = new JobEngine(); + jobEngine.startup(); + jobEngine.bindItemManagerService(manager); + + String sourceStoreName = "Local FTP"; + StoreUri storeUri = manager.ensureUri(sourceStoreName, "localhost", false, false, RemoteStoreProtocol.FTP); if (!"dani".equals(storeUri.getUserName())) { storeUri.setUserName("dani"); storeUri.setPassword("dani"); storeUri.modify(); } - StoreUri tempUri = manager.ensureUri("LOCAL", output, false, false, RemoteStoreProtocol.LOCAL); - StoreUri targetStoreUri = manager.ensureUri("TEST TARGET", output + "/target", false, false, RemoteStoreProtocol.LOCAL); - DownloadableMedia downloadable = new DownloadableMedia(); - long frames = 514; - downloadable.put("title", fileName); - downloadable.put("fileName", fileName); - downloadable.put("frames", frames); - downloadable.put("size", 0L); - downloadable.put("storeUriId", storeUri.getId()); - Files.write(escortFile, downloadable.toPrettyString("").getBytes()); + Files.createDirectories(Paths.get(output)); + + Map jobParams = new HashMap<>(); + jobParams.put("sourceStoreName", sourceStoreName); + jobParams.put("protocol", "FTP"); + jobParams.put("wildCard", ".mxf|.mov"); + jobParams.put("escortStoreName", "LOCAL"); + jobParams.put("limit", 1); + + CountDownLatch finishLatch = new CountDownLatch(1); + IJobRuntime jobRuntime = jobEngine.submit(null, e -> { + if (e.isRuntimeTerminated()) + finishLatch.countDown(); + }, "generic-archive-checker.xml", "Generic archive check", jobParams); + + if (jobRuntime != null) + finishLatch.await(); + + FileUtils.deleteDirectory(Paths.get(output).toFile()); + jobEngine.shutdown(); + + assertNotNull(jobRuntime); + assertEquals(JobStatus.FINISHED, jobRuntime.getStatus()); + + } + + @Test + public void testNexioArchiveChecker() throws Exception { + //Tesztelni a HirTV-n lehet jobEngine = new JobEngine(); jobEngine.startup(); jobEngine.bindItemManagerService(manager); + String sourceStoreName = "NEXIO"; + manager.ensureUri(sourceStoreName, System.getProperty("nexio.host"), false, false, RemoteStoreProtocol.NEXIO); + + Files.createDirectories(Paths.get(output)); + Map jobParams = new HashMap<>(); - jobParams.put("sourceStoreUri", storeUri); - jobParams.put("tempStoreUri", tempUri); - jobParams.put("targetStoreUri", targetStoreUri); - jobParams.put("fileName", fileName); - jobParams.put("expectedFrameNumber", frames); - jobParams.put("expectedSize", 0L); - //jobParams.put("expectedColorSpace", null); - jobParams.put("escortFile", escortFile.toAbsolutePath().toString()); - CountDownLatch finishLatch = new CountDownLatch(1); + jobParams.put("sourceStoreName", sourceStoreName); + jobParams.put("escortStoreName", "LOCAL"); + jobParams.put("limit", 1); + CountDownLatch finishLatch = new CountDownLatch(1); IJobRuntime jobRuntime = jobEngine.submit(null, e -> { if (e.isRuntimeTerminated()) finishLatch.countDown(); - }, "validate-and-archive.xml", "Validate and archive", jobParams); + }, "nexio-archive-checker.xml", "NEXIO archive check", jobParams); if (jobRuntime != null) finishLatch.await(); FileUtils.deleteDirectory(Paths.get(output).toFile()); jobEngine.shutdown(); + + assertNotNull(jobRuntime); + assertEquals(JobStatus.FINISHED, jobRuntime.getStatus()); + } @Test @@ -307,65 +319,91 @@ public class AMCTests { } @Test - public void testForkValidateAndRestore() throws Exception { + public void testSyncSubtitles() throws Exception { + jobEngine = new JobEngine(); + jobEngine.startup(); + jobEngine.bindItemManagerService(manager); + + String sourceStoreName = "SUB-IN"; + manager.ensureUri(sourceStoreName, "/SUB-IN", false, false, RemoteStoreProtocol.LOCAL); + String targetStoreName = "SUB-OUT"; + manager.ensureUri(targetStoreName, "/SUB-OUT", false, false, RemoteStoreProtocol.LOCAL); + + Map jobParams = new HashMap<>(); + jobParams.put("sourceStoreName", sourceStoreName); + jobParams.put("targetStoreName", targetStoreName); + + CountDownLatch finishLatch = new CountDownLatch(1); + IJobRuntime jobRuntime = null; + + try (RandomAccessFile reader = new RandomAccessFile("/SUB-IN/BUL/CCEM100323-01.stl", "rw")) { + FileLock lock = reader.getChannel().lock(); + + jobRuntime = jobEngine.submit(null, e -> { + if (e.isRuntimeTerminated()) + finishLatch.countDown(); + }, "sync-subtitles.xml", "Sync subtitles", jobParams); + + if (jobRuntime != null) + finishLatch.await(); + + lock.release(); + } + + jobEngine.shutdown(); + assertEquals(JobStatus.FINISHED, jobRuntime.getStatus()); + } + + @Test + public void testValidateAndArchive() throws Exception { Path statusPath = Paths.get(output, ".STATUS"); Files.createDirectories(statusPath); + String fileName = "HOSSZ_TEST.MXF"; + Path escortFile = Paths.get(statusPath.toString(), "Local FTP." + fileName + ".json"); - StoreUri videoStoreUri = manager.ensureUri("TEST SOURCE", "/_video", false, false, RemoteStoreProtocol.LOCAL); - StoreUri targetStoreUri = manager.ensureUri("TEST TARGET", output, false, false, RemoteStoreProtocol.LOCAL); + StoreUri storeUri = manager.ensureUri("Local FTP", "localhost", false, false, RemoteStoreProtocol.FTP); + if (!"dani".equals(storeUri.getUserName())) { + storeUri.setUserName("dani"); + storeUri.setPassword("dani"); + storeUri.modify(); + } + StoreUri tempUri = manager.ensureUri("LOCAL", output, false, false, RemoteStoreProtocol.LOCAL); + StoreUri targetStoreUri = manager.ensureUri("TEST TARGET", output + "/target", false, false, RemoteStoreProtocol.LOCAL); DownloadableMedia downloadable = new DownloadableMedia(); - String fileName = "HOSSZ_TEST.MXF"; - long frames = 24; - Path escortFile = Paths.get(statusPath.toString(), "NEXIO.HOSSZ_TEST.MXF.json"); + long frames = 514; + downloadable.put("title", fileName); downloadable.put("fileName", fileName); downloadable.put("frames", frames); - downloadable.put("size", 0); - downloadable.put("storeUriId", videoStoreUri.getId()); + downloadable.put("size", 0L); + downloadable.put("storeUriId", storeUri.getId()); Files.write(escortFile, downloadable.toPrettyString("").getBytes()); - CountDownLatch finishLatch = new CountDownLatch(1); - String childTemplate = "validate-and-restore.xml"; - jobEngine = new JobEngine() { - @Override - public IJobRuntime submit(String template, String name, Map parameters) throws JobEngineException { - if (childTemplate.equals(template)) { - assertEquals(videoStoreUri.getId(), ((StoreUri) parameters.get("sourceStoreUri")).getId()); - assertEquals(tempUri.getId(), ((StoreUri) parameters.get("tempStoreUri")).getId()); - assertEquals(targetStoreUri.getId(), ((StoreUri) parameters.get("targetStoreUri")).getId()); - assertEquals(fileName, parameters.get("sourceFileName")); - assertEquals(frames, parameters.get("expectedFrameNumber")); - assertEquals(0L, parameters.get("expectedSize")); - assertEquals(escortFile.toAbsolutePath().toString(), parameters.get("escortFile")); - finishLatch.countDown(); - return new JobRuntime(); - } - return super.submit(template, name, parameters); - } - - }; + jobEngine = new JobEngine(); jobEngine.startup(); jobEngine.bindItemManagerService(manager); Map jobParams = new HashMap<>(); - jobParams.put("sourceStoreName", "LOCAL"); - jobParams.put("targetStoreName", "TEST TARGET"); - jobParams.put("targetProtocol", "LOCAL"); - jobParams.put("template", childTemplate); - jobParams.put("limit", 1); + jobParams.put("sourceStoreUri", storeUri); + jobParams.put("tempStoreUri", tempUri); + jobParams.put("targetStoreUri", targetStoreUri); + jobParams.put("fileName", fileName); + jobParams.put("expectedFrameNumber", frames); + jobParams.put("expectedSize", 0L); + //jobParams.put("expectedColorSpace", null); + jobParams.put("escortFile", escortFile.toAbsolutePath().toString()); + CountDownLatch finishLatch = new CountDownLatch(1); IJobRuntime jobRuntime = jobEngine.submit(null, e -> { - // if (e.isRuntimeTerminated()) - // finishLatch.countDown(); - }, "fork-validate-and-restore.xml", "Fork validate and archive", jobParams); + if (e.isRuntimeTerminated()) + finishLatch.countDown(); + }, "validate-and-archive.xml", "Validate and archive", jobParams); if (jobRuntime != null) finishLatch.await(); FileUtils.deleteDirectory(Paths.get(output).toFile()); jobEngine.shutdown(); - //assertEquals(jobRuntime.getStatus()) - } @Test