From ca193aabc8cab02c2f3e2aa099175250548e0c27 Mon Sep 17 00:00:00 2001 From: "vasary.daniel" Date: Wed, 26 May 2021 19:02:52 +0000 Subject: [PATCH] git-tfs-id: [http://tfs.userrendszerhaz.hu:8080/tfs/DefaultCollection]$/MediaCube;C32229 --- .../LOCAL/jobs/steps/shared/MetadataType.java | 5 + .../steps/shared/MetadataTypeDetector.java | 49 ++++++ .../LOCAL/jobs/steps/shared/TestLib.java | 9 ++ .../LOCAL/jobs/steps/shared/TestLib1.java | 8 + server/user.jobengine.executors/.classpath | 13 +- server/user.jobengine.executors/.project | 6 + .../org.eclipse.core.resources.prefs | 2 + .../.settings/org.eclipse.jdt.core.prefs | 4 + .../.settings/org.eclipse.m2e.core.prefs | 4 + server/user.jobengine.executors/pom.xml | 8 +- .../server/steps/CancelableStep.java | 69 ++------ .../steps/CleanupMountedLocationStep.java | 39 +---- .../jobengine/server/steps/DeleteFile.java | 22 +++ .../server/steps/DeleteFileStep.java | 7 +- .../server/steps/IntegrationTestStep.java | 121 +++++++++----- .../jobengine/server/steps/MediaToolStep.java | 22 +-- .../server/steps/PrepareMediaRestoreStep.java | 7 +- ...epareRemoteTranscodeJobParametersStep.java | 32 ---- .../steps/PrepareRemoteTranscodeStep.java | 38 +++++ .../steps/QueryMissingProxyMediaStep.java | 30 ++-- .../jobengine/server/steps/RemoteJobStep.java | 44 +++--- .../server/steps/SaveMediaProxy.java | 22 +++ .../jobengine/server/steps/TSMBackupStep.java | 10 +- .../server/steps/TSMRestoreStep.java | 8 +- .../server/steps/TranscodeFFMpegStep.java | 39 ++--- .../server/steps/shared/EscortFiles.java | 10 +- .../steps/shared/ItemManagerExtensions.java | 40 ++++- .../steps/shared/MetadataTypeDetector.java | 12 +- .../user/commons/MediaCubeFinishMarker.java | 2 + .../remotestore/FtpDirectoryLister.java | 8 +- .../remotestore/LocalDirectoryLister.java | 13 +- .../META-INF/MANIFEST.MF | 4 + .../src/user/jobengine/server/IJobEngine.java | 6 +- .../src/user/jobengine/server/JobEngine.java | 32 ++-- .../server/JobEngineConfiguration.java | 148 ++++++++++-------- .../jobengine/server/JobStepExecutor.java | 26 ++- .../user/jobengine/server/steps/JobStep.java | 7 + .../pages/jobparameters.zul | 14 +- .../user/jobengine/zk/model/JobListModel.java | 6 +- .../jobengine/zk/model/JobSelectorModel.java | 28 +++- .../user/jobengine/zk/util/SessionUtil.java | 6 +- 41 files changed, 599 insertions(+), 381 deletions(-) create mode 100644 server/-product/production/LOCAL/jobs/steps/shared/MetadataType.java create mode 100644 server/-product/production/LOCAL/jobs/steps/shared/MetadataTypeDetector.java create mode 100644 server/-product/production/LOCAL/jobs/steps/shared/TestLib.java create mode 100644 server/-product/production/LOCAL/jobs/steps/shared/TestLib1.java create mode 100644 server/user.jobengine.executors/.settings/org.eclipse.core.resources.prefs create mode 100644 server/user.jobengine.executors/.settings/org.eclipse.m2e.core.prefs create mode 100644 server/user.jobengine.executors/src/user/jobengine/server/steps/DeleteFile.java delete mode 100644 server/user.jobengine.executors/src/user/jobengine/server/steps/PrepareRemoteTranscodeJobParametersStep.java create mode 100644 server/user.jobengine.executors/src/user/jobengine/server/steps/PrepareRemoteTranscodeStep.java create mode 100644 server/user.jobengine.executors/src/user/jobengine/server/steps/SaveMediaProxy.java diff --git a/server/-product/production/LOCAL/jobs/steps/shared/MetadataType.java b/server/-product/production/LOCAL/jobs/steps/shared/MetadataType.java new file mode 100644 index 00000000..0e27bf34 --- /dev/null +++ b/server/-product/production/LOCAL/jobs/steps/shared/MetadataType.java @@ -0,0 +1,5 @@ +package user.jobengine.server.steps.shared; + +public enum MetadataType { + TrafficMaterial, TrafficPromo, TrafficAD, OctopusStory, OctopusPlaceholder, Generic +} diff --git a/server/-product/production/LOCAL/jobs/steps/shared/MetadataTypeDetector.java b/server/-product/production/LOCAL/jobs/steps/shared/MetadataTypeDetector.java new file mode 100644 index 00000000..9e9e6be9 --- /dev/null +++ b/server/-product/production/LOCAL/jobs/steps/shared/MetadataTypeDetector.java @@ -0,0 +1,49 @@ +package user.jobengine.server.steps.shared; + +import org.apache.commons.lang.StringUtils; + +public class MetadataTypeDetector { + + private static final String HYPHEN = "-"; + private static final String DOT = "."; + + private static final String REGEXP_TRAFFICMATERIALID = "^M{1}[0-9]{6}[A-Z]{1}"; + private static final String REGEXP_TRAFFICADID = "^R{1}[0-9]{6}[A-Z]{1}"; + private static final String REGEXP_TRAFFICPROMOID = "^P{1}[0-9]{6}[A-Z]{1}"; + private static final String REGEXP_OCTOPUSSTORYID = "^[0-9]+"; + private static final String REGEXP_OCTOPUSPLACEHOLDERID = "^[0-9]+_[0-9]+"; + private static final String REGEXP_OCTOPUSPLACEHOLDERVERSIONEDID = "^[0-9]+_[0-9]+-[0-9]{3}"; + + public static MetadataType GuessMetadataType(String id) { + if (StringUtils.isBlank(id)) + return MetadataType.Generic; + if (id.matches(REGEXP_TRAFFICMATERIALID)) + return MetadataType.TrafficMaterial; + if (id.matches(REGEXP_TRAFFICPROMOID)) + return MetadataType.TrafficPromo; + if (id.matches(REGEXP_TRAFFICADID)) + return MetadataType.TrafficAD; + if (id.matches(REGEXP_OCTOPUSSTORYID)) + return MetadataType.OctopusStory; + if (id.matches(REGEXP_OCTOPUSPLACEHOLDERID)) + return MetadataType.OctopusPlaceholder; + if (id.matches(REGEXP_OCTOPUSPLACEHOLDERVERSIONEDID)) + return MetadataType.OctopusPlaceholder; + return MetadataType.Generic; + } + + public static String truncateExtension(String name) { + String result = name; + if (result != null && result.contains(DOT)) + result = result.substring(0, result.lastIndexOf(DOT)); + return result; + } + + public static String truncateVersion(String name) { + String result = name; + if (result != null && result.contains(HYPHEN)) + result = result.split(HYPHEN)[0]; + return result; + } + +} diff --git a/server/-product/production/LOCAL/jobs/steps/shared/TestLib.java b/server/-product/production/LOCAL/jobs/steps/shared/TestLib.java new file mode 100644 index 00000000..d12738ae --- /dev/null +++ b/server/-product/production/LOCAL/jobs/steps/shared/TestLib.java @@ -0,0 +1,9 @@ +package user.jobengine.server.steps.shared; + +public class TestLib { + + public void hello() { + TestLib1 lib = new TestLib1(); + lib.helo(); + } +} diff --git a/server/-product/production/LOCAL/jobs/steps/shared/TestLib1.java b/server/-product/production/LOCAL/jobs/steps/shared/TestLib1.java new file mode 100644 index 00000000..99d47e5f --- /dev/null +++ b/server/-product/production/LOCAL/jobs/steps/shared/TestLib1.java @@ -0,0 +1,8 @@ +package user.jobengine.server.steps.shared; + +public class TestLib1 { + + public void hello() { + System.out.println("Hello from lib1"); + } +} diff --git a/server/user.jobengine.executors/.classpath b/server/user.jobengine.executors/.classpath index 48482498..5e01dfaf 100644 --- a/server/user.jobengine.executors/.classpath +++ b/server/user.jobengine.executors/.classpath @@ -2,6 +2,15 @@ - - + + + + + + + + + + + diff --git a/server/user.jobengine.executors/.project b/server/user.jobengine.executors/.project index 1a808f87..65e764cf 100644 --- a/server/user.jobengine.executors/.project +++ b/server/user.jobengine.executors/.project @@ -30,8 +30,14 @@ + + org.eclipse.m2e.core.maven2Builder + + + + org.eclipse.m2e.core.maven2Nature org.eclipse.jdt.core.javanature org.eclipse.pde.PluginNature diff --git a/server/user.jobengine.executors/.settings/org.eclipse.core.resources.prefs b/server/user.jobengine.executors/.settings/org.eclipse.core.resources.prefs new file mode 100644 index 00000000..4824b802 --- /dev/null +++ b/server/user.jobengine.executors/.settings/org.eclipse.core.resources.prefs @@ -0,0 +1,2 @@ +eclipse.preferences.version=1 +encoding/=UTF-8 diff --git a/server/user.jobengine.executors/.settings/org.eclipse.jdt.core.prefs b/server/user.jobengine.executors/.settings/org.eclipse.jdt.core.prefs index ace45cee..39e9a954 100644 --- a/server/user.jobengine.executors/.settings/org.eclipse.jdt.core.prefs +++ b/server/user.jobengine.executors/.settings/org.eclipse.jdt.core.prefs @@ -8,5 +8,9 @@ org.eclipse.jdt.core.compiler.debug.lineNumber=generate org.eclipse.jdt.core.compiler.debug.localVariable=generate org.eclipse.jdt.core.compiler.debug.sourceFile=generate org.eclipse.jdt.core.compiler.problem.assertIdentifier=error +org.eclipse.jdt.core.compiler.problem.enablePreviewFeatures=disabled org.eclipse.jdt.core.compiler.problem.enumIdentifier=error +org.eclipse.jdt.core.compiler.problem.forbiddenReference=warning +org.eclipse.jdt.core.compiler.problem.reportPreviewFeatures=ignore +org.eclipse.jdt.core.compiler.release=disabled org.eclipse.jdt.core.compiler.source=1.8 diff --git a/server/user.jobengine.executors/.settings/org.eclipse.m2e.core.prefs b/server/user.jobengine.executors/.settings/org.eclipse.m2e.core.prefs new file mode 100644 index 00000000..14b697b7 --- /dev/null +++ b/server/user.jobengine.executors/.settings/org.eclipse.m2e.core.prefs @@ -0,0 +1,4 @@ +activeProfiles= +eclipse.preferences.version=1 +resolveWorkspaceProjects=true +version=1 diff --git a/server/user.jobengine.executors/pom.xml b/server/user.jobengine.executors/pom.xml index 33493a7a..7e76498a 100644 --- a/server/user.jobengine.executors/pom.xml +++ b/server/user.jobengine.executors/pom.xml @@ -9,9 +9,15 @@ 4.0.0 + user.jobengine user.jobengine.executors - eclipse-plugin 1.0.0 + + 1.8 + 1.8 + UTF-8 + UTF-8 + diff --git a/server/user.jobengine.executors/src/user/jobengine/server/steps/CancelableStep.java b/server/user.jobengine.executors/src/user/jobengine/server/steps/CancelableStep.java index f90415ce..287a2650 100644 --- a/server/user.jobengine.executors/src/user/jobengine/server/steps/CancelableStep.java +++ b/server/user.jobengine.executors/src/user/jobengine/server/steps/CancelableStep.java @@ -1,72 +1,23 @@ package user.jobengine.server.steps; -import org.apache.commons.net.ftp.FTPClient; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; -import org.apache.logging.log4j.Marker; -import org.apache.logging.log4j.MarkerManager; - -import user.commons.StoreUri; -import user.commons.remotestore.FtpDirectoryLister; -import user.commons.remotestore.RemoteStoreProtocol; -import user.jobengine.server.steps.shared.EscortFiles; public class CancelableStep extends JobStep { private static final Logger logger = LogManager.getLogger(); - int count = 10; @StepEntry - public Object[] execute(int param) throws Exception { - try { - - Marker marker = MarkerManager.getMarker("MEDIAPROFILE"); - logger.info(marker, "{}", EscortFiles.DOT_CATCHED); - getJobRuntime().setRelated("TESZT" + param); - - // ftpTest(); - - // if (param == 0) - // Thread.sleep(5000); - // if (param == 1) - // throw new Exception("Error teszt"); - - for (int i = 0; i < count; i++) { - if (getJobRuntime().isWaitingCancel()) - break; - - Thread.sleep(200); - int progress = (i + 1) * 100 / count; - setProgress(progress); - logger.info(marker, "{}", i); - } - } catch (Exception e) { - e.printStackTrace(); - throw e; + public Object[] execute() throws Exception { + logger.info(getMarker(), "Executing"); + for (int i = 0; i < 10; i++) { + logger.info(getMarker(), "Executing {}", i); + if (getJobRuntime().isWaitingCancel()) + break; + Thread.sleep(100); + int progress = (i + 1) * 100 / 10; + setProgress(progress); } - return new Object[] { param }; - } - - void ftpTest() throws Exception { - StoreUri source = null; - StoreUri target = null; - try { - source = getManager().getStoreUri("MEDIACUBE_UPLOADS", RemoteStoreProtocol.LOCAL); - target = getManager().getStoreUri("NEXIO1", RemoteStoreProtocol.FTP); - source.addProgressListener(e -> setProgress(e.getProgress())); - logger.info("Uploading file PART_TEST.mxf"); - source.transferFrom(target, "PART_TEST.mxf", "PART_TEST.mxf"); - FTPClient client = ((FtpDirectoryLister) target.getLister()).connect(); - logger.info("Renaming file PART_TEST.mxf to RENAME_TEST.mxf"); - client.rename("PART_TEST.mxf", "RENAME_TEST.mxf"); - } catch (Exception e) { - logger.catching(e); - } finally { - if (target != null) - target.cleanUp(); - if (source != null) - source.cleanUp(); - } - + return null; } } diff --git a/server/user.jobengine.executors/src/user/jobengine/server/steps/CleanupMountedLocationStep.java b/server/user.jobengine.executors/src/user/jobengine/server/steps/CleanupMountedLocationStep.java index 6a1d37a1..3ed0e4f4 100644 --- a/server/user.jobengine.executors/src/user/jobengine/server/steps/CleanupMountedLocationStep.java +++ b/server/user.jobengine.executors/src/user/jobengine/server/steps/CleanupMountedLocationStep.java @@ -23,10 +23,7 @@ import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.apache.logging.log4j.Marker; -import user.commons.RemoteFile; -import user.commons.StoreUri; -import user.commons.remotestore.RemoteStoreProtocol; -import user.jobengine.db.Store; +import user.jobengine.server.steps.shared.ItemManagerExtensions; public class CleanupMountedLocationStep extends JobStep implements FileVisitor { @@ -66,7 +63,6 @@ public class CleanupMountedLocationStep extends JobStep implements FileVisitor

killDateFiles) { @@ -97,14 +93,6 @@ public class CleanupMountedLocationStep extends JobStep implements FileVisitor

{ - tsmName[0] = rs.getString("relativepath"); - return false; - }, null); - - if (tsmName[0] != null) { - try { - RemoteFile remoteFile = tsmStoreUri.getRemoteFile(tsmName[0]); - result = remoteFile != null; - } catch (Exception e) { - logger.error(e.getMessage()); - } finally { - tsmStoreUri.cleanUp(); - } - } - return result; - } - @Override public FileVisitResult postVisitDirectory(Path dir, IOException exc) throws IOException { if (!dir.equals(sourcePath) && isEmpty(dir)) { @@ -247,7 +212,7 @@ public class CleanupMountedLocationStep extends JobStep implements FileVisitor

0) { - if (!isArchived(filePath)) { + if (!ItemManagerExtensions.isArchived(getManager(), filePath)) { logger.error(marker, "A(z) {} anyag törlésre van kijelölve, de nem található az archívumban.", filePath); return; } diff --git a/server/user.jobengine.executors/src/user/jobengine/server/steps/DeleteFile.java b/server/user.jobengine.executors/src/user/jobengine/server/steps/DeleteFile.java new file mode 100644 index 00000000..539faf74 --- /dev/null +++ b/server/user.jobengine.executors/src/user/jobengine/server/steps/DeleteFile.java @@ -0,0 +1,22 @@ +package user.jobengine.server.steps; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + +import user.commons.RemoteFile; +import user.commons.StoreUri; + +public class DeleteFile extends JobStep { + private static final Logger logger = LogManager.getLogger(); + + @StepEntry + public Object[] execute(String fileName, StoreUri tempStoreUri) throws Exception { + try { + RemoteFile remoteFile = tempStoreUri.getRemoteFile(fileName); + tempStoreUri.delete(remoteFile); + } catch (Exception e) { + logger.warn(getJobRuntime().getSessionMarker(), e.getMessage()); + } + return null; + } +} diff --git a/server/user.jobengine.executors/src/user/jobengine/server/steps/DeleteFileStep.java b/server/user.jobengine.executors/src/user/jobengine/server/steps/DeleteFileStep.java index 7682440f..79734f57 100644 --- a/server/user.jobengine.executors/src/user/jobengine/server/steps/DeleteFileStep.java +++ b/server/user.jobengine.executors/src/user/jobengine/server/steps/DeleteFileStep.java @@ -8,16 +8,13 @@ import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.apache.logging.log4j.Marker; -import user.jobengine.server.IJobEngine; -import user.jobengine.server.IJobRuntime; - public class DeleteFileStep extends JobStep { private static final Logger logger = LogManager.getLogger(); private Marker marker; @StepEntry - public Object[] execute(ArchiveItem archiveItem, boolean isDelete, IJobEngine jobEngine, IJobRuntime jobRuntime) throws Exception { - marker = jobRuntime.getSessionMarker(); + public Object[] execute(ArchiveItem archiveItem, boolean isDelete) throws Exception { + marker = getJobRuntime().getSessionMarker(); if (isDelete) { Path filePath = Paths.get(archiveItem.getMediaFile()); File file = filePath.toFile(); diff --git a/server/user.jobengine.executors/src/user/jobengine/server/steps/IntegrationTestStep.java b/server/user.jobengine.executors/src/user/jobengine/server/steps/IntegrationTestStep.java index 51bc6103..317edfed 100644 --- a/server/user.jobengine.executors/src/user/jobengine/server/steps/IntegrationTestStep.java +++ b/server/user.jobengine.executors/src/user/jobengine/server/steps/IntegrationTestStep.java @@ -1,5 +1,6 @@ package user.jobengine.server.steps; +import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.Paths; import java.util.concurrent.CountDownLatch; @@ -11,71 +12,117 @@ import user.commons.ListUtils; import user.jobengine.db.ArchivedMedia; import user.jobengine.db.Item; import user.jobengine.db.Media; +import user.jobengine.server.steps.shared.ItemManagerExtensions; public class IntegrationTestStep extends JobStep { private static final Logger logger = LogManager.getLogger(); + private ArchivedMedia archivedMedia; - @StepEntry - public Object[] execute() throws Exception { - try { - //test_cancelable(finishLatch); - ArchivedMedia archivedMedia = new ArchivedMedia(); - Media media = getManager().getMedia(15285); - Item item = getManager().getItem(media.getItemId()); - archivedMedia.setItem(item); - archivedMedia.setMedia(media); - test_retrieve_ondemand(archivedMedia); - Path output = Paths.get("/mediacube/data", archivedMedia.getMedia().getHouseId(), - archivedMedia.getMedia().getHouseId() + "-ARCH-" + archivedMedia.getMedia().getMediaFileRealName()); - if (!output.toFile().exists()) - throw new Exception("File not exists: " + output); - - test_delete_materials(output.getParent().toString()); - - if (output.toFile().exists()) - throw new Exception("File exists: " + output); - - } catch (Exception e) { - logger.info(e.getMessage()); - throw e; - } - return null; - } - - private void test_cancelable() throws Exception { + private void _00_test_cancelable() throws Exception { CountDownLatch finishLatch = new CountDownLatch(1); getEngine().submit(null, e -> { if (e.isRuntimeTerminated()) finishLatch.countDown(); }, "cancelable.xml", "Test cancelable", ListUtils.asMap("param", 1)); finishLatch.await(); + logger.info("_00_test_cancelable SUCCESS"); } - private void test_delete_materials(String source) throws Exception { + private void _01_test_retrieve_ondemand() throws Exception { + archivedMedia = new ArchivedMedia(); + Media media = getManager().getMedia(15285); + Item item = getManager().getItem(media.getItemId()); + archivedMedia.setItem(item); + archivedMedia.setMedia(media); + CountDownLatch finishLatch = new CountDownLatch(1); getEngine().submit(null, e -> { if (e.isRuntimeTerminated()) finishLatch.countDown(); - }, "delete-materials.xml", "Test delete-materials", ListUtils.asMap("sourcePath", source, "skipArchiveCheck", true)); + + }, "retrieve-ondemand.xml", "Test retrieve-ondemand", + ListUtils.asMap("globalRetrievePath", "file://10.11.1.100", "localRetrievePath", "/mediacube/data", "materialOutputFolder", "/", + "promoOutputFolder", "/", "advertisementOutputFolder", "/", "octopusOutputFolder", "/", "genericOutputFolder", "/", + "onlineOutputFolder", "/", "killDateDays", -1, "nexioAgency", "ARCHIVE_RESTORE", "nexioPort", 2098, "nexioUserName", "administrator", + "nexioPassword", "system", "archivedMedia", archivedMedia, "successRecipient", "vasary@elgekko.net", "houseId", + archivedMedia.getMedia().getHouseId(), "targetPathType", "0")); + finishLatch.await(); + Path output = Paths.get("/mediacube/data", archivedMedia.getMedia().getHouseId(), + archivedMedia.getMedia().getHouseId() + "-ARCH-" + archivedMedia.getMedia().getMediaFileRealName()); + if (!output.toFile().exists()) + throw new Exception("File not exists: " + output); + + logger.info("_01_test_retrieve_ondemand SUCCESS"); } - private void test_retrieve_ondemand(ArchivedMedia archivedMedia) throws Exception { + private void _02_test_archive_material() throws Exception { + Path input = Paths.get("/mediacube/data", archivedMedia.getMedia().getHouseId(), + archivedMedia.getMedia().getHouseId() + "-ARCH-" + archivedMedia.getMedia().getMediaFileRealName()); + + String outputName = "IntegrationTest_" + System.currentTimeMillis() + ".mxf"; + Path ouptput = Paths.get(input.getParent().toString(), outputName); + Files.copy(input, ouptput); + if (!ouptput.toFile().exists()) + throw new Exception("File not exists: " + ouptput); + + ArchiveItem archiveItem = new ArchiveItem(); + archiveItem.setMediaFile(ouptput.toString()); + archiveItem.setItemHouseId(outputName); + archiveItem.setItemTitle(outputName); + archiveItem.setMediaHouseId(outputName); + archiveItem.setMediaTitle(outputName); + archiveItem.setMediaType("Generic"); CountDownLatch finishLatch = new CountDownLatch(1); getEngine().submit(null, e -> { if (e.isRuntimeTerminated()) finishLatch.countDown(); + }, "archive-material.xml", "Test archive-material", ListUtils.asMap("archiveItem", archiveItem, "killDateDays", 0)); + finishLatch.await(); - }, "retrieve-ondemand.xml", "Test retrieve-ondemand", - ListUtils.asMap("globalRetrievePath", "file://10.11.1.100", "localRetrievePath", "/mediacube/data", "materialOutputFolder", "/", - "promoOutputFolder", "/", "advertisementOutputFolder", "/", "octopusOutputFolder", "/", "genericOutputFolder", "/", - "onlineOutputFolder", "/", "killDateDays", -1, "nexioAgency", "ARCHIVE_RESTORE", "nexioPort", 2098, "nexioUserName", "administrator", - "nexioPassword", "system", "archivedMedia", archivedMedia, "successRecipient", "vasary@elgekko.net", "houseId", - archivedMedia.getMedia().getHouseId(), "targetPathType", "0")); + if (!ItemManagerExtensions.isArchived(getManager(), ouptput)) + throw new Exception("File not archived: " + ouptput); + + Files.delete(ouptput); + logger.info("_02_test_archive_material SUCCESS"); + } + + private void _03_test_delete_materials() throws Exception { + Path output = Paths.get("/mediacube/data", archivedMedia.getMedia().getHouseId(), + archivedMedia.getMedia().getHouseId() + "-ARCH-" + archivedMedia.getMedia().getMediaFileRealName()); + String source = output.getParent().toString(); + + CountDownLatch finishLatch = new CountDownLatch(1); + getEngine().submit(null, e -> { + if (e.isRuntimeTerminated()) + finishLatch.countDown(); + }, "delete-materials.xml", "Test delete-materials", ListUtils.asMap("sourcePath", source, "skipArchiveCheck", true)); finishLatch.await(); + if (output.toFile().exists()) + throw new Exception("File exists: " + output); + } + + @StepEntry + public Object[] execute() throws Exception { + getJobRuntime().setDescription("_00_test_cancelable"); + _00_test_cancelable(); + setProgress(25); + + getJobRuntime().setDescription("_01_test_retrieve_ondemand"); + _01_test_retrieve_ondemand(); + setProgress(50); + + getJobRuntime().setDescription("_02_test_archive_material"); + _02_test_archive_material(); + setProgress(75); + + getJobRuntime().setDescription("_03_test_delete_materials"); + _03_test_delete_materials(); + setProgress(100); + return null; } } diff --git a/server/user.jobengine.executors/src/user/jobengine/server/steps/MediaToolStep.java b/server/user.jobengine.executors/src/user/jobengine/server/steps/MediaToolStep.java index c0c56f68..81bea7bd 100644 --- a/server/user.jobengine.executors/src/user/jobengine/server/steps/MediaToolStep.java +++ b/server/user.jobengine.executors/src/user/jobengine/server/steps/MediaToolStep.java @@ -5,35 +5,25 @@ import java.nio.file.Paths; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; -import org.apache.logging.log4j.Marker; -import user.commons.mediatool.MediaInfo; +import user.commons.mediaarea.MediaArea; import user.jobengine.db.Media; public class MediaToolStep extends JobStep { private static final Logger logger = LogManager.getLogger(); - private Marker marker; @StepEntry public Object[] execute(ArchiveItem archiveItem, Media mediaCubeMedia) throws Exception { - marker = getSessionMarker(); - - //TODO ez most minden esetben ujranezi - // if (mediaCubeMedia.getLength() > 0) - // return null; - Path filePath = Paths.get(archiveItem.getMediaFile()); - MediaInfo mi = new MediaInfo(filePath); - try { - mi.process(); - long frames = mi.getFrames(); + if (filePath.toFile().length() != 0) { + MediaArea ma = new MediaArea(filePath); + ma.process(); + long frames = ma.getFrameCount(); if (frames > 0) { + logger.info("Media {} length is {}", filePath, frames); mediaCubeMedia.setLength(frames); getManager().modify(mediaCubeMedia); } - } catch (Exception e) { - logger.catching(e); - logger.error(marker, "A '{}' média analizálása sikertelen. A rendszer üzenete: {}", filePath, e.getMessage()); } return null; } diff --git a/server/user.jobengine.executors/src/user/jobengine/server/steps/PrepareMediaRestoreStep.java b/server/user.jobengine.executors/src/user/jobengine/server/steps/PrepareMediaRestoreStep.java index e9246724..3708aa3e 100644 --- a/server/user.jobengine.executors/src/user/jobengine/server/steps/PrepareMediaRestoreStep.java +++ b/server/user.jobengine.executors/src/user/jobengine/server/steps/PrepareMediaRestoreStep.java @@ -1,5 +1,7 @@ package user.jobengine.server.steps; +import java.util.List; + import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; @@ -15,7 +17,10 @@ public class PrepareMediaRestoreStep extends JobStep { @StepEntry public Object[] execute(Media media, String tempStoreName, String tempStoreProtocol) throws Exception { StoreUri mediaStoreUri = null; - for (MediaFile mediaFile : media.getMediaFiles()) { + List mediaFiles = media.getMediaFiles(); + if (mediaFiles.size() > 1) + throw new Exception("Media " + media.getId() + " already has proxy"); + for (MediaFile mediaFile : mediaFiles) { Store store = mediaFile.getStore(); if (store.isSystem() && !store.isLowres()) { mediaStoreUri = store.getStoreUri(RemoteStoreProtocol.TSM); diff --git a/server/user.jobengine.executors/src/user/jobengine/server/steps/PrepareRemoteTranscodeJobParametersStep.java b/server/user.jobengine.executors/src/user/jobengine/server/steps/PrepareRemoteTranscodeJobParametersStep.java deleted file mode 100644 index 8fa6821e..00000000 --- a/server/user.jobengine.executors/src/user/jobengine/server/steps/PrepareRemoteTranscodeJobParametersStep.java +++ /dev/null @@ -1,32 +0,0 @@ -package user.jobengine.server.steps; - -import java.nio.file.Path; -import java.nio.file.Paths; - -import com.ibm.nosql.json.api.BasicDBObject; - -import user.commons.StoreUri; -import user.commons.remotestore.RemoteStoreProtocol; - -public class PrepareRemoteTranscodeJobParametersStep extends JobStep { - @StepEntry - public Object[] execute(String profileName, String tempStoreName, String tempStoreProtocol, String fileName) throws Exception { - Path inputPath = Paths.get("m:/", fileName); - - StoreUri tempStoreUri = getManager().getStoreUri(tempStoreName, Enum.valueOf(RemoteStoreProtocol.class, tempStoreProtocol)); - String outFileName = fileName.substring(0, fileName.lastIndexOf(".")) + ".mp4"; - Path outputPath = null; - if (fileName.length() > 2) { - outputPath = Paths.get(tempStoreUri.toString(true), fileName.substring(0, 1), fileName.substring(1, 2), fileName.substring(2, 3), outFileName); - } else { - outputPath = Paths.get(tempStoreUri.toString(true), fileName, outFileName); - } - - BasicDBObject parameters = new BasicDBObject(); - parameters.put("profile", profileName); - parameters.put("input", inputPath.toString()); - parameters.put("output", outputPath.toString()); - return new Object[] { parameters }; - } - -} diff --git a/server/user.jobengine.executors/src/user/jobengine/server/steps/PrepareRemoteTranscodeStep.java b/server/user.jobengine.executors/src/user/jobengine/server/steps/PrepareRemoteTranscodeStep.java new file mode 100644 index 00000000..50fe237f --- /dev/null +++ b/server/user.jobengine.executors/src/user/jobengine/server/steps/PrepareRemoteTranscodeStep.java @@ -0,0 +1,38 @@ +package user.jobengine.server.steps; + +import java.nio.file.Path; +import java.nio.file.Paths; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + +import com.ibm.nosql.json.api.BasicDBObject; + +public class PrepareRemoteTranscodeStep extends JobStep { + static private final Logger logger = LogManager.getLogger(); + + @StepEntry + public Object[] execute(String profileName, String fileName) throws Exception { + String hiResRoot = "m:/"; + String lowResRoot = "m:/lowres"; + + Path inputPath = Paths.get(hiResRoot, fileName); + + String outFileName = fileName.substring(0, fileName.lastIndexOf(".")) + ".mp4"; + + Path relativeOutputPath = null; + if (fileName.length() > 2) + relativeOutputPath = Paths.get(fileName.substring(0, 1), fileName.substring(1, 2), fileName.substring(2, 3), outFileName); + else + relativeOutputPath = Paths.get(fileName, outFileName); + + BasicDBObject parameters = new BasicDBObject(); + parameters.put("profile", profileName); + parameters.put("input", inputPath.toString()); + parameters.put("output", Paths.get(lowResRoot, relativeOutputPath.toString()).toString()); + + logger.info("Prepared for remote transode {}", parameters); + return new Object[] { parameters, relativeOutputPath.toString() }; + } + +} diff --git a/server/user.jobengine.executors/src/user/jobengine/server/steps/QueryMissingProxyMediaStep.java b/server/user.jobengine.executors/src/user/jobengine/server/steps/QueryMissingProxyMediaStep.java index 9e8d1949..56262737 100644 --- a/server/user.jobengine.executors/src/user/jobengine/server/steps/QueryMissingProxyMediaStep.java +++ b/server/user.jobengine.executors/src/user/jobengine/server/steps/QueryMissingProxyMediaStep.java @@ -17,31 +17,21 @@ public class QueryMissingProxyMediaStep extends JobStep { @StepEntry public Object[] execute() throws Exception { - Object[] result = new Object[] { null }; + setDescription("Looking for missing proxy"); DB db = NoSQLUtils.getNoSQLDB(); DBCollection collection = db.getCollection("missing_lowres"); Media media = getFirstUntranscodedMedia(collection); - try { - if (media == null) { - logger.info(new MediaCubeUndoMarker(getSessionMarker().getSessionID()), "Nincs feldolgozandó hiány."); - // throw new Exception("Nincs feldolgozandó hiány."); - cancel(); - return null; - } - - String name = media.getMediaFileRealName(); - collection.save(new BasicDBObject("name", name)); - logger.info(getSessionMarker(), "Processing mediaID: {}", media.getId()); - result[0] = media; - } catch (Exception e) { - logger.catching(e); - logger.error(getSessionMarker(), e.getMessage()); - throw e; - } finally { - setProgress(100); + if (media == null) { + logger.info(new MediaCubeUndoMarker(getSessionMarker().getSessionID()), "Nincs feldolgozandó hiány."); + cancel(); + return null; } - return result; + + String name = media.getMediaFileRealName(); + collection.save(new BasicDBObject("name", name)); + logger.info(getSessionMarker(), "Processing mediaID: {}", media.getId()); + return new Object[] { media }; } private Media getFirstUntranscodedMedia(DBCollection collection) { diff --git a/server/user.jobengine.executors/src/user/jobengine/server/steps/RemoteJobStep.java b/server/user.jobengine.executors/src/user/jobengine/server/steps/RemoteJobStep.java index de392cf3..d4900d76 100644 --- a/server/user.jobengine.executors/src/user/jobengine/server/steps/RemoteJobStep.java +++ b/server/user.jobengine.executors/src/user/jobengine/server/steps/RemoteJobStep.java @@ -7,36 +7,30 @@ import com.ibm.nosql.json.api.BasicDBObject; import user.jobengine.server.steps.shared.MediaCubeClient; -/* -import user.jobengine.server.steps.MediaCubeClient; -*/ public class RemoteJobStep extends JobStep { private static final Logger logger = LogManager.getLogger(); @StepEntry public Object[] execute(String remoteServer, String template, String name, BasicDBObject remoteJobParameters) throws Exception { - try { - MediaCubeClient mc = new MediaCubeClient(remoteServer); - long jobId = mc.startjob(template, name, remoteJobParameters); - logger.info(getMarker(), "Started {} on server {}", jobId, remoteServer); - while (true) { - BasicDBObject status = mc.getStatus(jobId); - if (status != null) - setProgress(status.getInt("progress")); - - Thread.sleep(3000); - - String jobStatus = status.getString("status"); - if ("SUSPENDED".equals(jobStatus)) - throw new Exception(status.getString("description")); - - if ("FINISHED".equals(jobStatus)) - break; - } - - } catch (Exception e) { - throw e; + setDescription("Executing on {}, template {}, profile {}", remoteServer, template, remoteJobParameters.getString("profile")); + MediaCubeClient mc = new MediaCubeClient(remoteServer); + long jobId = mc.startjob(template, name, remoteJobParameters); + logger.info(getMarker(), "Started {} on server {}", jobId, remoteServer); + while (true) { + BasicDBObject status = mc.getStatus(jobId); + if (status != null) + setProgress(status.getInt("progress")); + + Thread.sleep(2000); + + String jobStatus = status.getString("status"); + if ("SUSPENDED".equals(jobStatus)) + throw new Exception(status.getString("description")); + + if ("FINISHED".equals(jobStatus)) + break; } - return new Object[] { null }; + + return null; } } diff --git a/server/user.jobengine.executors/src/user/jobengine/server/steps/SaveMediaProxy.java b/server/user.jobengine.executors/src/user/jobengine/server/steps/SaveMediaProxy.java new file mode 100644 index 00000000..3f0f667d --- /dev/null +++ b/server/user.jobengine.executors/src/user/jobengine/server/steps/SaveMediaProxy.java @@ -0,0 +1,22 @@ +package user.jobengine.server.steps; + +import user.jobengine.db.IItemManager; +import user.jobengine.db.Media; +import user.jobengine.db.MediaFile; +import user.jobengine.db.Store; + +public class SaveMediaProxy extends JobStep { + + @StepEntry + public Object[] execute(Media media, String proxyRelativePath) throws Exception { + IItemManager manager = getManager(); + Store store = manager.getCurrentLowresStore(); + MediaFile mediaFile = new MediaFile(); + mediaFile.setMedia(media); + mediaFile.setStore(store); + mediaFile.setFileType(manager.getFileType("Low-res")); + mediaFile.setRelativePath(proxyRelativePath); + manager.add(mediaFile); + return null; + } +} diff --git a/server/user.jobengine.executors/src/user/jobengine/server/steps/TSMBackupStep.java b/server/user.jobengine.executors/src/user/jobengine/server/steps/TSMBackupStep.java index 98ea8cd0..c719116e 100644 --- a/server/user.jobengine.executors/src/user/jobengine/server/steps/TSMBackupStep.java +++ b/server/user.jobengine.executors/src/user/jobengine/server/steps/TSMBackupStep.java @@ -53,6 +53,14 @@ public class TSMBackupStep extends JobStep { File sourceMediaFile = new File(archiveItem.getMediaFile()); String sourceFileName = sourceMediaFile.getName(); + try { + Timecode timecode = new Timecode(mediaCubeMedia.getLength(), Type.PAL); + String details = String.format("%s (%s, %d bytes)", sourceFileName, timecode.toString(), sourceMediaFile.length()); + getJobRuntime().setDescription(details); + } catch (Exception e) { + String details = String.format("%s (%d bytes)", sourceFileName, sourceMediaFile.length()); + getJobRuntime().setDescription(details); + } Timecode timecode = new Timecode(mediaCubeMedia.getLength(), Type.PAL); String details = String.format("%s (%s, %d bytes)", sourceFileName, timecode.toString(), sourceMediaFile.length()); getJobRuntime().setDescription(details); @@ -63,6 +71,7 @@ public class TSMBackupStep extends JobStep { //TODO mxf helyett az osszes kiterjesztest!!!!! //A dupla ellenorzes a napon beluli ismetlesek miatt kell long existingMediaId = archiveItem.getExistingMediaId(); + if (sourceMediaFile.length() == 0 && existingMediaId == 0) { existingMediaId = ItemManagerExtensions.getExistingRundownMedia(manager, sourceFileName.replace(MXFEXT, "")); if (existingMediaId == 0) @@ -71,7 +80,6 @@ public class TSMBackupStep extends JobStep { if (existingMediaId == 0) existingMediaId = ItemManagerExtensions.getExistingRundownMedia(manager, sourceFileName.replace(MXFEXT, "")); - String targetFileName; if (RANDOMIZE_ARCHIVES) { // a-z, A-Z, 0-9. For example: WRMcpIk7, s57JwCVA diff --git a/server/user.jobengine.executors/src/user/jobengine/server/steps/TSMRestoreStep.java b/server/user.jobengine.executors/src/user/jobengine/server/steps/TSMRestoreStep.java index 56b522d3..26d8607b 100644 --- a/server/user.jobengine.executors/src/user/jobengine/server/steps/TSMRestoreStep.java +++ b/server/user.jobengine.executors/src/user/jobengine/server/steps/TSMRestoreStep.java @@ -41,7 +41,7 @@ public class TSMRestoreStep extends JobStep { private Marker marker; protected void afterRestore(StoreUri targetUri, String targetPath, int killDateDays, String targetFileName) throws IOException, Exception { - if (killDateDays > 0) + if (killDateDays != 0) EscortFiles.createUNCKillDate(targetPath, targetFileName, killDateDays, marker); } @@ -88,8 +88,10 @@ public class TSMRestoreStep extends JobStep { }); RemoteFile result = sourceUri.transferFrom(targetUri, sourceFileName, targetFileName); - String globalTargetPath = Paths.get(targetPath, targetFileName).getParent().toString().replace(Paths.get(localRetrievePath).toString(), - globalRetrievePath); + String globalTargetPath = Paths.get(targetPath, targetFileName) + .getParent() + .toString() + .replace(Paths.get(localRetrievePath).toString(), globalRetrievePath); logger.info(marker, "Az '{}' állomány visszatöltése sikeres volt '{}' néven. A célmappa a ide kattintva nyitható meg.", diff --git a/server/user.jobengine.executors/src/user/jobengine/server/steps/TranscodeFFMpegStep.java b/server/user.jobengine.executors/src/user/jobengine/server/steps/TranscodeFFMpegStep.java index 7e4c3a1f..c4cb01ac 100644 --- a/server/user.jobengine.executors/src/user/jobengine/server/steps/TranscodeFFMpegStep.java +++ b/server/user.jobengine.executors/src/user/jobengine/server/steps/TranscodeFFMpegStep.java @@ -1,40 +1,43 @@ package user.jobengine.server.steps; +import java.nio.file.Path; import java.nio.file.Paths; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import user.commons.mediaarea.MediaArea; +import user.commons.mediatool.Timecode; +import user.commons.mediatool.Timecode.Type; +import user.jobengine.server.steps.shared.EscortFiles; import user.jobengine.server.steps.shared.ExternalCommandExecutor; -import user.jobengine.server.steps.shared.IExternalCallback; -/* -import user.jobengine.server.steps.shared.ExternalProfile; -import user.jobengine.server.steps.shared.ExternalProfilesConfig; - */ -public class TranscodeFFMpegStep extends JobStep implements IExternalCallback { +public class TranscodeFFMpegStep extends JobStep { private static final Logger logger = LogManager.getLogger(); private long allFrameCount; private long processedFrameCount; @StepEntry public Object[] execute(String input, String output, String profile) throws Exception { - try { - MediaArea ma = new MediaArea(Paths.get(input)); - ma.process(); - allFrameCount = ma.getFrameCount(); - - ExternalCommandExecutor executor = new ExternalCommandExecutor(); - executor.execute(profile, input, output, this); - } catch (Exception e) { - logger.error(getMarker(), "Media information not available. System message: {}", e.getMessage()); - throw e; - } + Path inputPath = Paths.get(input); + MediaArea ma = new MediaArea(inputPath); + ma.process(); + allFrameCount = ma.getFrameCount(); + + Timecode timecode = new Timecode(allFrameCount, Type.PAL); + setDescription("Transcoding file {}, profile {}, media length {} ({} frames)", inputPath.getFileName(), profile, timecode, allFrameCount); + + Path lowresPath = Paths.get(output).getParent(); + logger.info("Ensure path exists {}", lowresPath); + EscortFiles.ensureUNCFolder(lowresPath); + + ExternalCommandExecutor executor = new ExternalCommandExecutor(); + executor.execute(profile, input, output, l -> { + onResponse(l); + }); return null; } - @Override public void onResponse(String line) { String[] values = line.split("="); if ("frame".equals(values[0])) { diff --git a/server/user.jobengine.executors/src/user/jobengine/server/steps/shared/EscortFiles.java b/server/user.jobengine.executors/src/user/jobengine/server/steps/shared/EscortFiles.java index 6057078d..da308d3c 100644 --- a/server/user.jobengine.executors/src/user/jobengine/server/steps/shared/EscortFiles.java +++ b/server/user.jobengine.executors/src/user/jobengine/server/steps/shared/EscortFiles.java @@ -211,17 +211,17 @@ public class EscortFiles { public static void ensureUNCFolder(Path filePath) throws IOException { File folder = filePath.toFile(); if (!folder.exists() || !folder.isDirectory()) { - Set perms = PosixFilePermissions.fromString("rwxrwxrwx"); - FileAttribute> attr = PosixFilePermissions.asFileAttribute(perms); try { + Set perms = PosixFilePermissions.fromString("rwxrwxrwx"); + FileAttribute> attr = PosixFilePermissions.asFileAttribute(perms); Files.createDirectories(filePath, attr); } catch (Exception e) { - // logger.catching(e); + //logger.catching(e); try { Files.createDirectories(filePath); } catch (Exception e1) { - logger.catching(e); - throw e; + logger.catching(e1); + throw e1; } } } diff --git a/server/user.jobengine.executors/src/user/jobengine/server/steps/shared/ItemManagerExtensions.java b/server/user.jobengine.executors/src/user/jobengine/server/steps/shared/ItemManagerExtensions.java index 9e6d4a2f..32f17cb5 100644 --- a/server/user.jobengine.executors/src/user/jobengine/server/steps/shared/ItemManagerExtensions.java +++ b/server/user.jobengine.executors/src/user/jobengine/server/steps/shared/ItemManagerExtensions.java @@ -1,9 +1,15 @@ package user.jobengine.server.steps.shared; +import java.nio.file.Path; + import com.ibm.nosql.json.api.BasicDBObject; +import user.commons.RemoteFile; +import user.commons.StoreUri; +import user.commons.remotestore.RemoteStoreProtocol; import user.jobengine.db.IItemManager; import user.jobengine.db.IResultSetConsumer; +import user.jobengine.db.Store; public class ItemManagerExtensions { @@ -27,8 +33,8 @@ public class ItemManagerExtensions { } public static long getExistingRundownMedia(IItemManager manager, String houseid) { - final long[] result = { 0 }; - final String[] idToCheck = { houseid }; + final long[] result = new long[] { 0 }; + final String[] idToCheck = new String[] { houseid }; int pos = houseid.lastIndexOf("-"); //a hivas a CopyForArchiveNEXIOMaterialsStep-bol is johet, ott meg nincs idobelyegezve a nev! if (pos > 0 && houseid.length() - pos > 4) @@ -55,4 +61,34 @@ public class ItemManagerExtensions { return result[0]; } + static public boolean isArchived(IItemManager manager, Path filePath) { + boolean result = false; + String name = filePath.getFileName().toString(); + String[] tsmName = new String[] { null }; + String query = String.format("SELECT relativepath FROM MEDIAFILE WHERE houseid = '%s'", name); + manager.executeQuery(query, rs -> { + tsmName[0] = rs.getString("relativepath"); + return false; + }, null); + + Store tsmStore = manager.getSystemStore(false); + if (tsmStore == null) + throw new NullPointerException("A TSM bejegyzés nem található!"); + + StoreUri tsmStoreUri = tsmStore.getSourceStoreUri(RemoteStoreProtocol.TSM); + if (tsmStoreUri == null) + throw new NullPointerException("A TSM forrás elérése nem található!"); + + if (tsmName[0] != null) { + try { + RemoteFile remoteFile = tsmStoreUri.getRemoteFile(tsmName[0]); + result = remoteFile != null; + } catch (Exception e) { + result = false; + } finally { + tsmStoreUri.cleanUp(); + } + } + return result; + } } diff --git a/server/user.jobengine.executors/src/user/jobengine/server/steps/shared/MetadataTypeDetector.java b/server/user.jobengine.executors/src/user/jobengine/server/steps/shared/MetadataTypeDetector.java index c221f884..9e9e6be9 100644 --- a/server/user.jobengine.executors/src/user/jobengine/server/steps/shared/MetadataTypeDetector.java +++ b/server/user.jobengine.executors/src/user/jobengine/server/steps/shared/MetadataTypeDetector.java @@ -7,12 +7,12 @@ public class MetadataTypeDetector { private static final String HYPHEN = "-"; private static final String DOT = "."; - private static final String REGEXP_TRAFFICMATERIALID = "^M{1}[0-9]{6}[A-Z]{1}$"; - private static final String REGEXP_TRAFFICADID = "^R{1}[0-9]{6}[A-Z]{1}$"; - private static final String REGEXP_TRAFFICPROMOID = "^P{1}[0-9]{6}[A-Z]{1}$"; - private static final String REGEXP_OCTOPUSSTORYID = "^[0-9]+$"; - private static final String REGEXP_OCTOPUSPLACEHOLDERID = "^[0-9]+_[0-9]+$"; - private static final String REGEXP_OCTOPUSPLACEHOLDERVERSIONEDID = "^[0-9]+_[0-9]+-[0-9]{3}$"; + private static final String REGEXP_TRAFFICMATERIALID = "^M{1}[0-9]{6}[A-Z]{1}"; + private static final String REGEXP_TRAFFICADID = "^R{1}[0-9]{6}[A-Z]{1}"; + private static final String REGEXP_TRAFFICPROMOID = "^P{1}[0-9]{6}[A-Z]{1}"; + private static final String REGEXP_OCTOPUSSTORYID = "^[0-9]+"; + private static final String REGEXP_OCTOPUSPLACEHOLDERID = "^[0-9]+_[0-9]+"; + private static final String REGEXP_OCTOPUSPLACEHOLDERVERSIONEDID = "^[0-9]+_[0-9]+-[0-9]{3}"; public static MetadataType GuessMetadataType(String id) { if (StringUtils.isBlank(id)) diff --git a/server/user.jobengine.osgi.commons/src/user/commons/MediaCubeFinishMarker.java b/server/user.jobengine.osgi.commons/src/user/commons/MediaCubeFinishMarker.java index a0bee642..d1c35c4a 100644 --- a/server/user.jobengine.osgi.commons/src/user/commons/MediaCubeFinishMarker.java +++ b/server/user.jobengine.osgi.commons/src/user/commons/MediaCubeFinishMarker.java @@ -8,11 +8,13 @@ public class MediaCubeFinishMarker extends MediaCubeMarker { setSessionName(parent.getSessionName()); setTo(parent.getTo()); setUseSessionLog(parent.isUseSessionLog()); + setParents(parent); } public MediaCubeFinishMarker(MediaCubeMarker parent, String to) { super(to); setSessionID(parent.getSessionID()); setSessionName(parent.getSessionName()); + setParents(parent); } } diff --git a/server/user.jobengine.osgi.commons/src/user/commons/remotestore/FtpDirectoryLister.java b/server/user.jobengine.osgi.commons/src/user/commons/remotestore/FtpDirectoryLister.java index e2256c76..7339d006 100644 --- a/server/user.jobengine.osgi.commons/src/user/commons/remotestore/FtpDirectoryLister.java +++ b/server/user.jobengine.osgi.commons/src/user/commons/remotestore/FtpDirectoryLister.java @@ -1,5 +1,6 @@ package user.commons.remotestore; +import java.io.FileNotFoundException; import java.io.IOException; import java.io.InputStream; import java.io.OutputStream; @@ -117,12 +118,17 @@ public class FtpDirectoryLister implements IDirectoryLister { ftpClient.login(userName, storeUri.getPassword()); ftpClient.setFileType(FTP.BINARY_FILE_TYPE); ftpClient.enterLocalPassiveMode(); + + //210510 ha tud, akkor menjen a gyoker konyvtarba ld. AMC ISILON valtas + ftpClient.changeWorkingDirectory("/"); + if (pathElements.length > 1) { for (int i = 1; i < pathElements.length; i++) { ftpClient.changeWorkingDirectory(pathElements[i]); } } + logger.info("Current working directory is {}", ftpClient.printWorkingDirectory()); return ftpClient; } @@ -175,7 +181,7 @@ public class FtpDirectoryLister implements IDirectoryLister { // cleanUp(); } if (result == null) - throw new Exception("File not exists: " + fileName + " on uri " + storeUri); + throw new FileNotFoundException(storeUri.toString() + "/" + fileName); return result; } diff --git a/server/user.jobengine.osgi.commons/src/user/commons/remotestore/LocalDirectoryLister.java b/server/user.jobengine.osgi.commons/src/user/commons/remotestore/LocalDirectoryLister.java index 2e7ac124..f8917cf5 100644 --- a/server/user.jobengine.osgi.commons/src/user/commons/remotestore/LocalDirectoryLister.java +++ b/server/user.jobengine.osgi.commons/src/user/commons/remotestore/LocalDirectoryLister.java @@ -2,6 +2,7 @@ package user.commons.remotestore; import java.io.File; import java.io.FileInputStream; +import java.io.FileNotFoundException; import java.io.FileOutputStream; import java.io.IOException; import java.io.InputStream; @@ -48,9 +49,11 @@ public class LocalDirectoryLister implements IDirectoryLister { } @Override - public void delete(RemoteFile remoteFile) { - // TODO Auto-generated method stub - + public void delete(RemoteFile remoteFile) throws Exception { + String path = storeUri.toString(true); + File file = new File(path + remoteFile.getName()); + if (file.exists() && !file.isDirectory()) + file.delete(); } @Override @@ -82,6 +85,10 @@ public class LocalDirectoryLister implements IDirectoryLister { public RemoteFile get(String fileName) throws Exception { String path = storeUri.toString(true); File file = new File(path + fileName); + + if (!file.exists()) + throw new FileNotFoundException(storeUri.toString() + "/" + fileName); + return toRemoteFile(file); } diff --git a/server/user.jobengine.osgi.server/META-INF/MANIFEST.MF b/server/user.jobengine.osgi.server/META-INF/MANIFEST.MF index 12679c8d..75e2d8bd 100644 --- a/server/user.jobengine.osgi.server/META-INF/MANIFEST.MF +++ b/server/user.jobengine.osgi.server/META-INF/MANIFEST.MF @@ -17,6 +17,10 @@ Import-Package: com.fasterxml.jackson.databind;version="2.4.5", org.apache.commons.io.output;version="2.2.0", org.apache.logging.log4j;version="2.8.2", org.apache.logging.log4j.message;version="2.8.2", + org.codehaus.groovy.ast;version="3.0.3", + org.codehaus.groovy.classgen;version="3.0.3", + org.codehaus.groovy.control;version="3.0.3", + org.codehaus.groovy.control.customizers;version="3.0.3", org.eclipse.core.runtime.adaptor, org.eclipse.osgi.framework.console;version="1.1.0", org.jboss.resteasy.client.jaxrs, diff --git a/server/user.jobengine.osgi.server/src/user/jobengine/server/IJobEngine.java b/server/user.jobengine.osgi.server/src/user/jobengine/server/IJobEngine.java index 38bbd174..0e171f2a 100644 --- a/server/user.jobengine.osgi.server/src/user/jobengine/server/IJobEngine.java +++ b/server/user.jobengine.osgi.server/src/user/jobengine/server/IJobEngine.java @@ -14,9 +14,7 @@ public interface IJobEngine { static final int QUEUE_POLL_INTERVAL_MS = 50; static final String DEFAULT_OWNER = "MediaCube"; - void addJobChangedEventListener(IJobChangedListener listener); - - void addManagedJobChangedListener(IJobChangedListener listener); + boolean addJobChangedEventListener(IJobChangedListener listener); void addToExecutorQueue(IJobRuntime jobRuntime); @@ -80,7 +78,7 @@ public interface IJobEngine { void jobCleanup(IJobRuntime jobRuntime); - void keepAliveJobChangedListener(IJobChangedListener listener); + boolean keepAliveJobChangedListener(IJobChangedListener listener); void keepAliveWorker(String remoteAddr); diff --git a/server/user.jobengine.osgi.server/src/user/jobengine/server/JobEngine.java b/server/user.jobengine.osgi.server/src/user/jobengine/server/JobEngine.java index 07c9aa9c..88e4177a 100644 --- a/server/user.jobengine.osgi.server/src/user/jobengine/server/JobEngine.java +++ b/server/user.jobengine.osgi.server/src/user/jobengine/server/JobEngine.java @@ -7,9 +7,9 @@ import java.util.ArrayList; import java.util.Collection; import java.util.List; import java.util.Map; +import java.util.Set; import java.util.concurrent.BlockingQueue; import java.util.concurrent.ConcurrentHashMap; -import java.util.concurrent.CopyOnWriteArrayList; import java.util.concurrent.CyclicBarrier; import java.util.concurrent.LinkedBlockingQueue; import java.util.concurrent.PriorityBlockingQueue; @@ -201,7 +201,7 @@ public class JobEngine implements IJobEngine { private AtomicLong nextJobId; private SchedulerService schedulerService = null; - private List jobChangedListenerList = new CopyOnWriteArrayList<>(); + private Set jobChangedListenerList = ConcurrentHashMap.newKeySet(); private Map remoteWorkers; private String masterServerAddress = System.getProperty("jobengine.master.server", ""); private final JobEngineRemote remoteEngine; @@ -237,19 +237,12 @@ public class JobEngine implements IJobEngine { } @Override - public void addJobChangedEventListener(IJobChangedListener listener) { - if (listener != null && !keepAliveJobChangedListeners.containsKey(listener)) { + public boolean addJobChangedEventListener(IJobChangedListener listener) { + boolean result = !jobChangedListenerList.contains(listener); + if (listener != null && result) jobChangedListenerList.add(listener); - } - } - @Override - public void addManagedJobChangedListener(IJobChangedListener listener) { - if (listener != null) { - jobChangedListenerList.add(listener); - keepAliveJobChangedListeners.put(listener, System.currentTimeMillis()); - logger.info("Managed listener added {}", listener); - } + return result; } @Override @@ -589,14 +582,17 @@ public class JobEngine implements IJobEngine { } @Override - public void keepAliveJobChangedListener(IJobChangedListener listener) { + public boolean keepAliveJobChangedListener(IJobChangedListener listener) { + boolean result = false; if (keepAliveJobChangedListeners != null) { long now = System.currentTimeMillis(); //ha mar hozza van adva, nem adja hozza - addJobChangedEventListener(listener); + result = addJobChangedEventListener(listener); keepAliveJobChangedListeners.put(listener, now); - //logger.info("Refreshing listener {}, now {} ({})", listener, now, keepAliveJobChangedListeners.size()); + logger.debug("Refreshing listener {}, now {} ({})", listener, now, keepAliveJobChangedListeners.size()); } + + return result; } @Deprecated @@ -772,11 +768,11 @@ public class JobEngine implements IJobEngine { if ((now - lastMod) > 5 * 1000) { toBeRemoved.add(listener); - logger.info("{} refreshed {}, now {}", listener, lastMod, now); + logger.debug("{} refreshed {}, now {}", listener, lastMod, now); } } toBeRemoved.forEach(r -> { - logger.info("Removing listener {}", r); + logger.debug("Removing listener {}", r); removeJobChangedEventListener(r); keepAliveJobChangedListeners.remove(r); }); diff --git a/server/user.jobengine.osgi.server/src/user/jobengine/server/JobEngineConfiguration.java b/server/user.jobengine.osgi.server/src/user/jobengine/server/JobEngineConfiguration.java index 331a8d53..86d2fdc3 100644 --- a/server/user.jobengine.osgi.server/src/user/jobengine/server/JobEngineConfiguration.java +++ b/server/user.jobengine.osgi.server/src/user/jobengine/server/JobEngineConfiguration.java @@ -3,11 +3,13 @@ package user.jobengine.server; import java.io.FileNotFoundException; import java.io.IOException; import java.io.InputStream; +import java.net.MalformedURLException; import java.net.URL; import java.net.URLClassLoader; import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.Paths; +import java.security.CodeSource; import java.util.ArrayList; import java.util.Collections; import java.util.LinkedHashMap; @@ -20,6 +22,8 @@ import java.util.stream.Stream; import org.apache.commons.lang.StringUtils; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; +import org.codehaus.groovy.control.CompilationUnit; +import org.codehaus.groovy.control.CompilerConfiguration; import com.ibm.nosql.json.JSONUtil; import com.ibm.nosql.json.api.BasicDBList; @@ -49,6 +53,7 @@ public class JobEngineConfiguration implements IJobEngineConfiguration { private final Map programs = new LinkedHashMap(); private final List templates = Collections.synchronizedList(new ArrayList()); private final Map schedules = new LinkedHashMap(); + private URLClassLoader stepsClassLoader; public synchronized void bindSystemConfiguration(Object service) { if (service instanceof IConfiguration) { @@ -57,47 +62,69 @@ public class JobEngineConfiguration implements IJobEngineConfiguration { } } + private GroovyClassLoader createGroovyClassLoader(URLClassLoader stepsClassLoader) throws FileNotFoundException { + String stepsDir = systemConfig.getConfig(DIR_STEPS); + GroovyClassLoader gcl = new GroovyClassLoader(stepsClassLoader) { + + @Override + protected CompilationUnit createCompilationUnit(CompilerConfiguration config, CodeSource source) { + CompilationUnit result = new CompilationUnit(config, source, this); + loadStepSources(result, Paths.get(stepsDir, "shared")); + loadStepSources(result, Paths.get(stepsDir)); + return result; + } + + }; + return gcl; + } + @SuppressWarnings("unchecked") @Override public IJobStep createJobStep(String stepUnitName) throws Exception { IJobStep result = null; boolean isGroovyClass = stepUnitName.toLowerCase().endsWith(".java") || stepUnitName.toLowerCase().endsWith(".groovy"); + logger.info("Looking for {} step classloader requirement", stepUnitName); - URLClassLoader stepsClassLoader = null; - try { - String stepClassesDir = systemConfig.getConfig(DIR_CLASSES); - Path path = Paths.get(stepClassesDir); - URL[] urls = { path.toUri().toURL() }; - stepsClassLoader = URLClassLoader.newInstance(urls, Thread.currentThread().getContextClassLoader()); - } catch (FileNotFoundException e) { - //nincs jobs/classes mappa - URL[] urls = {}; - stepsClassLoader = URLClassLoader.newInstance(urls, Thread.currentThread().getContextClassLoader()); - } + if (stepsClassLoader == null) + stepsClassLoader = createParentClassLoader(); Class stepClass = null; if (isGroovyClass) { - try { - GroovyClassLoader gcl = new GroovyClassLoader(stepsClassLoader); - String stepsDir = systemConfig.getConfig(DIR_STEPS); - stepClass = (Class) parseClassHierarchy(gcl, stepsDir, stepUnitName); - } catch (Exception e) { - throw e; - } - + logger.info("Creating GroovyClassLoader"); + GroovyClassLoader gcl = createGroovyClassLoader(stepsClassLoader); + stepClass = (Class) loadClassFromSourceCode(gcl, stepUnitName); } else stepClass = (Class) stepsClassLoader.loadClass(stepUnitName); - if (stepClass == null) { - throw new Exception("Missing " + stepUnitName); - } + if (stepClass == null) + throw new Exception("Cant load " + stepUnitName); + result = stepClass.newInstance(); + logger.info("Class for step {} successfully created", stepUnitName); return result; } + private URLClassLoader createParentClassLoader() throws MalformedURLException { + try { + String stepClassesDir = systemConfig.getConfig(DIR_CLASSES); + Path path = Paths.get(stepClassesDir); + URL[] urls = { path.toUri().toURL() }; + logger.info("Creating URLClassLoader with {}", this.getClass().getClassLoader()); + // stepsClassLoader = URLClassLoader.newInstance(urls, Thread.currentThread().getContextClassLoader()); + stepsClassLoader = URLClassLoader.newInstance(urls, Thread.currentThread().getContextClassLoader()); + } catch (FileNotFoundException e) { + //nincs jobs/classes mappa + URL[] urls = {}; + logger.info("Creating URLClassLoader"); + // stepsClassLoader = URLClassLoader.newInstance(urls, Thread.currentThread().getContextClassLoader()); + stepsClassLoader = URLClassLoader.newInstance(urls, this.getClass().getClassLoader()); + } + return stepsClassLoader; + } + @Override public ScheduledJob createScheduledJob(BasicDBObject jobJSON, IJobEngine jobEngine) { ScheduledJob sj = new ScheduledJob(); @@ -186,6 +213,20 @@ public class JobEngineConfiguration implements IJobEngineConfiguration { loadSchedules(); } + private Class loadClassFromSourceCode(GroovyClassLoader gcl, String className) throws IOException { + Class result = null; + + String stepsDir = systemConfig.getConfig(DIR_STEPS); + Path path = Paths.get(stepsDir, className); + if (!path.toFile().exists()) + return result; + + result = gcl.parseClass(path.toFile()); + if (result != null) + logger.info("Successfully loaded {}", className); + return result; + } + @Override public void loadExecutors(IJobEngine jobEngine, boolean autoStart) { InputStream stream = null; @@ -239,11 +280,32 @@ public class JobEngineConfiguration implements IJobEngineConfiguration { BasicDBObject dbo = (BasicDBObject) JSONUtil.jsonToDbObject(jsonConfig); BasicDBList scheduleJobs = NoSQLUtils.asDBList(dbo, "joblist"); scheduleJobs.forEach(o -> { - BasicDBObject j = (BasicDBObject) o; - schedules.put(j.getString("template"), j); + try { + BasicDBObject j = (BasicDBObject) o; + String template = j.getString("template"); + String xml = loadTemplateXml(template); + j.put("xml", xml); + schedules.put(template, j); + } catch (Exception e) { + logger.error("Scheduled job error! {}", e.getMessage()); + } + }); } + private void loadStepSources(CompilationUnit compilationUnit, Path path) { + if (path.toFile().isDirectory()) { + Predicate filter = file -> !Files.isDirectory(file) && file.getFileName().toString().endsWith(".java"); + try (Stream stream = Files.list(path)) { + stream.filter(filter).forEach(p -> { + compilationUnit.addSource(p.toFile()); + }); + } catch (Exception e) { + } + + } + } + private void loadTemplate(Path config) { InputStream stream = null; try { @@ -304,42 +366,4 @@ public class JobEngineConfiguration implements IJobEngineConfiguration { } return result; } - - private Class parseClassHierarchy(GroovyClassLoader cl, String root, String className) throws IOException { - Path path = Paths.get(root, className); - - Class result = null; - if (!path.toFile().exists()) - return result; - List lines = Files.readAllLines(path); - - String packageName = null; - for (String line : lines) { - String trimmedLine = line.trim(); - if (trimmedLine.startsWith("package")) { - String[] tokens = trimmedLine.split(" "); - packageName = tokens[1].trim().replace(";", ""); - } - - if (trimmedLine.startsWith("import") && trimmedLine.contains(packageName)) { - String[] tokens = trimmedLine.split("\\."); - String name = tokens[tokens.length - 1]; - name = name.replace(";", ""); - logger.info("Try loading imported file {}", name); - parseClassHierarchy(cl, root, name + ".java"); - } - - if (trimmedLine.contains("extends")) { - String[] tokens = trimmedLine.split("extends"); - String parent = tokens[tokens.length - 1]; - parent = parent.replace("{", "").trim(); - //logger.info("Class {} needs parent class {}", className, parent); - logger.info("Try loading parent file {}", parent); - parseClassHierarchy(cl, root, parent + ".java"); - break; - } - } - result = cl.parseClass(path.toFile()); - return result; - } } diff --git a/server/user.jobengine.osgi.server/src/user/jobengine/server/JobStepExecutor.java b/server/user.jobengine.osgi.server/src/user/jobengine/server/JobStepExecutor.java index 54bd8515..588bfe59 100644 --- a/server/user.jobengine.osgi.server/src/user/jobengine/server/JobStepExecutor.java +++ b/server/user.jobengine.osgi.server/src/user/jobengine/server/JobStepExecutor.java @@ -111,10 +111,14 @@ public class JobStepExecutor implements IJobStepExecutor { logger.error("Error in {}", jobRuntime); Throwable t = e.getCause() == null ? e : e.getCause(); logger.catching(t); - if (JobStatus.EXECUTING.equals(jobRuntime.getStatus())) - jobEngine.suspendExecutingJob(t, jobRuntime); - if (JobStatus.WAIT_EXECUTOR.equals(jobRuntime.getStatus())) - jobEngine.suspendWaitExecutorJob(t, jobRuntime); + + if (jobRuntime != null) { + logger.error(jobRuntime.getSessionMarker(), t.getMessage()); + if (JobStatus.EXECUTING.equals(jobRuntime.getStatus())) + jobEngine.suspendExecutingJob(t, jobRuntime); + if (JobStatus.WAIT_EXECUTOR.equals(jobRuntime.getStatus())) + jobEngine.suspendWaitExecutorJob(t, jobRuntime); + } } finally { if (step != null) { step.cleanup(); @@ -144,10 +148,16 @@ public class JobStepExecutor implements IJobStepExecutor { if (step != null) { ClassLoader cl = step.getClass().getClassLoader(); if (cl instanceof GroovyClassLoader) { - URLClassLoader parentCl = (URLClassLoader) cl.getParent(); - GroovyClassLoader groovyCl = (GroovyClassLoader) cl; - groovyCl.close(); - parentCl.close(); + + GroovyClassLoader stepClassLoader = (GroovyClassLoader) cl; + stepClassLoader.close(); + + URLClassLoader stepParentClassLoader = (URLClassLoader) cl.getParent(); + stepParentClassLoader.close(); + } else if (cl instanceof URLClassLoader) { + URLClassLoader stepClassLoader = (URLClassLoader) cl; + stepClassLoader.close(); + } } } diff --git a/server/user.jobengine.osgi.server/src/user/jobengine/server/steps/JobStep.java b/server/user.jobengine.osgi.server/src/user/jobengine/server/steps/JobStep.java index 41928c9d..02a2a796 100644 --- a/server/user.jobengine.osgi.server/src/user/jobengine/server/steps/JobStep.java +++ b/server/user.jobengine.osgi.server/src/user/jobengine/server/steps/JobStep.java @@ -12,6 +12,8 @@ import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.apache.logging.log4j.Marker; import org.apache.logging.log4j.MarkerManager; +import org.apache.logging.log4j.message.Message; +import org.apache.logging.log4j.message.ParameterizedMessage; import org.osgi.framework.Bundle; import org.osgi.framework.BundleContext; import org.osgi.framework.FrameworkUtil; @@ -199,6 +201,11 @@ public class JobStep implements IJobStep { return result; } + protected void setDescription(String description, Object... arguments) { + Message msg = new ParameterizedMessage(description, arguments); + getJobRuntime().setDescription(msg.getFormattedMessage()); + } + protected void setProgress(int progress) { if (jobRuntime != null) jobRuntime.incrementProgress(progress); diff --git a/server/user.mediacube.gui/pages/jobparameters.zul b/server/user.mediacube.gui/pages/jobparameters.zul index 36d57dd2..099e0f2d 100644 --- a/server/user.mediacube.gui/pages/jobparameters.zul +++ b/server/user.mediacube.gui/pages/jobparameters.zul @@ -1,7 +1,12 @@ - + + + @@ -19,7 +24,8 @@ - -