From 0aa3fe6ae5dccbb8f55ae3af6705f6df3014f455 Mon Sep 17 00:00:00 2001 From: "vasary.daniel" Date: Thu, 24 Mar 2022 11:38:28 +0000 Subject: [PATCH] git-tfs-id: [http://tfs.userrendszerhaz.hu:8080/tfs/DefaultCollection]$/MediaCube;C33106 --- .../production/AMC/jobs/executors.xml | 9 ++- ...archive.xml => prores-archive-checker.xml} | 4 +- .../mediacube/executors/tests/SmallTests.java | 39 +++++++++- .../server/steps/AnalyzeMediaFilesStep.java | 22 +++--- .../steps/GenerateJSONMetadataStep.java | 73 ++++++++++--------- .../steps/ProResArchiveCheckerStep.java | 35 +++++++++ .../server/steps/ProResArchiveStep.java | 58 --------------- .../server/steps/ValidateProResStep.java | 28 ++++--- .../src/user/commons/mediaarea/MediaArea.java | 4 +- ....sql => 039_create_local_prores_store.sql} | 0 .../scripts/040_create_AMC_prores_store.sql | 17 +++++ 11 files changed, 164 insertions(+), 125 deletions(-) rename server/-product/production/AMC/jobs/templates/{prores-archive.xml => prores-archive-checker.xml} (90%) create mode 100644 server/user.jobengine.executors/src/user/jobengine/server/steps/ProResArchiveCheckerStep.java delete mode 100644 server/user.jobengine.executors/src/user/jobengine/server/steps/ProResArchiveStep.java rename server/user.jobengine.osgi.db/migrations/scripts/{039_amc_prores_create_store_entry.sql => 039_create_local_prores_store.sql} (100%) create mode 100644 server/user.jobengine.osgi.db/migrations/scripts/040_create_AMC_prores_store.sql diff --git a/server/-product/production/AMC/jobs/executors.xml b/server/-product/production/AMC/jobs/executors.xml index f0543ff1..b14141f3 100644 --- a/server/-product/production/AMC/jobs/executors.xml +++ b/server/-product/production/AMC/jobs/executors.xml @@ -2,6 +2,11 @@ + + + + + @@ -36,8 +41,4 @@ - - - - \ No newline at end of file diff --git a/server/-product/production/AMC/jobs/templates/prores-archive.xml b/server/-product/production/AMC/jobs/templates/prores-archive-checker.xml similarity index 90% rename from server/-product/production/AMC/jobs/templates/prores-archive.xml rename to server/-product/production/AMC/jobs/templates/prores-archive-checker.xml index 2d49bcba..3f2b38c6 100644 --- a/server/-product/production/AMC/jobs/templates/prores-archive.xml +++ b/server/-product/production/AMC/jobs/templates/prores-archive-checker.xml @@ -17,10 +17,10 @@ - + - + diff --git a/server/hu.user.mediacube.executors.tests/src/hu/user/mediacube/executors/tests/SmallTests.java b/server/hu.user.mediacube.executors.tests/src/hu/user/mediacube/executors/tests/SmallTests.java index 302286e7..f8558bb4 100644 --- a/server/hu.user.mediacube.executors.tests/src/hu/user/mediacube/executors/tests/SmallTests.java +++ b/server/hu.user.mediacube.executors.tests/src/hu/user/mediacube/executors/tests/SmallTests.java @@ -41,6 +41,7 @@ import java.util.TreeMap; import java.util.TreeSet; import java.util.concurrent.TimeUnit; import java.util.regex.Pattern; +import java.util.stream.Collectors; import org.apache.commons.io.FileUtils; import org.apache.commons.io.FilenameUtils; @@ -1478,8 +1479,42 @@ public class SmallTests { @Test public void test999999() throws Exception { - System.out.println("Hello"); - +// Path path = Paths.get("c:\\data\\output.mov"); +// System.out.println(FilenameUtils.removeExtension(path.getFileName().toString())); +// System.out.println(path.getParent().toString()); +// +// try { +// throw new RuntimeException(); +// } catch (Exception e) { +// System.out.println("Catched"); +// } + +// Stream stream = Files.walk(Paths.get("/opt")); +// Set collect = stream.map(Path::toString).collect(Collectors.toSet()); +// collect.forEach(file -> { +// System.out.println(file); +// }); + +// MediaArea m = new MediaArea(Paths.get("/data/video/sample_960x400_ocean_with_audio.mxf")); +// m.process(); +// System.out.println(m.getFrameCount()); + +// String string = Paths.get("/data/video/sample_960x400_ocean_with_audio.mxf").toAbsolutePath().toString(); +// System.out.println(string); +// MediaArea m = new MediaArea(Paths.get(string)); +// System.out.println(m.getFrameCount()); + + Map inputFiles = new HashMap(); + inputFiles.put("A", 1); + inputFiles.put("B", 2); + + Map result = inputFiles.entrySet().stream().filter(fileEntry -> { + return fileEntry.getValue() > 1; + }).collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue)); + + result.entrySet().forEach(i -> { + System.out.println(i.getKey() + " " + i.getValue()); + }); } } diff --git a/server/user.jobengine.executors/src/user/jobengine/server/steps/AnalyzeMediaFilesStep.java b/server/user.jobengine.executors/src/user/jobengine/server/steps/AnalyzeMediaFilesStep.java index 98cf7fa9..eed9ba4f 100644 --- a/server/user.jobengine.executors/src/user/jobengine/server/steps/AnalyzeMediaFilesStep.java +++ b/server/user.jobengine.executors/src/user/jobengine/server/steps/AnalyzeMediaFilesStep.java @@ -14,15 +14,19 @@ public class AnalyzeMediaFilesStep extends JobStep { private static final Logger logger = LogManager.getLogger(); @StepEntry - public Object[] execute(List foundFiles) { - Map proResFiles = new HashMap(foundFiles.size()); - - for (int i = 0; i < foundFiles.size(); i++) { - MediaArea mediaArea = new MediaArea(Paths.get(foundFiles.get(i))); - mediaArea.process(); - proResFiles.put(Paths.get(foundFiles.get(i)).toAbsolutePath().toString(), mediaArea); + public Object[] execute(List inputFiles) { + Map result = new HashMap(); + for (int i = 0; i < inputFiles.size(); i++) { + String inputFileLocation = inputFiles.get(i); + MediaArea mediaArea = new MediaArea(Paths.get(inputFileLocation)); + try { + mediaArea.process(); + result.put(inputFileLocation, mediaArea); + } catch (Exception e) { + logger.error(getSessionMarker(), "Can't analize input {}, skipping. System message is: {}", + e.getMessage()); + } } - - return new Object[] { proResFiles }; + return new Object[] { result }; } } diff --git a/server/user.jobengine.executors/src/user/jobengine/server/steps/GenerateJSONMetadataStep.java b/server/user.jobengine.executors/src/user/jobengine/server/steps/GenerateJSONMetadataStep.java index 210d7354..d322ff43 100644 --- a/server/user.jobengine.executors/src/user/jobengine/server/steps/GenerateJSONMetadataStep.java +++ b/server/user.jobengine.executors/src/user/jobengine/server/steps/GenerateJSONMetadataStep.java @@ -1,9 +1,11 @@ package user.jobengine.server.steps; import java.io.IOException; +import java.nio.file.Path; import java.nio.file.Paths; import java.sql.Timestamp; import java.util.Map; +import java.util.Set; import org.apache.commons.io.FilenameUtils; import org.apache.logging.log4j.LogManager; @@ -21,40 +23,15 @@ public class GenerateJSONMetadataStep extends JobStep { @StepEntry public Object[] execute(Map files, String sourceStoreName, String targetStoreName, String escortStoreFolder) { - if (!files.isEmpty()) { - for (String fullPath : files.keySet()) { - MediaArea mediaArea = files.get(fullPath); - String title = FilenameUtils.getBaseName(fullPath).replace(FilenameUtils.getExtension(fullPath), ""); - String pathOnly = fullPath.substring(0, fullPath.lastIndexOf("\\") + 1); - String filenameOnly = FilenameUtils.getBaseName(fullPath); - Timestamp created = null; - Timestamp modified = null; - long frameCount = mediaArea.getFrameCount(); - long mediaId = 0; - StoreUri sourceStoreUri = getManager().getStoreUri(sourceStoreName, RemoteStoreProtocol.LOCAL); - StoreUri targetStoreUri = getManager().getStoreUri(targetStoreName, RemoteStoreProtocol.LOCAL); - StoreUri escortStoreUri = getManager().getStoreUri(escortStoreFolder, RemoteStoreProtocol.LOCAL); - + if (files != null && !files.isEmpty()) { + Set keys = files.keySet(); + for (String inputFileLocation : keys) { + MediaArea mediaArea = files.get(inputFileLocation); try { - String outputPath = Paths.get(escortStoreUri.toString(true)).toString(); - } catch (Exception e1) { - e1.printStackTrace(); - } - - DownloadableMedia downloadable = DownloadableMedia.create(title, filenameOnly, modified, created, - frameCount, 0, sourceStoreUri.getId(), targetStoreUri.getId(), mediaId); - String escortFileName = targetStoreName + "." + downloadable.getString("fileName"); // needed without - // extension - try { - if (EscortFiles.createMetadataIfNotExists(pathOnly, escortFileName, - downloadable.toPrettyString(""))) { - logger.info(getMarker(), "Archive status file created for {}", fullPath); - } else { - logger.info(getMarker(), "Archive status file already exists for {}", fullPath); - } - } catch (IOException e) { - logger.error("{}", e.getCause()); - e.printStackTrace(); + processItem(sourceStoreName, targetStoreName, escortStoreFolder, inputFileLocation, mediaArea); + } catch (Exception e) { + logger.error(getSessionMarker(), e.getMessage()); + logger.catching(e); } } } else { @@ -62,4 +39,34 @@ public class GenerateJSONMetadataStep extends JobStep { } return null; } + + private void processItem(String sourceStoreName, String targetStoreName, String escortStoreFolder, + String inputFileLocation, MediaArea mediaArea) throws Exception { + Path inputPath = Paths.get(inputFileLocation); + + String fileName = inputPath.getFileName().toString(); + String title = FilenameUtils.removeExtension(fileName); + Timestamp created = null; + Timestamp modified = null; + long frameCount = mediaArea.getFrameCount(); + StoreUri sourceStoreUri = getManager().getStoreUri(sourceStoreName, RemoteStoreProtocol.LOCAL); + StoreUri targetStoreUri = getManager().getStoreUri(targetStoreName, RemoteStoreProtocol.LOCAL); + StoreUri escortStoreUri = getManager().getStoreUri(escortStoreFolder, RemoteStoreProtocol.LOCAL); + String outputPath = Paths.get(escortStoreUri.toString(true)).toString(); + + DownloadableMedia downloadable = DownloadableMedia.create(title, fileName, modified, created, frameCount, 0, + sourceStoreUri.getId(), targetStoreUri.getId(), 0); + String escortFileName = targetStoreName + "." + downloadable.getString("fileName"); // needed without + // extension + try { + if (EscortFiles.createMetadataIfNotExists(outputPath, escortFileName, downloadable.toPrettyString(""))) { + logger.info(getMarker(), "Archive status file created for {}", inputFileLocation); + } else { + logger.info(getMarker(), "Archive status file already exists for {}", inputFileLocation); + } + } catch (IOException e) { + logger.error("{}", e.getCause()); + e.printStackTrace(); + } + } } diff --git a/server/user.jobengine.executors/src/user/jobengine/server/steps/ProResArchiveCheckerStep.java b/server/user.jobengine.executors/src/user/jobengine/server/steps/ProResArchiveCheckerStep.java new file mode 100644 index 00000000..ad0685b7 --- /dev/null +++ b/server/user.jobengine.executors/src/user/jobengine/server/steps/ProResArchiveCheckerStep.java @@ -0,0 +1,35 @@ + +package user.jobengine.server.steps; + +import java.nio.file.Path; + +import com.ibm.nosql.json.api.BasicDBObject; + +import user.commons.StoreUri; +import user.commons.remotestore.RemoteStoreProtocol; +import user.jobengine.server.steps.shared.EscortFiles; +import user.jobengine.server.steps.shared.FileSearchFilterOptions; + +public class ProResArchiveCheckerStep extends PathItemsCollectorStep { + public static final String DOT_CATCHED = ".catched"; + public static final String STATUSFOLDER = ".STATUS"; + + @Override + protected FileSearchFilterOptions createFileFilter(BasicDBObject filter) { + FileSearchFilterOptions result = new FileSearchFilterOptions(filter) { + @Override + public boolean acceptFile(Path file) { + return EscortFiles.isMediaCatched(file) ? false : super.acceptFile(file); + } + }; + return result; + } + + @Override + @StepEntry + public Object[] execute(String sourceStoreName, BasicDBObject filter) throws Exception { + StoreUri sourceStoreUri = getManager().getStoreUri(sourceStoreName, RemoteStoreProtocol.LOCAL); + String sourceStoreFolder = sourceStoreUri.toString(false); + return super.execute(sourceStoreFolder, filter); + } +} diff --git a/server/user.jobengine.executors/src/user/jobengine/server/steps/ProResArchiveStep.java b/server/user.jobengine.executors/src/user/jobengine/server/steps/ProResArchiveStep.java deleted file mode 100644 index d3fcfaf9..00000000 --- a/server/user.jobengine.executors/src/user/jobengine/server/steps/ProResArchiveStep.java +++ /dev/null @@ -1,58 +0,0 @@ - -package user.jobengine.server.steps; - -import java.nio.file.Files; -import java.nio.file.Path; -import java.nio.file.Paths; - -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; - -import com.ibm.nosql.json.api.BasicDBObject; - -import user.jobengine.server.steps.shared.FileSearchFilterOptions; - -public class ProResArchiveStep extends PathItemsCollectorStep { - private static final Logger logger = LogManager.getLogger(); - public static final String DOT_CATCHED = ".catched"; - public static final String STATUSFOLDER = ".STATUS"; - - @Override - protected FileSearchFilterOptions createFileFilter(BasicDBObject filter) { - String fileName = filter.getString("fileName").replaceAll("[^a-zA-Z0-9]", ""); - - FileSearchFilterOptions result = new FileSearchFilterOptions(filter) { - @Override - public boolean acceptFile(Path file) { - if (!isMediaCatched(file)) {/* !catched van-e */ - return super.acceptFile(file); - } - return false; - } - }; - return result; - } - - @Override - @StepEntry - public Object[] execute(String sourceStoreFolder, BasicDBObject filter) throws Exception { - return super.execute(sourceStoreFolder, filter); - } - - /*** - * A média elérési útján alapján a .STATUS almappában vizsgálja .catch fajl - * létezését. - * - * @param mediaFile - * @return - */ - public static boolean isMediaCatched(Path mediaFile) { - Path catchedFile = createMediaCatchFilePath(mediaFile); - return Files.exists(catchedFile); - } - - public static Path createMediaCatchFilePath(Path mediaFile) { - String fileName = mediaFile.getFileName().toString() + DOT_CATCHED; - return Paths.get(mediaFile.getParent().toString(), STATUSFOLDER, fileName); - } -} diff --git a/server/user.jobengine.executors/src/user/jobengine/server/steps/ValidateProResStep.java b/server/user.jobengine.executors/src/user/jobengine/server/steps/ValidateProResStep.java index e36f784d..d515cbea 100644 --- a/server/user.jobengine.executors/src/user/jobengine/server/steps/ValidateProResStep.java +++ b/server/user.jobengine.executors/src/user/jobengine/server/steps/ValidateProResStep.java @@ -1,7 +1,7 @@ package user.jobengine.server.steps; -import java.util.HashMap; import java.util.Map; +import java.util.stream.Collectors; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; @@ -9,24 +9,22 @@ import org.apache.logging.log4j.Logger; import user.commons.mediaarea.MediaArea; public class ValidateProResStep extends JobStep { + private static final double FRAMERATE = 23.976; + private static final String PRORES = "PRORES"; private static final Logger logger = LogManager.getLogger(); @StepEntry - public Object[] execute(Map proResFiles) { - Map files = new HashMap(); + public Object[] execute(Map inputFiles) { - if (!proResFiles.isEmpty()) { - for (String fileName : proResFiles.keySet()) { - MediaArea mediaArea = proResFiles.get(fileName); - String videoFormat = mediaArea.getFormat(); + Map result = inputFiles.entrySet().stream().filter(fileEntry -> { + String fileLocation = fileEntry.getKey(); + MediaArea mediaArea = inputFiles.get(fileLocation); + String videoFormat = mediaArea.getFormat(); + boolean isProRes = mediaArea.getFrameRate() == FRAMERATE && videoFormat.equals(PRORES); + logger.info(getSessionMarker(), "{} is PRORES {}", fileLocation, isProRes); + return isProRes; + }).collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue)); - if (mediaArea.getFrameRate() == 23.976 && videoFormat.equals("PRORES")) { - files.put(fileName, mediaArea); - } - } - } else { - logger.info("proResFiles is empty!"); - } - return new Object[] { files }; + return new Object[] { result }; } } diff --git a/server/user.jobengine.osgi.commons/src/user/commons/mediaarea/MediaArea.java b/server/user.jobengine.osgi.commons/src/user/commons/mediaarea/MediaArea.java index 6fa5251b..46fcfcb3 100644 --- a/server/user.jobengine.osgi.commons/src/user/commons/mediaarea/MediaArea.java +++ b/server/user.jobengine.osgi.commons/src/user/commons/mediaarea/MediaArea.java @@ -60,7 +60,7 @@ public class MediaArea { return result; } - public void process() { + public void process() throws Exception { // auto closable try (MediaInfo mi = new MediaInfo(filePath.toAbsolutePath().toString())) { height = Integer.parseInt(mi.get(StreamKind.Video, 0, "Height")); @@ -71,7 +71,7 @@ public class MediaArea { frameRate = Double.parseDouble(mi.get(StreamKind.Video, 0, "FrameRate")); frameCount = Integer.parseInt(mi.get(StreamKind.Video, 0, "FrameCount")); } catch (Exception ex) { - throw new RuntimeException(ex); + throw new Exception(ex); } } } diff --git a/server/user.jobengine.osgi.db/migrations/scripts/039_amc_prores_create_store_entry.sql b/server/user.jobengine.osgi.db/migrations/scripts/039_create_local_prores_store.sql similarity index 100% rename from server/user.jobengine.osgi.db/migrations/scripts/039_amc_prores_create_store_entry.sql rename to server/user.jobengine.osgi.db/migrations/scripts/039_create_local_prores_store.sql diff --git a/server/user.jobengine.osgi.db/migrations/scripts/040_create_AMC_prores_store.sql b/server/user.jobengine.osgi.db/migrations/scripts/040_create_AMC_prores_store.sql new file mode 100644 index 00000000..9078c1d6 --- /dev/null +++ b/server/user.jobengine.osgi.db/migrations/scripts/040_create_AMC_prores_store.sql @@ -0,0 +1,17 @@ +-- // Creates a store entry in STORE table +-- Migration SQL that makes the change goes here. + +INSERT INTO STORE(NAME,ISSYSTEM,ISLOWRES) VALUES +('PRORES','N','N') +@ + +INSERT INTO STOREURI(STOREID,PROTOCOL,DELIVERY,URI,ISSTREAM,ISSOURCE,ISTARGET,USERNAME,PASSWORD,ROOTPATH,PORTNUMBER) VALUES +((SELECT ID FROM STORE WHERE NAME='PRORES'),'LOCAL',null,'/mediacube/data','N','Y','Y',null,null,null,null) +@ + +-- //@UNDO +-- SQL to undo the change goes here. +DELETE FROM STOREURI WHERE STOREID=(SELECT ID FROM STORE WHERE NAME='AMC_LOCAL') +@ +DELETE FROM STORE WHERE NAME='AMC_LOCAL' +@ -- 2.54.0