From eda2fd816124346fa2894668ad1f8d88155b666b Mon Sep 17 00:00:00 2001 From: "vasary.daniel" Date: Tue, 5 Apr 2022 11:43:47 +0000 Subject: [PATCH] =?utf8?q?#319=20ProRes=20archiv=C3=A1l=C3=A1s?= MIME-Version: 1.0 Content-Type: text/plain; charset=utf8 Content-Transfer-Encoding: 8bit git-tfs-id: [http://tfs.userrendszerhaz.hu:8080/tfs/DefaultCollection]$/MediaCube;C33132 --- .../production/AMC/jobs/schedules.json | 10 +-- .../templates/PRORES-validate-and-archive.xml | 79 +++++++++++++++++++ .../jobs/templates/prores-archive-checker.xml | 9 +-- .../mediacube/executors/tests/SmallTests.java | 21 +++++ .../server/steps/AnalyzeMediaFilesStep.java | 7 ++ .../server/steps/ArchiveRecursive.java | 27 ++----- .../server/steps/FileCleanupStep.java | 21 ++++- .../server/steps/ForkDownloadStep.java | 70 ++++++++++------ .../steps/GenerateJSONMetadataStep.java | 60 +++++++++----- .../server/steps/MetadataPersisterStep.java | 31 +++++--- .../server/steps/PRORESTransferFromStep.java | 20 +++++ .../steps/ProResArchiveCheckerStep.java | 9 ++- .../server/steps/SafeDeleteRecursiveStep.java | 39 ++++++--- .../server/steps/TSMTransferToStep.java | 9 +-- .../server/steps/ValidateProResStep.java | 17 +++- .../server/steps/shared/EscortFiles.java | 9 ++- .../scripts/040_create_AMC_prores_store.sql | 6 +- 17 files changed, 325 insertions(+), 119 deletions(-) create mode 100644 server/-product/production/AMC/jobs/templates/PRORES-validate-and-archive.xml create mode 100644 server/user.jobengine.executors/src/user/jobengine/server/steps/PRORESTransferFromStep.java diff --git a/server/-product/production/AMC/jobs/schedules.json b/server/-product/production/AMC/jobs/schedules.json index 5017a2e2..87e40df9 100644 --- a/server/-product/production/AMC/jobs/schedules.json +++ b/server/-product/production/AMC/jobs/schedules.json @@ -145,17 +145,15 @@ ] }, { - "template": "prores-archive.xml", - "name" : "ProRes archiválás", + "template": "PRORES-archive-checker.xml", + "name" : "PRORES archiválás előkészítés", "active": false, "executeimmediate": false, "parameters": [ - { "name": "sourceStoreFolder", "value": "c:\\data", "type": "java.lang.String" }, - { "name": "sourceStoreName", "value": "AMC_LOCAL", "type": "java.lang.String" }, + { "name": "sourceStoreName", "value": "PRORES", "type": "java.lang.String" }, { "name": "filter", "value": {"fileName": ".*\\.(mov)$"}, "type": "java.lang.String" }, - { "name": "targetStoreName", "value": "AMC_LOCAL", "type": "java.lang.String" }, - { "name": "escortStoreFolder", "value": "AMC_LOCAL", "type": "java.lang.String" } + { "name": "escortStoreName", "value": "TEST_DOWNLOADS", "type": "java.lang.String" } ] } ] diff --git a/server/-product/production/AMC/jobs/templates/PRORES-validate-and-archive.xml b/server/-product/production/AMC/jobs/templates/PRORES-validate-and-archive.xml new file mode 100644 index 00000000..65c05b5b --- /dev/null +++ b/server/-product/production/AMC/jobs/templates/PRORES-validate-and-archive.xml @@ -0,0 +1,79 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/server/-product/production/AMC/jobs/templates/prores-archive-checker.xml b/server/-product/production/AMC/jobs/templates/prores-archive-checker.xml index 3f2b38c6..41951758 100644 --- a/server/-product/production/AMC/jobs/templates/prores-archive-checker.xml +++ b/server/-product/production/AMC/jobs/templates/prores-archive-checker.xml @@ -4,10 +4,8 @@ - - - + @@ -65,10 +63,7 @@ - - - - + diff --git a/server/hu.user.mediacube.executors.tests/src/hu/user/mediacube/executors/tests/SmallTests.java b/server/hu.user.mediacube.executors.tests/src/hu/user/mediacube/executors/tests/SmallTests.java index 8427465e..2b11696c 100644 --- a/server/hu.user.mediacube.executors.tests/src/hu/user/mediacube/executors/tests/SmallTests.java +++ b/server/hu.user.mediacube.executors.tests/src/hu/user/mediacube/executors/tests/SmallTests.java @@ -41,6 +41,8 @@ import java.util.TreeMap; import java.util.TreeSet; import java.util.concurrent.TimeUnit; import java.util.regex.Pattern; +import java.util.stream.Collector; +import java.util.stream.Collectors; import org.apache.commons.io.FileUtils; import org.apache.commons.io.FilenameUtils; @@ -1551,4 +1553,23 @@ public class SmallTests { // } } + @Test + public void test9999991() throws Exception { + Map sut = new HashMap(); + + sut.put("E1", 1); + sut.put("E2", 2); + sut.put("E3", 3); + + Collector, ?, Map> collector = Collectors.toMap(x -> { + Map.Entry entry = x; + return entry.getKey(); + }, y -> { + Map.Entry entry = y; + return entry.getValue(); + }); + + Map collected = sut.entrySet().stream().collect(collector); + + } } diff --git a/server/user.jobengine.executors/src/user/jobengine/server/steps/AnalyzeMediaFilesStep.java b/server/user.jobengine.executors/src/user/jobengine/server/steps/AnalyzeMediaFilesStep.java index 8f255029..8b167929 100644 --- a/server/user.jobengine.executors/src/user/jobengine/server/steps/AnalyzeMediaFilesStep.java +++ b/server/user.jobengine.executors/src/user/jobengine/server/steps/AnalyzeMediaFilesStep.java @@ -16,17 +16,24 @@ public class AnalyzeMediaFilesStep extends JobStep { @StepEntry public Object[] execute(List inputFiles) { Map result = new HashMap(); + + int progress = 0; + for (int i = 0; i < inputFiles.size(); i++) { String inputFileLocation = inputFiles.get(i); MediaArea mediaArea = new MediaArea(Paths.get(inputFileLocation)); try { mediaArea.process(); result.put(inputFileLocation, mediaArea); + progress = i * 100 / inputFiles.size(); + setProgress(progress); } catch (Exception e) { logger.error(getSessionMarker(), "Can't analyze input {}, skipping. System message is: {}", e.getMessage()); } } + + setProgress(100); return new Object[] { result }; } } diff --git a/server/user.jobengine.executors/src/user/jobengine/server/steps/ArchiveRecursive.java b/server/user.jobengine.executors/src/user/jobengine/server/steps/ArchiveRecursive.java index 35fd6530..cb4db0c2 100644 --- a/server/user.jobengine.executors/src/user/jobengine/server/steps/ArchiveRecursive.java +++ b/server/user.jobengine.executors/src/user/jobengine/server/steps/ArchiveRecursive.java @@ -102,36 +102,21 @@ public class ArchiveRecursive extends JobStep implements FileVisitor { String fileName = filePath.getFileName().toString(); String mediaHouseId = FilenameUtils.removeExtension(fileName); + + String mediaHouseIdUpper = mediaHouseId.toUpperCase(); + if (mediaHouseIdUpper.endsWith("_CLEAN")) + mediaHouseId = mediaHouseIdUpper.replace("_CLEAN", ""); + try { result = getPlanAirMetadata(mediaHouseId); - if (result != null) { + if (result != null) result.setMediaFile(filePath.toString()); - // 210617 proxy keszites tiltasa - result.setDisableProxy(disableProxy); - } - logger.info(getSessionMarker(), "PlanAir query done for {}", filePath); } catch (Exception e) { logger.error(getSessionMarker(), "PlanAir metadata error", e); - // nem latja a drivert pl. - throw e; } -//210616 Ha nincs metaadat, nem archivalunk -// if (result == null) { -// result = new ArchiveItem(); -// BasicFileAttributes attr = Files.readAttributes(filePath, BasicFileAttributes.class); -// result.setItemHouseId(df.format(attr.lastModifiedTime().toMillis())); -// result.setItemTitle(filePath.getParent().toString()); -// result.setMediaHouseId(mediaHouseId); -// result.setMediaTitle(fileName); -// // TODO kesobb kivenni -// // result.setMediaDescription("/OMARCHIVE"); -// result.setMediaType("Generic"); -// result.setMediaFile(filePath.toString()); -// } - return result; } diff --git a/server/user.jobengine.executors/src/user/jobengine/server/steps/FileCleanupStep.java b/server/user.jobengine.executors/src/user/jobengine/server/steps/FileCleanupStep.java index 8e1a6f4c..5b07be6a 100644 --- a/server/user.jobengine.executors/src/user/jobengine/server/steps/FileCleanupStep.java +++ b/server/user.jobengine.executors/src/user/jobengine/server/steps/FileCleanupStep.java @@ -14,6 +14,17 @@ import user.jobengine.server.steps.shared.EscortFiles; public class FileCleanupStep extends JobStep { private static final Logger logger = LogManager.getLogger(); + /*** + * Torli a sourceStoreUri + fileName fajlt, az escortFile-t es az + * escortFile.CATCHED file-t. + * + * @param sourceStoreUri + * @param fileName + * @param escortFile + * @return + * @throws Exception + */ + @StepEntry public Object[] execute(StoreUri sourceStoreUri, String fileName, String escortFile) throws Exception { try { @@ -23,8 +34,14 @@ public class FileCleanupStep extends JobStep { Path filePath = Paths.get(sourceStoreUri.toString(true), fileName); EscortFiles.remove(filePath); - EscortFiles.remove(Paths.get(escortFile)); - EscortFiles.remove(Paths.get(escortFile + EscortFiles.DOT_CATCHED)); + + Path escortFilePath = Paths.get(escortFile); + if (escortFilePath.toFile().exists()) + EscortFiles.remove(escortFilePath); + + Path escortCatchedFilePath = Paths.get(escortFile + EscortFiles.DOT_CATCHED); + if (escortCatchedFilePath.toFile().exists()) + EscortFiles.remove(escortCatchedFilePath); } catch (Exception e) { logger.error(e.getMessage()); throw e; diff --git a/server/user.jobengine.executors/src/user/jobengine/server/steps/ForkDownloadStep.java b/server/user.jobengine.executors/src/user/jobengine/server/steps/ForkDownloadStep.java index 35431e50..19467d11 100644 --- a/server/user.jobengine.executors/src/user/jobengine/server/steps/ForkDownloadStep.java +++ b/server/user.jobengine.executors/src/user/jobengine/server/steps/ForkDownloadStep.java @@ -142,32 +142,20 @@ public class ForkDownloadStep extends JobStep { if (targetExists(tempStoreUri, sourceFileName)) priority = 51; - //parameterek: IJobRuntime parent, String template, String name, int priority, Map parameters + // parameterek: IJobRuntime parent, String template, String name, int priority, + // Map parameters IJobRuntime child = getEngine().submit(null, e -> { IJobRuntime r = (IJobRuntime) e.getSource(); - if (r.isDisableRetry()) - return; - - if (e.getStatus().equals(JobStatus.CANCELED) || e.getStatus().equals(JobStatus.SUSPENDED)) - EscortFiles.removeCatchedFile(escortFile); - - if (e.getStatus().equals(JobStatus.FINISHED) && "PBARCHIVE".equals(sourceStoreName)) { - try { - FtpDirectoryLister lister = (FtpDirectoryLister) sourceStoreUri.getLister(); - RemoteFile remoteFile = lister.get(sourceFileName); - if (remoteFile == null) - logger.info(getMarker(), "Remote file for {} is not available", sourceFileName); - else { - logger.info(getMarker(), "Remote file {} successfully archived, deleting...", - sourceFileName); - lister.delete(remoteFile); - } - } catch (Exception ex) { - logger.error(getMarker(), ex.getMessage()); - } finally { - if (sourceStoreUri != null) - sourceStoreUri.cleanUp(); - } + if (e.getStatus().equals(JobStatus.CANCELED) || e.getStatus().equals(JobStatus.SUSPENDED)) { + if (r.isDisableRetry()) + EscortFiles.removeCatchedFile(escortFile); + } + + if (e.getStatus().equals(JobStatus.FINISHED)) { + if ("PBARCHIVE".equals(sourceStoreName)) + successPBARCHIVEPostProcess(sourceStoreUri, sourceFileName, r); + if ("PRORES".equals(sourceStoreName)) + successPRORESPostProcess(sourceStoreUri, sourceFileName, r); } }, template, "Archiválás", priority, IJobEngine.DEFAULT_OWNER, parameters); @@ -188,6 +176,40 @@ public class ForkDownloadStep extends JobStep { return true; } + private void successPBARCHIVEPostProcess(StoreUri sourceStoreUri, String sourceFileName, IJobRuntime runtime) { + try { + FtpDirectoryLister lister = (FtpDirectoryLister) sourceStoreUri.getLister(); + RemoteFile remoteFile = lister.get(sourceFileName); + if (remoteFile == null) + logger.info(getMarker(), "Remote file for {} is not available", sourceFileName); + else { + logger.info(getMarker(), "Remote file {} successfully archived, deleting...", sourceFileName); + lister.delete(remoteFile); + } + } catch (Exception ex) { + logger.error(getMarker(), ex.getMessage()); + } finally { + if (sourceStoreUri != null) + sourceStoreUri.cleanUp(); + } + } + + private void successPRORESPostProcess(StoreUri sourceStoreUri, String sourceFileName, IJobRuntime runtime) { + try { + Path filePath = Paths.get(sourceStoreUri.toString(true), sourceFileName); + EscortFiles.remove(filePath); + + if (EscortFiles.isMediaCatched(filePath)) + EscortFiles.remove(EscortFiles.createMediaCatchFilePath(filePath)); + + } catch (Exception ex) { + logger.error(getMarker(), ex.getMessage()); + } finally { + if (sourceStoreUri != null) + sourceStoreUri.cleanUp(); + } + } + private boolean targetExists(StoreUri storeUri, String fileName) { try { Path targetFile = Paths.get(storeUri.toString(true), fileName); diff --git a/server/user.jobengine.executors/src/user/jobengine/server/steps/GenerateJSONMetadataStep.java b/server/user.jobengine.executors/src/user/jobengine/server/steps/GenerateJSONMetadataStep.java index d322ff43..b20304cd 100644 --- a/server/user.jobengine.executors/src/user/jobengine/server/steps/GenerateJSONMetadataStep.java +++ b/server/user.jobengine.executors/src/user/jobengine/server/steps/GenerateJSONMetadataStep.java @@ -1,9 +1,13 @@ package user.jobengine.server.steps; import java.io.IOException; +import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.Paths; +import java.nio.file.attribute.BasicFileAttributes; +import java.nio.file.attribute.FileTime; import java.sql.Timestamp; +import java.util.Date; import java.util.Map; import java.util.Set; @@ -21,49 +25,65 @@ public class GenerateJSONMetadataStep extends JobStep { private static final Logger logger = LogManager.getLogger(); @StepEntry - public Object[] execute(Map files, String sourceStoreName, String targetStoreName, - String escortStoreFolder) { + public Object[] execute(Map files, String sourceStoreName, String escortStoreName) + throws Exception { + + int progress = 0; + int i = 0; if (files != null && !files.isEmpty()) { + + StoreUri sourceStoreUri = getManager().getStoreUri(sourceStoreName, RemoteStoreProtocol.LOCAL); + StoreUri targetStoreUri = getManager().getStoreUri("TSM", RemoteStoreProtocol.TSM); + StoreUri escortStoreUri = getManager().getStoreUri(escortStoreName, RemoteStoreProtocol.LOCAL); + String outputPath = Paths.get(escortStoreUri.toString(true)).toString(); + Set keys = files.keySet(); for (String inputFileLocation : keys) { MediaArea mediaArea = files.get(inputFileLocation); try { - processItem(sourceStoreName, targetStoreName, escortStoreFolder, inputFileLocation, mediaArea); + processItem(inputFileLocation, sourceStoreName, sourceStoreUri, targetStoreUri, outputPath, + mediaArea); + progress = i * 100 / files.size(); + setProgress(progress); + i++; } catch (Exception e) { logger.error(getSessionMarker(), e.getMessage()); logger.catching(e); } } - } else { - logger.info("files is empty!"); } + + setProgress(100); + return null; } - private void processItem(String sourceStoreName, String targetStoreName, String escortStoreFolder, - String inputFileLocation, MediaArea mediaArea) throws Exception { - Path inputPath = Paths.get(inputFileLocation); - - String fileName = inputPath.getFileName().toString(); + private void processItem(String inputFileLocation, String sourceStoreName, StoreUri sourceStoreUri, + StoreUri targetStoreUri, String outputPath, MediaArea mediaArea) throws Exception { + Path inputFilePath = Paths.get(inputFileLocation); + String fileName = inputFilePath.getFileName().toString(); String title = FilenameUtils.removeExtension(fileName); - Timestamp created = null; - Timestamp modified = null; - long frameCount = mediaArea.getFrameCount(); - StoreUri sourceStoreUri = getManager().getStoreUri(sourceStoreName, RemoteStoreProtocol.LOCAL); - StoreUri targetStoreUri = getManager().getStoreUri(targetStoreName, RemoteStoreProtocol.LOCAL); - StoreUri escortStoreUri = getManager().getStoreUri(escortStoreFolder, RemoteStoreProtocol.LOCAL); - String outputPath = Paths.get(escortStoreUri.toString(true)).toString(); + BasicFileAttributes attr = Files.readAttributes(inputFilePath, BasicFileAttributes.class); + FileTime creationTime = attr.creationTime(); + FileTime lastModifiedTime = attr.lastModifiedTime(); - DownloadableMedia downloadable = DownloadableMedia.create(title, fileName, modified, created, frameCount, 0, + Timestamp created = Timestamp.from(new Date(creationTime.toMillis()).toInstant()); + Timestamp modified = Timestamp.from(new Date(lastModifiedTime.toMillis()).toInstant()); + long frameCount = mediaArea.getFrameCount(); + long size = inputFilePath.toFile().length(); + DownloadableMedia downloadable = DownloadableMedia.create(title, fileName, modified, created, frameCount, size, sourceStoreUri.getId(), targetStoreUri.getId(), 0); - String escortFileName = targetStoreName + "." + downloadable.getString("fileName"); // needed without - // extension + downloadable.append("addStoreNameAsFilePrefix", true); + + String escortFileName = sourceStoreName + "." + downloadable.getString("fileName"); + try { if (EscortFiles.createMetadataIfNotExists(outputPath, escortFileName, downloadable.toPrettyString(""))) { logger.info(getMarker(), "Archive status file created for {}", inputFileLocation); } else { logger.info(getMarker(), "Archive status file already exists for {}", inputFileLocation); } + EscortFiles.createMediaCatch(inputFilePath); } catch (IOException e) { logger.error("{}", e.getCause()); e.printStackTrace(); diff --git a/server/user.jobengine.executors/src/user/jobengine/server/steps/MetadataPersisterStep.java b/server/user.jobengine.executors/src/user/jobengine/server/steps/MetadataPersisterStep.java index 2a04cc47..26b2e68c 100644 --- a/server/user.jobengine.executors/src/user/jobengine/server/steps/MetadataPersisterStep.java +++ b/server/user.jobengine.executors/src/user/jobengine/server/steps/MetadataPersisterStep.java @@ -25,7 +25,7 @@ public class MetadataPersisterStep extends JobStep { try { Path escortFilePath = Paths.get(escortFile); - //ha nem ment at a letoltott file az ellenorzesen, akkor már torolve lett + // ha nem ment at a letoltott file az ellenorzesen, akkor már torolve lett if (escortFilePath.toFile().exists()) { String escortFileName = escortFilePath.getFileName().toString(); String sourceStoreName = escortFileName.substring(0, escortFileName.indexOf(".")); @@ -34,6 +34,7 @@ public class MetadataPersisterStep extends JobStep { String fileName = downloadable.getString("fileName"); long mediaId = downloadable.containsKey("mediaId") ? downloadable.getLong("mediaId") : 0; String fileTypeName = fileName.substring(fileName.lastIndexOf(".") + 1).toUpperCase(); + boolean addStoreNameAsFilePrefix = downloadable.getBoolean("addStoreNameAsFilePrefix"); IItemManager manager = getManager(); @@ -41,7 +42,7 @@ public class MetadataPersisterStep extends JobStep { mediaId = tryGetMediaId(title, mediaId); } - //TODO ha van mediaId, akkor update!!!!!!!!! + // TODO ha van mediaId, akkor update!!!!!!!!! Timestamp modified = downloadable.getTimestamp("modified"); Timestamp created = downloadable.getTimestamp("created"); @@ -49,14 +50,17 @@ public class MetadataPersisterStep extends JobStep { String targetStoreName = getManager().getStore(targetStoreUri.getStoreId()).getName(); if (mediaId == 0) { - //ez elszallhat, mert nincs filetype vagy store - MediaFile mediaFile = manager.createMediaFile(fileName, fileTypeName, targetStoreName); - //ez elszallhat, mert nincs itemtype + // ez elszallhat, mert nincs filetype vagy store + String mediaFileName = fileName; + if (addStoreNameAsFilePrefix) + mediaFileName = sourceStoreName + "_" + fileName; + MediaFile mediaFile = manager.createMediaFile(mediaFileName, fileTypeName, targetStoreName); + // ez elszallhat, mert nincs itemtype Item item = manager.createItem(GENERIC, title, sourceStoreName, null); - //ez elszallhat, mert nincs itemtype + // ez elszallhat, mert nincs itemtype Media media = manager.createMedia(GENERIC, title, sourceStoreName, null); - //TODO megoldani egy tranzakcioban + // TODO megoldani egy tranzakcioban item.add(); media.setItemId(item.getId()); media.setLength(frames); @@ -65,8 +69,8 @@ public class MetadataPersisterStep extends JobStep { media.setArchived(Timestamp.from(Instant.now())); media.add(); - //TODO - //mediaFile.setFileSize(fileSize); + // TODO + // mediaFile.setFileSize(fileSize); mediaFile.setMediaId(media.getId()); mediaFile.setLastModified(modified); mediaFile.add(); @@ -78,8 +82,8 @@ public class MetadataPersisterStep extends JobStep { media.setArchived(Timestamp.from(Instant.now())); manager.modify(media); - //TODO - //mediaFile.setFileSize(fileSize); + // TODO + // mediaFile.setFileSize(fileSize); MediaFile mediaFile = manager.getSystemMediaFile(media); if (mediaFile == null) { mediaFile = manager.createMediaFile(fileName, fileTypeName, targetStoreName); @@ -104,13 +108,14 @@ public class MetadataPersisterStep extends JobStep { private long tryGetMediaId(String title, long mediaId) { Media media = null; - //duplikatum miatt + // duplikatum miatt try { media = getManager().getMedia(title); if (media != null) mediaId = media.getId(); } catch (Exception e) { - logger.error(getMarker(), "Media with title {} is not available. System message is: {}", title, e.getMessage()); + logger.error(getMarker(), "Media with title {} is not available. System message is: {}", title, + e.getMessage()); } return mediaId; } diff --git a/server/user.jobengine.executors/src/user/jobengine/server/steps/PRORESTransferFromStep.java b/server/user.jobengine.executors/src/user/jobengine/server/steps/PRORESTransferFromStep.java new file mode 100644 index 00000000..28688e7b --- /dev/null +++ b/server/user.jobengine.executors/src/user/jobengine/server/steps/PRORESTransferFromStep.java @@ -0,0 +1,20 @@ +package user.jobengine.server.steps; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + +import user.commons.StoreUri; + +public class PRORESTransferFromStep extends TransferStep { + private static final Logger logger = LogManager.getLogger(); + + @Override + @StepEntry + public Object[] execute(StoreUri sourceStoreUri, String sourceFileName, StoreUri targetStoreUri, + String targetFileName) throws Exception { + String proresFileName = "PRORES_" + targetFileName; + logger.info("Saving {}", proresFileName); + super.execute(sourceStoreUri, sourceFileName, targetStoreUri, proresFileName); + return new Object[] { proresFileName }; + } +} diff --git a/server/user.jobengine.executors/src/user/jobengine/server/steps/ProResArchiveCheckerStep.java b/server/user.jobengine.executors/src/user/jobengine/server/steps/ProResArchiveCheckerStep.java index ad0685b7..96d13718 100644 --- a/server/user.jobengine.executors/src/user/jobengine/server/steps/ProResArchiveCheckerStep.java +++ b/server/user.jobengine.executors/src/user/jobengine/server/steps/ProResArchiveCheckerStep.java @@ -3,6 +3,9 @@ package user.jobengine.server.steps; import java.nio.file.Path; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + import com.ibm.nosql.json.api.BasicDBObject; import user.commons.StoreUri; @@ -13,6 +16,7 @@ import user.jobengine.server.steps.shared.FileSearchFilterOptions; public class ProResArchiveCheckerStep extends PathItemsCollectorStep { public static final String DOT_CATCHED = ".catched"; public static final String STATUSFOLDER = ".STATUS"; + public static final Logger logger = LogManager.getLogger(); @Override protected FileSearchFilterOptions createFileFilter(BasicDBObject filter) { @@ -28,8 +32,11 @@ public class ProResArchiveCheckerStep extends PathItemsCollectorStep { @Override @StepEntry public Object[] execute(String sourceStoreName, BasicDBObject filter) throws Exception { + StoreUri sourceStoreUri = getManager().getStoreUri(sourceStoreName, RemoteStoreProtocol.LOCAL); - String sourceStoreFolder = sourceStoreUri.toString(false); + String sourceStoreFolder = sourceStoreUri.toString(true); + + logger.info(getMarker(), "Source path is {}", sourceStoreFolder); return super.execute(sourceStoreFolder, filter); } } diff --git a/server/user.jobengine.executors/src/user/jobengine/server/steps/SafeDeleteRecursiveStep.java b/server/user.jobengine.executors/src/user/jobengine/server/steps/SafeDeleteRecursiveStep.java index 91ca9119..6092a231 100644 --- a/server/user.jobengine.executors/src/user/jobengine/server/steps/SafeDeleteRecursiveStep.java +++ b/server/user.jobengine.executors/src/user/jobengine/server/steps/SafeDeleteRecursiveStep.java @@ -107,6 +107,11 @@ public class SafeDeleteRecursiveStep extends JobStep implements FileVisitor inputFiles) { + int[] progress = new int[] { 0 }; + int[] i = new int[] { 0 }; + + logger.info(getMarker(), "Starting ValidateProResStep"); Map result = inputFiles.entrySet().stream().filter(fileEntry -> { String fileLocation = fileEntry.getKey(); MediaArea mediaArea = inputFiles.get(fileLocation); String videoFormat = mediaArea.getFormat(); - boolean isProRes = mediaArea.getFrameRate() == FRAMERATE && videoFormat.equals(PRORES); - logger.info(getSessionMarker(), "{} is PRORES {}", fileLocation, isProRes); + boolean isProRes = mediaArea.getFrameRate() == FRAMERATE && videoFormat != null + && videoFormat.toUpperCase().equals(PRORES); + logger.info(getMarker(), "{} is PRORES {}", fileLocation, isProRes); + i[0]++; + progress[0] = i[0] * 100 / inputFiles.size(); + int p = progress[0]; + setProgress(p); return isProRes; - }).collect(Collectors.toMap(Map.Entry::getKey, Map.Entry::getValue)); + }).collect(Collectors.toMap(t -> t.getKey(), t -> t.getValue())); + + setProgress(100); return new Object[] { result }; } diff --git a/server/user.jobengine.executors/src/user/jobengine/server/steps/shared/EscortFiles.java b/server/user.jobengine.executors/src/user/jobengine/server/steps/shared/EscortFiles.java index 3068a030..8e890031 100644 --- a/server/user.jobengine.executors/src/user/jobengine/server/steps/shared/EscortFiles.java +++ b/server/user.jobengine.executors/src/user/jobengine/server/steps/shared/EscortFiles.java @@ -81,7 +81,8 @@ public class EscortFiles { } public static void createCatchedFile(Path escortFile) throws IOException { - Path catchedFilePath = Paths.get(escortFile.toString() + DOT_CATCHED); + String catchedFileName = escortFile.toString() + DOT_CATCHED; + Path catchedFilePath = Paths.get(catchedFileName); Files.createFile(catchedFilePath); } @@ -102,7 +103,8 @@ public class EscortFiles { } public static Path createMediaCatchFilePath(Path mediaFile) { - return Paths.get(mediaFile.getParent().toString(), STATUSFOLDER, mediaFile.getFileName() + DOT_CATCHED); + String catchFileName = mediaFile.getFileName().toString() + DOT_CATCHED; + return Paths.get(mediaFile.getParent().toString(), STATUSFOLDER, catchFileName); } public static void createMetadata(String filePath, String fileName, String metadata) throws IOException { @@ -237,7 +239,8 @@ public class EscortFiles { } public static boolean isCatchedFileExists(Path escortFile) { - Path catchedFilePath = Paths.get(escortFile.toString() + DOT_CATCHED); + String catchedFileName = escortFile.toString() + DOT_CATCHED; + Path catchedFilePath = Paths.get(catchedFileName); return Files.exists(catchedFilePath); } diff --git a/server/user.jobengine.osgi.db/migrations/scripts/040_create_AMC_prores_store.sql b/server/user.jobengine.osgi.db/migrations/scripts/040_create_AMC_prores_store.sql index 9078c1d6..5056f584 100644 --- a/server/user.jobengine.osgi.db/migrations/scripts/040_create_AMC_prores_store.sql +++ b/server/user.jobengine.osgi.db/migrations/scripts/040_create_AMC_prores_store.sql @@ -6,12 +6,12 @@ INSERT INTO STORE(NAME,ISSYSTEM,ISLOWRES) VALUES @ INSERT INTO STOREURI(STOREID,PROTOCOL,DELIVERY,URI,ISSTREAM,ISSOURCE,ISTARGET,USERNAME,PASSWORD,ROOTPATH,PORTNUMBER) VALUES -((SELECT ID FROM STORE WHERE NAME='PRORES'),'LOCAL',null,'/mediacube/data','N','Y','Y',null,null,null,null) +((SELECT ID FROM STORE WHERE NAME='PRORES'),'LOCAL',null,'s:/','N','Y','Y',null,null,null,null) @ -- //@UNDO -- SQL to undo the change goes here. -DELETE FROM STOREURI WHERE STOREID=(SELECT ID FROM STORE WHERE NAME='AMC_LOCAL') +DELETE FROM STOREURI WHERE STOREID=(SELECT ID FROM STORE WHERE NAME='PRORES') @ -DELETE FROM STORE WHERE NAME='AMC_LOCAL' +DELETE FROM STORE WHERE NAME='PRORES' @ -- 2.54.0