From d9d1e64fef7bbab71958d2e0f62714817676e859 Mon Sep 17 00:00:00 2001 From: "vasary.daniel" Date: Thu, 31 Mar 2022 11:10:25 +0000 Subject: [PATCH] git-tfs-id: [http://tfs.userrendszerhaz.hu:8080/tfs/DefaultCollection]$/MediaCube;C33128 --- .../executors/tests/PracticeTests.java | 31 +++++ .../server/steps/ArchiveRecursive.java | 124 ++++++++++++++++-- .../steps/MetadataSyncRecursiveStep.java | 68 ++++------ .../server/steps/shared/EscortFiles.java | 2 +- .../jobengine/server/steps/ArchiveItem.java | 57 ++++---- .../mediacube/metadata/planair/PlanAirAD.java | 9 +- .../metadata/planair/PlanAirMaterial.java | 1 + .../metadata/planair/PlanAirPromo.java | 15 ++- 8 files changed, 214 insertions(+), 93 deletions(-) create mode 100644 server/hu.user.mediacube.executors.tests/src/hu/user/mediacube/executors/tests/PracticeTests.java diff --git a/server/hu.user.mediacube.executors.tests/src/hu/user/mediacube/executors/tests/PracticeTests.java b/server/hu.user.mediacube.executors.tests/src/hu/user/mediacube/executors/tests/PracticeTests.java new file mode 100644 index 00000000..6d0daa2d --- /dev/null +++ b/server/hu.user.mediacube.executors.tests/src/hu/user/mediacube/executors/tests/PracticeTests.java @@ -0,0 +1,31 @@ +package hu.user.mediacube.executors.tests; + +import java.util.Arrays; + +import org.junit.Test; + +public class PracticeTests { + + public static int[] _01_Two_Sum(int[] nums, int target) { + for (int j = 0; j < nums.length; j++) { + for (int i = j + 1; i < nums.length; i++) { + if (nums[i] + nums[j] == target) + return new int[] { j, i }; + // System.out.println("compare " + j + " " + i); + } + } + return null; + } + + @Test + public void test() throws Exception { + + int[] nums = { 2, 7, 11, 15 }; + int target = 9; + int[] result = _01_Two_Sum(nums, target); + + System.out.println(Arrays.toString(result)); + + } + +} diff --git a/server/user.jobengine.executors/src/user/jobengine/server/steps/ArchiveRecursive.java b/server/user.jobengine.executors/src/user/jobengine/server/steps/ArchiveRecursive.java index 1f320e48..f270fc77 100644 --- a/server/user.jobengine.executors/src/user/jobengine/server/steps/ArchiveRecursive.java +++ b/server/user.jobengine.executors/src/user/jobengine/server/steps/ArchiveRecursive.java @@ -9,21 +9,21 @@ import java.nio.file.Path; import java.nio.file.Paths; import java.nio.file.attribute.BasicFileAttributes; import java.util.Arrays; +import java.util.LinkedHashSet; import java.util.List; import java.util.Map; +import java.util.Set; +import org.apache.commons.io.FileUtils; import org.apache.commons.io.FilenameUtils; import org.apache.commons.lang.StringUtils; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; -import org.apache.logging.log4j.message.Message; -import org.apache.logging.log4j.message.ParameterizedMessage; import com.ibm.nosql.json.api.BasicDBObject; import user.commons.JobStatus; import user.commons.ListUtils; -import user.commons.log4j2.marker.MediaCubeMarker; import user.commons.mediatool.MediaInfo; import user.jobengine.server.IJobEngine; import user.jobengine.server.IJobRuntime; @@ -55,6 +55,7 @@ public class ArchiveRecursive extends JobStep implements FileVisitor { private int submitted; private int killDateDays; private boolean disableProxy; + private Set includeList; private boolean canReadMediaInfo(Path mediaFilePath) { boolean result = false; @@ -141,6 +142,9 @@ public class ArchiveRecursive extends JobStep implements FileVisitor { this.disableProxy = disableProxy; logger.info(getSessionMarker(), "Starting in {}", sourcePath); try { + String location = "/opt/test-mediacube/file_list_original.txt"; + includeList = loadIncludeList(location); + if (getJobRuntime().forkPrepare()) { Files.walkFileTree(Paths.get(sourcePath), this); } @@ -197,6 +201,25 @@ public class ArchiveRecursive extends JobStep implements FileVisitor { result.setMediaDescription(json.getString(MEDIA_DESCRIPTION)); result.setMediaType(json.getString(MEDIA_TYPE)); } + + if (result == null) { + logger.info(getSessionMarker(), "Metadata not available for {}", mediaHouseId); + + if (mediaHouseId.charAt(mediaHouseId.length() - 2) == '-') { + mediaHouseId = mediaHouseId.substring(0, mediaHouseId.length() - 2); + logger.info(getSessionMarker(), "Checking metadata for {}", mediaHouseId); + ArchiveItem metadata = getPlanAirMetadata(mediaHouseId); + // elenorizni, hogy ez tenyleg az e + + if (metadata == null) + logger.info(getSessionMarker(), "Metadata not available for {}", mediaHouseId); + else + logger.info(getSessionMarker(), "Metadata is available for {}: {}", mediaHouseId, + metadata.getMediaTitle()); + } + + } + return result; } @@ -215,6 +238,21 @@ public class ArchiveRecursive extends JobStep implements FileVisitor { return FileVisitResult.CONTINUE; } + private boolean removeFile(Path filePath) { + boolean result = false; + try { + File file = filePath.toFile(); + if (file.exists()) { + logger.info(getSessionMarker(), "Törlés {}", filePath); + result = file.delete(); + } + } catch (Exception e) { + logger.error(getSessionMarker(), "A {} fájl nem törölhető. A rendszer hibaüzenete: {}", filePath, + e.getMessage()); + } + return result; + } + @Override public FileVisitResult preVisitDirectory(Path dir, BasicFileAttributes paramBasicFileAttributes) throws IOException { @@ -230,6 +268,13 @@ public class ArchiveRecursive extends JobStep implements FileVisitor { } private boolean processPathItem(Path mediaPath) throws Exception { + + boolean includeContains = includeList.contains(mediaPath.toString()); + if (!includeContains) { + logger.info(getSessionMarker(), "Not on whitelist {}, skipping", mediaPath); + return false; + } + if (limit != 0 && submitted == limit) { logger.info(getSessionMarker(), "Limit reached {}, canceling", limit); return false; @@ -243,19 +288,28 @@ public class ArchiveRecursive extends JobStep implements FileVisitor { File mediaFile = mediaPath.toFile(); if (mediaFile.isDirectory()) { - // logger.info(getSessionMarker(), "Skipping directory {}", mediaPath); - return false; - } - - if (EscortFiles.isMediaCatched(mediaPath)) { - // logger.info(getSessionMarker(), "Skipping already catched {}", mediaPath); return false; } Path nomdFile = Paths.get(mediaPath.toString() + ".nomd"); if (nomdFile.toFile().exists()) { - // logger.info(getSessionMarker(), "Skipping nomd file exists {}", nomdFile); + // toroljuk es ujbol letrehozzuk ha kell + if (!removeFile(nomdFile)) { + logger.info(getSessionMarker(), "Can't delete nomd file for {}, skipping", nomdFile); + return false; + } + if (EscortFiles.isMediaCatched(mediaPath)) { + Path catchedFile = EscortFiles.createMediaCatchFilePath(mediaPath); + if (!removeFile(catchedFile)) { + logger.info(getSessionMarker(), "Can't delete catched file for {}, skipping", nomdFile); + return false; + } + } + } + + if (EscortFiles.isMediaCatched(mediaPath)) { + // logger.info(getSessionMarker(), "Skipping already catched {}", mediaPath); return false; } @@ -272,9 +326,9 @@ public class ArchiveRecursive extends JobStep implements FileVisitor { ArchiveItem archiveItem = createArchiveItem(mediaPath); if (archiveItem == null) { - Message msg = new ParameterizedMessage("Nincs metaadat!"); - logger.info(new MediaCubeMarker("vasary@elgekko.net,muszak@mediavivantis.hu", - "Értesítés problémás " + mediaPath.getFileName().toString() + " archiválásról"), msg); +// Message msg = new ParameterizedMessage("Nincs metaadat!"); +// logger.info(new MediaCubeMarker("vasary@elgekko.net,muszak@mediavivantis.hu", +// "Értesítés problémás " + mediaPath.getFileName().toString() + " archiválásról"), msg); Files.createFile(nomdFile); return false; } @@ -298,6 +352,7 @@ public class ArchiveRecursive extends JobStep implements FileVisitor { logger.error(getSessionMarker(), "A(z) '{}' állomány archiválási kísérlete sikertelen. A rendszer üzenete: {}", mediaPath, e.getMessage()); + return false; } return true; @@ -319,4 +374,47 @@ public class ArchiveRecursive extends JobStep implements FileVisitor { logger.info("Error archive {}", filePath); return FileVisitResult.CONTINUE; } + + private Set loadIncludeList(String location) throws IOException { + Set result = new LinkedHashSet<>(); + + Path path = Paths.get(location); + List lines = FileUtils.readLines(path.toFile()); + + String lastDir = null; + for (String line : lines) { + line = line.trim(); + if (line.startsWith("Directory of")) { + lastDir = line; + lastDir = lastDir.replace("Directory of", ""); + lastDir = lastDir.replace("X:", ""); + lastDir = lastDir.replace("\\", "/"); + lastDir = lastDir.trim(); + // if (!lastDir.endsWith(".STATUS")) + // System.out.println(">> " + lastDir); + } + + if (lastDir != null && lastDir.endsWith(".STATUS")) + continue; + + if (line.startsWith("2") && line.length() > 39) { + String file = line.substring(39).trim(); + + if (file.equals(".") || file.equals("..")) + continue; + + String len = line.substring(21, 39).trim(); + if (len.length() != 0) { + String fullpath = "/mnt/POLC/FINISHED_SHOWS" + lastDir + "/" + file; + // System.out.println("/mnt/POLC/FINISHED_SHOWS" + lastDir + "/" + file + " : " + // + len); + result.add(fullpath); + } + } + + } + + return result; + } + } diff --git a/server/user.jobengine.executors/src/user/jobengine/server/steps/MetadataSyncRecursiveStep.java b/server/user.jobengine.executors/src/user/jobengine/server/steps/MetadataSyncRecursiveStep.java index c133bcf5..43875901 100644 --- a/server/user.jobengine.executors/src/user/jobengine/server/steps/MetadataSyncRecursiveStep.java +++ b/server/user.jobengine.executors/src/user/jobengine/server/steps/MetadataSyncRecursiveStep.java @@ -12,7 +12,6 @@ import java.util.Arrays; import java.util.List; import org.apache.commons.io.FilenameUtils; -import org.apache.ibatis.jdbc.SQL; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; @@ -38,13 +37,6 @@ import user.mediacube.metadata.interfaces.PlanAirMetadataListOptions; public class MetadataSyncRecursiveStep extends JobStep implements FileVisitor { private static final Logger logger = LogManager.getLogger(); - private static final String ITEM_TITLE = "itemTitle"; - private static final String ITEM_HOUSEID = "itemHouseId"; - private static final String MEDIA_HOUSEID = "mediaHouseId"; - private static final String MEDIA_TITLE = "mediaTitle"; - private static final String MEDIA_DESCRIPTION = "mediaDescription"; - private static final String MEDIA_TYPE = "mediaType"; - private List skipPathNames = Arrays.asList("!ARCHIVALAS_ALATT", EscortFiles.STATUSFOLDER, EscortFiles.CONFLICTFOLDER); @@ -151,13 +143,8 @@ public class MetadataSyncRecursiveStep extends JobStep implements FileVisitor