From 409cd849f5561ac3a49750851668075b6072b6eb Mon Sep 17 00:00:00 2001 From: "vasary.daniel" Date: Tue, 10 Aug 2021 13:41:28 +0000 Subject: [PATCH] git-tfs-id: [http://tfs.userrendszerhaz.hu:8080/tfs/DefaultCollection]$/MediaCube;C32300 --- .../mediacube/executors/tests/SmallTests.java | 92 +++++++++---- .../server/steps/SafeDeleteRecursive.java | 124 ++++++++++++++---- .../resources/i3-label_hu.properties | 3 +- .../user/jobengine/gui/ComponentBinder.java | 1 - 4 files changed, 166 insertions(+), 54 deletions(-) diff --git a/server/hu.user.mediacube.executors.tests/src/hu/user/mediacube/executors/tests/SmallTests.java b/server/hu.user.mediacube.executors.tests/src/hu/user/mediacube/executors/tests/SmallTests.java index da86f88d..3c11fab1 100644 --- a/server/hu.user.mediacube.executors.tests/src/hu/user/mediacube/executors/tests/SmallTests.java +++ b/server/hu.user.mediacube.executors.tests/src/hu/user/mediacube/executors/tests/SmallTests.java @@ -248,8 +248,9 @@ public class SmallTests { Integer nexioPort = 2098; String nexioUserName = "administrator"; String nexioPassword = "system"; - Object[] inputs = { mediaCubeMedia, targetPath, targetNamePattern, successRecipient, killDateDays, localRetrievePath, globalRetrievePath, - useNexioTarget, nexioAgency, nexioPort, nexioUserName, nexioPassword }; + Object[] inputs = { mediaCubeMedia, targetPath, targetNamePattern, successRecipient, killDateDays, + localRetrievePath, globalRetrievePath, useNexioTarget, nexioAgency, nexioPort, nexioUserName, + nexioPassword }; // Object[] inputs = { null, null, null, null, 0, null, null, false, null, 0, null, null }; sut.run(null, null, inputs); } @@ -318,8 +319,8 @@ public class SmallTests { @Test public void test9() throws Exception { - List files = Arrays.asList(Paths.get("/opt/20200101100000.xxx.json"), Paths.get("/opt/xxx.json"), Paths.get("/opt/yyy.json"), - Paths.get("/opt/20200202200000.xxx.json")); + List files = Arrays.asList(Paths.get("/opt/20200101100000.xxx.json"), Paths.get("/opt/xxx.json"), + Paths.get("/opt/yyy.json"), Paths.get("/opt/20200202200000.xxx.json")); files.sort((a, b) -> { return a.compareTo(b); @@ -506,7 +507,8 @@ public class SmallTests { @Test public void test99() throws Exception { System.setProperty("jobengine.ffmpeg.location", "/Programs/ffmpeg/bin/ffmpeg.exe"); - System.setProperty("java.library.path", System.getProperty("java.library.path") + ";/_workspace/USER/MediaCube/server/-dependencies/libs"); + System.setProperty("java.library.path", + System.getProperty("java.library.path") + ";/_workspace/USER/MediaCube/server/-dependencies/libs"); Path input = Paths.get("/_XDCAM/Sub/0410_1006_DEMO_mariagyud_mixS01.MXF"); Path output = Paths.get("/_XDCAM/out"); @@ -559,7 +561,8 @@ public class SmallTests { public void test992() throws Exception { Timestamp nexio = Timestamp.from(Instant.now().minus(2, ChronoUnit.HOURS)); LocalDateTime ldt = LocalDateTime.ofInstant(Instant.now(), ZoneOffset.systemDefault()); - LocalDateTime ldn = LocalDateTime.ofInstant(Instant.now().minus(2, ChronoUnit.HOURS), ZoneOffset.systemDefault()); + LocalDateTime ldn = LocalDateTime.ofInstant(Instant.now().minus(2, ChronoUnit.HOURS), + ZoneOffset.systemDefault()); long modifiedHours = Duration.between(ldt, ldn).toHours(); System.out.println(modifiedHours); System.out.println(DateTimeFormatter.ISO_LOCAL_DATE_TIME.format(ldt)); @@ -623,7 +626,8 @@ public class SmallTests { } } System.out.println(String.format("TSM hiány %d, polcon van %d, omben van %d", fCount, pCount, oCount)); - System.out.println(String.format("Egyéb %d, Reklám %d, Promó %d, Anyag %d, Adásban volt %d", other, ad, promo, material, inrd)); + System.out.println(String.format("Egyéb %d, Reklám %d, Promó %d, Anyag %d, Adásban volt %d", other, ad, promo, + material, inrd)); // Files.write(Paths.get("/opt/AMC/mcmissing.txt"), mcmissing); } @@ -680,7 +684,8 @@ public class SmallTests { errorCount++; } float archivePercent = archivedCount * 100 / omCount; - System.out.println(String.format("Archiválandó %d, archivált %d, nem archivált %d, százalék %f", omCount, archivedCount, errorCount, archivePercent)); + System.out.println(String.format("Archiválandó %d, archivált %d, nem archivált %d, százalék %f", omCount, + archivedCount, errorCount, archivePercent)); // Files.write(Paths.get("/opt/AMC/mcmissing.txt"), mcmissing); } @@ -749,7 +754,8 @@ public class SmallTests { } } - String excludedReport = String.format("Kihagyva %d db, összméret %s", omExcludeCount, readableFileSize(omExcludeSizeSum)); + String excludedReport = String.format("Kihagyva %d db, összméret %s", omExcludeCount, + readableFileSize(omExcludeSizeSum)); System.out.println(excludedReport); Files.write(Paths.get("/opt/mv-excluded-parlament-byid.txt"), out); @@ -783,7 +789,8 @@ public class SmallTests { } } - String excludedReport = String.format("%d db, összméret %s", omExcludeCount, readableFileSize(omExcludeSizeSum)); + String excludedReport = String.format("%d db, összméret %s", omExcludeCount, + readableFileSize(omExcludeSizeSum)); System.out.println(excludedReport); Files.write(Paths.get("/opt/mv-excluded-parlament-name.txt"), out); @@ -859,7 +866,8 @@ public class SmallTests { continue; } } - System.out.println("All (no PTV) " + omcount + " missing " + count + " completed " + ((count * 100) / omcount) + "%"); + System.out.println( + "All (no PTV) " + omcount + " missing " + count + " completed " + ((count * 100) / omcount) + "%"); // String excludedReport = String.format("Kihagyva %d db, összméret %s", omExcludeCount, readableFileSize(omExcludeSizeSum)); // System.out.println(excludedReport); // @@ -910,12 +918,11 @@ public class SmallTests { return id.split(" / ")[0]; else return id; - }) - .forEach(id1 -> { - //System.out.println(id1.toString()); - mcjob.add(id1.toString()); + }).forEach(id1 -> { + //System.out.println(id1.toString()); + mcjob.add(id1.toString()); - }); + }); // List mc = Files.readAllLines(Paths.get("/opt/AMC/mc-all-modified.txt")); Map tsmmap = new HashMap<>(); @@ -967,8 +974,10 @@ public class SmallTests { vchecked++; - if (!df.format(mcdate).substring(0, 10).equals(df.format(tsmdate).substring(0, 10)) && tsmdate.before(mcdate)) { - System.out.println("Old version in TSM " + id + " mc " + df.format(mcdate) + " tsm " + df.format(tsmdate)); + if (!df.format(mcdate).substring(0, 10).equals(df.format(tsmdate).substring(0, 10)) + && tsmdate.before(mcdate)) { + System.out.println( + "Old version in TSM " + id + " mc " + df.format(mcdate) + " tsm " + df.format(tsmdate)); vreport.add(id); vmissings++; } @@ -979,7 +988,8 @@ public class SmallTests { } } - System.out.println(tsm.size() + " " + mc.size() + " " + missings + " vchecked" + vchecked + " vmissing" + vmissings); + System.out.println( + tsm.size() + " " + mc.size() + " " + missings + " vchecked" + vchecked + " vmissing" + vmissings); Files.write(Paths.get("/opt/AMC/mreport.txt"), mreport); Files.write(Paths.get("/opt/AMC/vreport.txt"), vreport); } @@ -989,8 +999,8 @@ public class SmallTests { GroovyClassLoader cl = new GroovyClassLoader(this.getClass().getClassLoader()) { @Override - public Class loadClass(String name, boolean lookupScriptFiles, boolean preferClassOverScript, boolean resolve) - throws ClassNotFoundException, CompilationFailedException { + public Class loadClass(String name, boolean lookupScriptFiles, boolean preferClassOverScript, + boolean resolve) throws ClassNotFoundException, CompilationFailedException { return super.loadClass(name, lookupScriptFiles, preferClassOverScript, resolve); } @@ -1206,8 +1216,10 @@ public class SmallTests { public void test99997() throws Exception { System.setProperty("jna.library.path", "/users/elgekko"); - List files = Arrays.asList("c:\\opt\\AMC\\2021\\ISSUES\\#203\\IFL000000071_43.mxf", "c:\\opt\\AMC\\2021\\ISSUES\\#203\\MX_12290041.mxf", - "c:\\opt\\AMC\\2021\\ISSUES\\#203\\DK_20344100_02_high_16_9.mxf", "c:\\opt\\AMC\\2021\\ISSUES\\#203\\IPR000011027_169.mxf"); + List files = Arrays.asList("c:\\opt\\AMC\\2021\\ISSUES\\#203\\IFL000000071_43.mxf", + "c:\\opt\\AMC\\2021\\ISSUES\\#203\\MX_12290041.mxf", + "c:\\opt\\AMC\\2021\\ISSUES\\#203\\DK_20344100_02_high_16_9.mxf", + "c:\\opt\\AMC\\2021\\ISSUES\\#203\\IPR000011027_169.mxf"); for (String file : files) { try (user.commons.mediaarea.MediaInfo mi = new user.commons.mediaarea.MediaInfo(file)) { //Display aspect ratio @@ -1396,9 +1408,39 @@ public class SmallTests { }); } - @Test public void test999996() throws Exception { - System.out.println("valami.txt".substring(0, "valami.txt".lastIndexOf("."))); + Path path = Paths.get("/opt/MV/file_list_original.txt"); + List lines = FileUtils.readLines(path.toFile()); + Set contents = new LinkedHashSet<>(); + + String lastDir = null; + for (String line : lines) { + line = line.trim(); + if (line.startsWith("Directory of")) { + lastDir = line; + lastDir = lastDir.replace("Directory of", ""); + lastDir = lastDir.replace("X:", ""); + lastDir = lastDir.replace("\\", "/"); + lastDir = lastDir.trim(); + // if (!lastDir.endsWith(".STATUS")) + // System.out.println(">> " + lastDir); + } + + if (lastDir != null && lastDir.endsWith(".STATUS")) + continue; + + if (line.startsWith("2") && line.length() > 39) { + String file = line.substring(39).trim(); + + if (file.equals(".") || file.equals("..")) + continue; + + String len = line.substring(21, 39).trim(); + if (len.length() != 0) + System.out.println("/mnt/POLC/FINISHED_SHOWS" + lastDir + "/" + file + " : " + len); + } + + } } } diff --git a/server/user.jobengine.executors/src/user/jobengine/server/steps/SafeDeleteRecursive.java b/server/user.jobengine.executors/src/user/jobengine/server/steps/SafeDeleteRecursive.java index 224fdcca..42c1bf9b 100644 --- a/server/user.jobengine.executors/src/user/jobengine/server/steps/SafeDeleteRecursive.java +++ b/server/user.jobengine.executors/src/user/jobengine/server/steps/SafeDeleteRecursive.java @@ -14,8 +14,11 @@ import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; import java.util.Date; +import java.util.LinkedHashSet; import java.util.List; +import java.util.Set; +import org.apache.commons.io.FileUtils; import org.apache.commons.io.FilenameUtils; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; @@ -54,6 +57,7 @@ public class SafeDeleteRecursive extends JobStep implements FileVisitor { private List skipPathNames = Arrays.asList("!ARCHIVALAS_ALATT", EscortFiles.STATUSFOLDER, EscortFiles.CONFLICTFOLDER); + private Set includeList; private boolean canReadMediaInfo(Path mediaFilePath) { boolean result = false; @@ -72,24 +76,33 @@ public class SafeDeleteRecursive extends JobStep implements FileVisitor { return false; String itemHouseId = archiveItem.getItemHouseId() == null ? "" : archiveItem.getItemHouseId(); - if (!itemHouseId.equals(item.getHouseId())) + if (!itemHouseId.equals(item.getHouseId())) { + logger.error(getSessionMarker(), "ItemHouseId"); return false; + } String itemTitle = archiveItem.getItemTitle() == null ? "" : archiveItem.getItemTitle(); - if (!itemTitle.equals(item.getTitle())) + if (!itemTitle.equals(item.getTitle())) { + logger.error(getSessionMarker(), "ItemTitle"); return false; - + } String mediaHouseId = archiveItem.getMediaHouseId() == null ? "" : archiveItem.getMediaHouseId(); - if (!mediaHouseId.equals(media.getHouseId())) + if (!mediaHouseId.equals(media.getHouseId())) { + logger.error(getSessionMarker(), "MediaHouseId"); return false; - + } String mediaTitle = archiveItem.getMediaTitle() == null ? "" : archiveItem.getMediaTitle(); - if (!mediaTitle.equals(media.getTitle())) + if (!mediaTitle.equals(media.getTitle())) { + logger.error(getSessionMarker(), "MediaTitle"); return false; + } String mediaDescription = archiveItem.getMediaDescription() == null ? "" : archiveItem.getMediaDescription(); - if (!mediaDescription.equals(media.getDescription())) + String storedMediaDescription = media.getDescription() == null ? "" : media.getDescription(); + if (!mediaDescription.equals(storedMediaDescription)) { + logger.error(getSessionMarker(), "MediaDescription"); return false; + } return true; } @@ -115,19 +128,18 @@ public class SafeDeleteRecursive extends JobStep implements FileVisitor { @StepEntry public Object[] execute(String sourcePath) throws Exception { logger.info(getSessionMarker(), "Starting in {}", sourcePath); - logger.info(getSessionMarker(), "{};{};{};{};{};{};{};{};{};{};{};{};{};", "Napló időbélyeg", "Törlés sikeres", - "Neve", "Elérése", "Mérete", "Létrehozva", "Módosítva", "Catched", "Killdate", "Archivált (MC)", - "Archiválva (MC)", "Archivált (TSM)", "Archiválva (TSM)", "Méret (TSM)", "Metaadat egyezés"); + logger.info(getSessionMarker(), "{};{};{};{};{};{};{};{};{};{};{};{};{};{};{};{};{};{};", "Napló időbélyeg", + "Neve", "Elérése", "Mérete", "Létrehozva", "Módosítva", "Archiválva (MC)", "Archiválva (TSM)", + "Méret (TSM)", ".catched", ".killdate", ".nomd", "Archivált (MC)", "Archivált (TSM)", + "Metaadat egyezés", "Méret egyezés", "Törölhető", "Kezelendő"); try { - if (getJobRuntime().forkPrepare()) { - Files.walkFileTree(Paths.get(sourcePath), this); - } + String location = "/opt/test-mediacube/file_list_original.txt"; + includeList = loadIncludeList(location); + Files.walkFileTree(Paths.get(sourcePath), this); } catch (Exception e) { logger.error(getSessionMarker(), "Az '{}' mappa elérése sikertelen. A rendszer hibaüzenete: {}", sourcePath, e.getMessage()); - } finally { - getJobRuntime().forkWaitComplete(); } return null; } @@ -187,22 +199,30 @@ public class SafeDeleteRecursive extends JobStep implements FileVisitor { Path dirName = dir.getFileName(); if (skipPathNames.contains(dirName.toString())) { - logger.info(getSessionMarker(), "PreVisit skip {}", dir); + //logger.info(getSessionMarker(), "PreVisit skip {}", dir); return FileVisitResult.SKIP_SUBTREE; - } else - logger.info(getSessionMarker(), "PreVisit {}", dir); + } //else + //logger.info(getSessionMarker(), "PreVisit {}", dir); return FileVisitResult.CONTINUE; } private boolean processPathItem(Path mediaPath) throws Exception { + if (getJobRuntime().isWaitingCancel()) { + cancel(); + return false; + } + BasicFileAttributes attr = Files.readAttributes(mediaPath, BasicFileAttributes.class); File mediaFSFile = mediaPath.toFile(); + String fileName = mediaPath.getFileName().toString(); if (mediaFSFile.isDirectory()) - return false; + return true; + + if (fileName.startsWith(".") || fileName.endsWith(".nomd")) + return true; - String fileName = mediaPath.getFileName().toString(); Path nomdFile = Paths.get(mediaPath.toString() + ".nomd"); ArchiveItem archiveItem = createArchiveItem(mediaPath); List killDateFiles = getKillDateFiles(mediaPath); @@ -219,9 +239,11 @@ public class SafeDeleteRecursive extends JobStep implements FileVisitor { boolean mcArchived = mediaFile != null; boolean metadataEquals = false; boolean tsmArchived = false; + boolean sizeEquals = false; Date mcArchivedDate = null; long tsmSize = 0; Date tsmBackupDate = null; + boolean includeContains = includeList.contains(mediaPath.toString()); if (mcArchived) { //metadata @@ -237,15 +259,17 @@ public class SafeDeleteRecursive extends JobStep implements FileVisitor { if (tsmArchived) { tsmSize = tsmFile.getSize(); tsmBackupDate = getTSMBackupDate(tsmFileName); + sizeEquals = tsmSize == size; } } - boolean canDelete = !noMDExists && catchedExists && killdateExists && mcArchived && tsmArchived; + boolean canDelete = !noMDExists && catchedExists && killdateExists && mcArchived && tsmArchived + && metadataEquals; Date now = new Date(System.currentTimeMillis()); - logger.info(getSessionMarker(), "{};{};{};{};{};{};{};{};{};{};{};{};{};", df.format(now), YN(canDelete), - fileName, mediaPath.getParent(), size, df.format(createDate), df.format(lastModifiedDate), - YN(catchedExists), YN(killdateExists), YN(mcArchived), df.format(mcArchivedDate), YN(tsmArchived), - df.format(tsmBackupDate), tsmSize, YN(metadataEquals)); + logger.info(getSessionMarker(), "{};{};{};{};{};{};{};{};{};{};{};{};{};{};{};{};{};{};", D(now), fileName, + mediaPath.getParent(), size, D(createDate), D(lastModifiedDate), D(mcArchivedDate), D(tsmBackupDate), + tsmSize, YN(catchedExists), YN(killdateExists), YN(noMDExists), YN(mcArchived), YN(tsmArchived), + YN(metadataEquals), YN(sizeEquals), YN(canDelete), YN(includeContains)); return true; } @@ -281,11 +305,16 @@ public class SafeDeleteRecursive extends JobStep implements FileVisitor { return value ? 'Y' : 'N'; } + private String D(Date value) { + return value == null ? "" : df.format(value); + } + @Override public FileVisitResult visitFile(Path filePath, BasicFileAttributes paramBasicFileAttributes) throws IOException { - // logger.info(getSessionMarker(), "Will checked {}", filePath); + //logger.info(getSessionMarker(), "Will checked {}", filePath); try { - processPathItem(filePath); + if (!processPathItem(filePath)) + return FileVisitResult.TERMINATE; } catch (Exception e) { logger.catching(e); } @@ -343,4 +372,45 @@ public class SafeDeleteRecursive extends JobStep implements FileVisitor { return result; } + private Set loadIncludeList(String location) throws IOException { + Set result = new LinkedHashSet<>(); + + Path path = Paths.get(location); + List lines = FileUtils.readLines(path.toFile()); + + String lastDir = null; + for (String line : lines) { + line = line.trim(); + if (line.startsWith("Directory of")) { + lastDir = line; + lastDir = lastDir.replace("Directory of", ""); + lastDir = lastDir.replace("X:", ""); + lastDir = lastDir.replace("\\", "/"); + lastDir = lastDir.trim(); + // if (!lastDir.endsWith(".STATUS")) + // System.out.println(">> " + lastDir); + } + + if (lastDir != null && lastDir.endsWith(".STATUS")) + continue; + + if (line.startsWith("2") && line.length() > 39) { + String file = line.substring(39).trim(); + + if (file.equals(".") || file.equals("..")) + continue; + + String len = line.substring(21, 39).trim(); + if (len.length() != 0) { + String fullpath = "/mnt/POLC/FINISHED_SHOWS" + lastDir + "/" + file; + //System.out.println("/mnt/POLC/FINISHED_SHOWS" + lastDir + "/" + file + " : " + len); + result.add(fullpath); + } + } + + } + + return result; + } + } diff --git a/server/user.mediacube.gui/resources/i3-label_hu.properties b/server/user.mediacube.gui/resources/i3-label_hu.properties index 0cea4c1d..3f20f895 100644 --- a/server/user.mediacube.gui/resources/i3-label_hu.properties +++ b/server/user.mediacube.gui/resources/i3-label_hu.properties @@ -1,4 +1,5 @@ -version=2.7.5 +version=2.7.6 +#2.7.6 TSM Metadata provider, javitott GroovyClassLoader #2.7.5 ArchiveItem disableProxy, MediaFile disableProxy footer=2016-2021 © Copyright User Rendszerház Kft. diff --git a/server/user.mediacube.gui/src/user/jobengine/gui/ComponentBinder.java b/server/user.mediacube.gui/src/user/jobengine/gui/ComponentBinder.java index 9a970611..ccf8423d 100644 --- a/server/user.mediacube.gui/src/user/jobengine/gui/ComponentBinder.java +++ b/server/user.mediacube.gui/src/user/jobengine/gui/ComponentBinder.java @@ -29,7 +29,6 @@ public class ComponentBinder { ServiceReference[] serviceReferences = bundleContext.getServiceReferences((String) null, null); for (ServiceReference r : serviceReferences) { Object s = bundleContext.getService(r); - System.err.println("Found " + s); } } catch (InvalidSyntaxException e) { e.printStackTrace(); -- 2.54.0