From 87f1eae25468dd28bc09e605654ce680b3d0c13c Mon Sep 17 00:00:00 2001 From: =?utf8?q?V=C3=A1s=C3=A1ry=20D=C3=A1niel?= Date: Wed, 24 Oct 2018 09:33:13 +0000 Subject: [PATCH] =?utf8?q?Duplik=C3=A1tumok=20kezel=C3=A9s=C3=A9nek=20m?= =?utf8?q?=C3=B3dos=C3=ADt=C3=A1sa?= MIME-Version: 1.0 Content-Type: text/plain; charset=utf8 Content-Transfer-Encoding: 8bit git-tfs-id: [http://tfs.userrendszerhaz.hu:8080/tfs/DefaultCollection]$/MediaCube;C31332 --- .../jobtemplates/archive-limited.xml | 2 +- .../jobtemplates/archive-ondemand.xml | 2 +- .../server/steps/ArchiveListBuilderStep.java | 11 +- .../steps/ArchiveMaterialSubmitStep.java | 4 +- .../server/steps/ItemManagerExtensions.java | 36 +++--- .../jobengine/server/steps/TSMBackupStep.java | 4 +- .../user/jobengine/server/IT/Support.java | 7 ++ .../server/IT/SupportRemoveDuplicates.java | 116 ++++++++++++++++++ 8 files changed, 159 insertions(+), 23 deletions(-) create mode 100644 server/user.jobengine.osgi.server/test/user/jobengine/server/IT/SupportRemoveDuplicates.java diff --git a/server/user.jobengine.executors/jobtemplates/archive-limited.xml b/server/user.jobengine.executors/jobtemplates/archive-limited.xml index c9a671a2..3b8654aa 100644 --- a/server/user.jobengine.executors/jobtemplates/archive-limited.xml +++ b/server/user.jobengine.executors/jobtemplates/archive-limited.xml @@ -10,7 +10,7 @@ - + diff --git a/server/user.jobengine.executors/jobtemplates/archive-ondemand.xml b/server/user.jobengine.executors/jobtemplates/archive-ondemand.xml index c9a671a2..3b8654aa 100644 --- a/server/user.jobengine.executors/jobtemplates/archive-ondemand.xml +++ b/server/user.jobengine.executors/jobtemplates/archive-ondemand.xml @@ -10,7 +10,7 @@ - + diff --git a/server/user.jobengine.executors/src/user/jobengine/server/steps/ArchiveListBuilderStep.java b/server/user.jobengine.executors/src/user/jobengine/server/steps/ArchiveListBuilderStep.java index 56367a85..7cbfffa6 100644 --- a/server/user.jobengine.executors/src/user/jobengine/server/steps/ArchiveListBuilderStep.java +++ b/server/user.jobengine.executors/src/user/jobengine/server/steps/ArchiveListBuilderStep.java @@ -6,7 +6,7 @@ import java.nio.file.DirectoryStream; import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.Paths; -import java.util.ArrayList; +import java.util.LinkedList; import java.util.List; import org.apache.commons.lang.StringUtils; @@ -88,7 +88,7 @@ public class ArchiveListBuilderStep extends JobStep { @StepEntry public Object[] execute(String sourcePath, int limit, IJobEngine jobEngine, IJobRuntime jobRuntime) { marker = jobRuntime.getMarker(); - List archiveList = new ArrayList(); + List archiveList = new LinkedList(); DirectoryStream directoryStream = null; int counter = 0; try { @@ -178,7 +178,12 @@ public class ArchiveListBuilderStep extends JobStep { logger.warn("{} has no Media Title specified in metadata.", mediaFile.getName()); return false; } - archiveList.add(archiveItem); + + //A tenyleges archivalast vesszuk elore + if (mediaFile.length() == 0) + archiveList.add(archiveItem); + else + archiveList.add(0, archiveItem); createCatchedFile(catchedFilePath); return true; } diff --git a/server/user.jobengine.executors/src/user/jobengine/server/steps/ArchiveMaterialSubmitStep.java b/server/user.jobengine.executors/src/user/jobengine/server/steps/ArchiveMaterialSubmitStep.java index 95e846a2..b1232e53 100644 --- a/server/user.jobengine.executors/src/user/jobengine/server/steps/ArchiveMaterialSubmitStep.java +++ b/server/user.jobengine.executors/src/user/jobengine/server/steps/ArchiveMaterialSubmitStep.java @@ -1,7 +1,7 @@ package user.jobengine.server.steps; import java.io.File; -import java.util.ArrayList; +import java.util.List; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; @@ -23,7 +23,7 @@ public class ArchiveMaterialSubmitStep extends JobStep { private Marker marker; @StepEntry - public Object[] execute(String globalSourcePath, ArrayList archiveList, String transcoderTargetPath, String webPath, int killDateDays, + public Object[] execute(String globalSourcePath, List archiveList, String transcoderTargetPath, String webPath, int killDateDays, IJobEngine jobEngine, IJobRuntime jobRuntime) throws Exception { marker = jobRuntime.getMarker(); if (archiveList == null || archiveList.size() == 0) diff --git a/server/user.jobengine.executors/src/user/jobengine/server/steps/ItemManagerExtensions.java b/server/user.jobengine.executors/src/user/jobengine/server/steps/ItemManagerExtensions.java index 2e6852d2..b77be5de 100644 --- a/server/user.jobengine.executors/src/user/jobengine/server/steps/ItemManagerExtensions.java +++ b/server/user.jobengine.executors/src/user/jobengine/server/steps/ItemManagerExtensions.java @@ -4,6 +4,7 @@ import com.ibm.nosql.json.api.BasicDBObject; import user.jobengine.db.IItemManager; import user.jobengine.db.IResultSetConsumer; +import user.jobengine.server.steps.MetadataTypeDetector.MetadataType; public class ItemManagerExtensions { @@ -28,21 +29,26 @@ public class ItemManagerExtensions { public static long getExistingRundownMedia(IItemManager manager, String houseid) { final long[] result = { 0 }; - StringBuilder query = new StringBuilder(); - query.append("select mediaid, mediafilehouseid, filename"); - query.append(" "); - query.append(String.format("from vw_rundown_items where mediafilehouseid like '%s%%'", houseid)); - query.append(" "); - query.append("order by filename, mediaid"); - IResultSetConsumer consumer = rs -> { - String fileName = rs.getString("filename"); - if (houseid.equals(fileName)) { - result[0] = rs.getLong("mediaid"); - return false; - } else - return true; - }; - manager.executeQuery(query.toString(), consumer, null); + final String idToCheck = houseid.substring(0, houseid.lastIndexOf("-")); + MetadataType metadataType = MetadataTypeDetector.GuessMetadataType(idToCheck); + if (metadataType == MetadataType.OctopusPlaceholder) { + StringBuilder query = new StringBuilder(); + query.append("select mediaid, mediafilehouseid, filename"); + query.append(" "); + query.append(String.format("from vw_rundown_items where mediafilehouseid like '%s%%'", idToCheck)); + query.append(" "); + query.append("order by filename, mediaid"); + IResultSetConsumer consumer = rs -> { + String fileName = rs.getString("filename"); + if (idToCheck.equals(fileName)) { + result[0] = rs.getLong("mediaid"); + return false; + } else + return true; + }; + manager.executeQuery(query.toString(), consumer, null); + } + return result[0]; } diff --git a/server/user.jobengine.executors/src/user/jobengine/server/steps/TSMBackupStep.java b/server/user.jobengine.executors/src/user/jobengine/server/steps/TSMBackupStep.java index f15743c3..a891452b 100644 --- a/server/user.jobengine.executors/src/user/jobengine/server/steps/TSMBackupStep.java +++ b/server/user.jobengine.executors/src/user/jobengine/server/steps/TSMBackupStep.java @@ -79,7 +79,9 @@ public class TSMBackupStep extends JobStep { } }); RemoteFile remoteFile = sourceUri.transferFrom(targetUri, sourceFileName, sourceFileName); - } else + } + + if (existingMediaId > 0) logger.info(marker, "Az '{}' TSM mentése nem szükséges, mert már megtalálható az archívumban.", sourceFileName); //Fel kell szabadítani, hogy a kovetkezo archivalaskor is nekifusson diff --git a/server/user.jobengine.osgi.server/test/user/jobengine/server/IT/Support.java b/server/user.jobengine.osgi.server/test/user/jobengine/server/IT/Support.java index e6419c40..b71d720e 100644 --- a/server/user.jobengine.osgi.server/test/user/jobengine/server/IT/Support.java +++ b/server/user.jobengine.osgi.server/test/user/jobengine/server/IT/Support.java @@ -645,4 +645,11 @@ public class Support { mediaInfo.process(); System.out.println("Bitrate: " + mediaInfo.getBitRate()); } + + @Test + public void test4() throws Exception { + String houseid = "133519696_1-001-132654584"; + String idToCheck = houseid.substring(0, houseid.lastIndexOf("-")); + System.out.println(idToCheck); + } } diff --git a/server/user.jobengine.osgi.server/test/user/jobengine/server/IT/SupportRemoveDuplicates.java b/server/user.jobengine.osgi.server/test/user/jobengine/server/IT/SupportRemoveDuplicates.java new file mode 100644 index 00000000..bb5f8c81 --- /dev/null +++ b/server/user.jobengine.osgi.server/test/user/jobengine/server/IT/SupportRemoveDuplicates.java @@ -0,0 +1,116 @@ +package user.jobengine.server.IT; + +import java.io.File; +import java.nio.file.Paths; +import java.util.Date; + +import org.junit.AfterClass; +import org.junit.BeforeClass; +import org.junit.Test; + +import user.commons.logging.LogUtils; +import user.jobengine.db.IItemManager; +import user.jobengine.db.IResultSetConsumer; +import user.jobengine.db.IStatementDecorator; +import user.jobengine.db.ItemManager; +import user.jobengine.db.MediaFile; + +public class SupportRemoveDuplicates { + public class BD { + Date fBD; + Date nBD; + } + + protected static IItemManager manager = null; + + private static final String LOWRES_ROOT = "//10.10.1.27/mclowres/www/video"; + + private static final long[] duplicateLength = { 0 }; + + @BeforeClass + static public void setUpConnection() { + // System.setProperty("jobengine.octopus.rundowns.name", "test_rundowns"); + // System.setProperty("jobengine.octopus.stories.name", "test_stories"); + // System.setProperty("jobengine.octopus.storyfolders.name", "test_storyfolders"); + System.setProperty("jobengine.octopus.rundowns.name", "rundowns180620"); + System.setProperty("jobengine.octopus.stories.name", "stories180620"); + System.setProperty("jobengine.octopus.storyfolders.name", "storyfolders180620"); + + System.setProperty("jobengine.nosql.db.url", "jdbc:db2://10.10.1.27:50000/mc:retrieveMessagesFromServerOnGetMessage=true;"); + System.setProperty("jobengine.nosql.db.user", "db2admin"); + System.setProperty("jobengine.nosql.db.password", "password"); + System.setProperty(ItemManager.DBURL, "jdbc:db2://10.10.1.27:50000/mc:retrieveMessagesFromServerOnGetMessage=true;"); + System.setProperty(ItemManager.DBUSERNAME, "db2admin"); + System.setProperty(ItemManager.DBPASSWORD, "password"); + System.setProperty(LogUtils.LOGLEVEL, "INFO"); + // System.setProperty(ItemManager.DBURL, + + manager = new ItemManager(); + manager.connect(); + } + + @AfterClass + static public void tearDownConnection() { + manager.disconnect(); + } + + private void processLowresDuplicateGroup(String fileName) { + //System.out.println("*** Processing: " + fileName); + String query = "select mediafileid, mediafilehouseid, relativepath from vw_items_rd_lh where filename = ?"; + IStatementDecorator decorator = st -> { + st.setString(1, fileName); + }; + + MediaFile[] masterMediaFile = { null }; + + IResultSetConsumer consumer = rs -> { + long mediaFileId = rs.getLong("mediafileid"); + MediaFile mediaFile = (MediaFile) manager.get(MediaFile.class, mediaFileId); + if (masterMediaFile[0] == null) { + masterMediaFile[0] = mediaFile; + return true; + } + + String path = rs.getString("relativepath"); + File file = Paths.get(LOWRES_ROOT, path).toFile(); + duplicateLength[0] += file.length(); + + if (file.delete()) { + System.out.println(String.format("%d %s %s", mediaFile.getMediaId(), mediaFile.getRelativePath(), masterMediaFile[0].getRelativePath())); + mediaFile.setRelativePath(masterMediaFile[0].getRelativePath()); + manager.modify(mediaFile); + } + return true; + }; + manager.executeQuery(query, consumer, decorator); + } + + @Test + public void processLowresDuplicates() { + long count[] = { 0, 0 }; + manager.executeQuery("select count(filecount) from vw_items_rd_dup", rs -> { + count[0] = rs.getLong(1); + return false; + }, null); + + int limit[] = { 1, 0 }; + int progress[] = { 0, 0 }; + manager.executeQuery("select filename from vw_items_rd_dup order by filecount desc", rs -> { + String fileName = rs.getString("filename"); + processLowresDuplicateGroup(fileName); + count[1]++; + progress[1] = (int) ((double) count[1] * 100 / count[0]); + if (progress[1] > progress[0]) { + progress[0] = progress[1]; + } + System.out.println("Completed " + progress[0] + " " + count[1] + "/" + count[0] + " available " + duplicateLength[0]); + limit[1]++; + + if (limit[0] == limit[1]) + return false; + else + return true; + }, null); + } + +} -- 2.54.0