From 21f898f56af207423db9b669e21851799c373237 Mon Sep 17 00:00:00 2001 From: =?utf8?q?V=C3=A1s=C3=A1ry=20D=C3=A1niel?= Date: Fri, 19 Oct 2018 07:52:18 +0000 Subject: [PATCH] =?utf8?q?Duplik=C3=A1tumok=20megsz=C3=BCntet=C3=A9se=20ja?= =?utf8?q?v=C3=ADt=C3=A1s?= MIME-Version: 1.0 Content-Type: text/plain; charset=utf8 Content-Transfer-Encoding: 8bit git-tfs-id: [http://tfs.userrendszerhaz.hu:8080/tfs/DefaultCollection]$/MediaCube;C31327 --- .../CopyForArchiveNEXIOMaterialsStep.java | 23 ++++++++++++++--- .../jobengine/server/steps/TSMBackupStep.java | 25 +++++++++++++++---- .../server/steps/TranscodeSELENIOStep.java | 4 +-- .../user/jobengine/server/IT/Support.java | 19 +++++++++----- 4 files changed, 54 insertions(+), 17 deletions(-) diff --git a/server/user.jobengine.executors/src/user/jobengine/server/steps/CopyForArchiveNEXIOMaterialsStep.java b/server/user.jobengine.executors/src/user/jobengine/server/steps/CopyForArchiveNEXIOMaterialsStep.java index b69535f3..a3e73b6a 100644 --- a/server/user.jobengine.executors/src/user/jobengine/server/steps/CopyForArchiveNEXIOMaterialsStep.java +++ b/server/user.jobengine.executors/src/user/jobengine/server/steps/CopyForArchiveNEXIOMaterialsStep.java @@ -3,6 +3,7 @@ package user.jobengine.server.steps; import java.io.OutputStream; import java.net.InetAddress; import java.net.URI; +import java.util.ArrayList; import java.util.Calendar; import java.util.Date; import java.util.List; @@ -58,6 +59,7 @@ public class CopyForArchiveNEXIOMaterialsStep extends JobStep { private int nexioKillDateDays; private String nexioAgency; private Marker systemMarker; + private List transferredFileNames = null; private int check(int value, String name) { if (value == 0) { @@ -92,12 +94,25 @@ public class CopyForArchiveNEXIOMaterialsStep extends JobStep { String fileName = String.format("%s-%s", origFileName, rundownArchive.getItemHouseId()); String videoFileName = fileName + MXFEXT; + if (transferredFileNames == null) + transferredFileNames = new ArrayList<>(); + //A mar letezo mozikat nem archivaljuk le ujra, csak a metaadatot - long existingMediaId = ItemManagerExtensions.getExistingRundownMedia(manager, origFileName); - if (existingMediaId == 0) - transferFile(origFileName + MXFEXT, videoFileName); - else + long existingMediaId = 0; + if (transferredFileNames.contains(origFileName)) { + logger.info(systemMarker, "A '{}' file archiválásra felkészítése egy másik story kapcsán már megtörtént, ezért csak metaadat archiválás szükséges.", + origFileName); transferChunk(videoFileName); + } else { + existingMediaId = ItemManagerExtensions.getExistingRundownMedia(manager, origFileName); + if (existingMediaId == 0) { + transferredFileNames.add(origFileName); + transferFile(origFileName + MXFEXT, videoFileName); + } else { + logger.info(systemMarker, "A '{}' file archiválása már megtörtént, ezért csak metaadat archiválás szükséges.", origFileName); + transferChunk(videoFileName); + } + } BasicDBObject metadata = createMetadata(rundownArchive, storyArchive, fileArchive, existingMediaId); transferMetadata(videoFileName, metadata); diff --git a/server/user.jobengine.executors/src/user/jobengine/server/steps/TSMBackupStep.java b/server/user.jobengine.executors/src/user/jobengine/server/steps/TSMBackupStep.java index 11349f6f..f15743c3 100644 --- a/server/user.jobengine.executors/src/user/jobengine/server/steps/TSMBackupStep.java +++ b/server/user.jobengine.executors/src/user/jobengine/server/steps/TSMBackupStep.java @@ -53,6 +53,12 @@ public class TSMBackupStep extends JobStep { //A dupla ellenorzes a napon beluli ismetlesek miatt kell long existingMediaId = archiveItem.getExistingMediaId(); + if (sourceMediaFile.length() == 0 && existingMediaId == 0) { + existingMediaId = ItemManagerExtensions.getExistingRundownMedia(manager, sourceFileName.replace(MXFEXT, "")); + if (existingMediaId == 0) + existingMediaId = -1; + } + if (existingMediaId == 0) existingMediaId = ItemManagerExtensions.getExistingRundownMedia(manager, sourceFileName.replace(MXFEXT, "")); @@ -73,14 +79,23 @@ public class TSMBackupStep extends JobStep { } }); RemoteFile remoteFile = sourceUri.transferFrom(targetUri, sourceFileName, sourceFileName); - } else { + } else logger.info(marker, "Az '{}' TSM mentése nem szükséges, mert már megtalálható az archívumban.", sourceFileName); + + //Fel kell szabadítani, hogy a kovetkezo archivalaskor is nekifusson + if (existingMediaId == -1) { + logger.info(marker, "Az '{}' mentése jelenleg nem lehetséges, mert a szükséges metaadat még nem található meg az archívumban.", sourceFileName); + if (!archiveItem.removeCatchedFile()) + logger.error(marker, + "Az '{}' állomány .catched jelző állománya nem törölhető. Az újabb archiválási kísérlethez annak kézi eltávolítása szükséges!", + sourceMediaFile.getName()); + } else { + saveMetadata(mediaCubeMedia, sourceFileName, existingMediaId); + logger.info(marker, "Az '{}' archiválása sikeres.", sourceFileName); + if (killDateDays > 0) + EscortFiles.createUNCKillDate(sourceMediaFile.getParent(), sourceFileName, killDateDays, marker); } - saveMetadata(mediaCubeMedia, sourceFileName, existingMediaId); - logger.info(marker, "Az '{}' archiválása sikeres.", sourceFileName); - if (killDateDays > 0) - EscortFiles.createUNCKillDate(sourceMediaFile.getParent(), sourceFileName, killDateDays, marker); } catch (Exception e) { logger.catching(e); Message m = new ParameterizedMessage("Az '{}' állomány archiválása sikertelen. A rendszer hibaüzenete: {}", details, e.getMessage()); diff --git a/server/user.jobengine.executors/src/user/jobengine/server/steps/TranscodeSELENIOStep.java b/server/user.jobengine.executors/src/user/jobengine/server/steps/TranscodeSELENIOStep.java index 7d25c60c..d4bf472d 100644 --- a/server/user.jobengine.executors/src/user/jobengine/server/steps/TranscodeSELENIOStep.java +++ b/server/user.jobengine.executors/src/user/jobengine/server/steps/TranscodeSELENIOStep.java @@ -101,12 +101,12 @@ public class TranscodeSELENIOStep extends JobStep { File sourceMediaFile = new File(archiveItem.getMediaFile()); sourceFileName = sourceMediaFile.getName(); - //Nincs mit transzkódolni, a TSMBackupStep csinal masolatot a mediafileokrol + //Nincs mit transzkodolni, a TSMBackupStep csinal masolatot a mediafileokrol //A dupla ellenorzes a napon beluli ismetlesek miatt kell long existingMediaId = archiveItem.getExistingMediaId(); if (existingMediaId == 0) existingMediaId = ItemManagerExtensions.getExistingRundownMedia(manager, sourceFileName.replace(MXFEXT, "")); - if (existingMediaId != 0) + if (existingMediaId != 0 || sourceMediaFile.length() == 0) return null; String details = String.format("%s (%d bytes)", sourceFileName, sourceMediaFile.length()); diff --git a/server/user.jobengine.osgi.server/test/user/jobengine/server/IT/Support.java b/server/user.jobengine.osgi.server/test/user/jobengine/server/IT/Support.java index 62074d69..25f040d1 100644 --- a/server/user.jobengine.osgi.server/test/user/jobengine/server/IT/Support.java +++ b/server/user.jobengine.osgi.server/test/user/jobengine/server/IT/Support.java @@ -373,19 +373,20 @@ public class Support { IResultSetConsumer consumer = rs -> { long mediaFileId = rs.getLong("mediafileid"); MediaFile mediaFile = (MediaFile) manager.get(MediaFile.class, mediaFileId); - String mediaFileHouseId = rs.getString("mediafilehouseid"); if (masterMediaFile[0] == null) { masterMediaFile[0] = mediaFile; - //System.out.println("Skipping: " + mediaFileHouseId); return true; } String path = rs.getString("relativepath"); - // System.out.println("Inspecting: " + path + " filename " + fileName + " mediaFileHouseId " + mediaFileHouseId); - File file = Paths.get(LOWRES_ROOT, path).toFile(); duplicateLength[0] += file.length(); - //System.out.println("Inspecting: " + mediaFileHouseId + " now available " + duplicateLength[0]); + + if (file.delete()) { + System.out.println(String.format("%d %s %s", mediaFile.getMediaId(), mediaFile.getRelativePath(), masterMediaFile[0].getRelativePath())); + mediaFile.setRelativePath(masterMediaFile[0].getRelativePath()); + manager.modify(mediaFile); + } return true; }; manager.executeQuery(query, consumer, decorator); @@ -399,6 +400,7 @@ public class Support { return false; }, null); + int limit[] = { 1, 0 }; int progress[] = { 0, 0 }; manager.executeQuery("select filename from vw_items_rd_dup order by filecount desc", rs -> { String fileName = rs.getString("filename"); @@ -409,7 +411,12 @@ public class Support { progress[0] = progress[1]; } System.out.println("Completed " + progress[0] + " " + count[1] + "/" + count[0] + " available " + duplicateLength[0]); - return true; + limit[1]++; + + if (limit[0] == limit[1]) + return false; + else + return true; }, null); } -- 2.54.0