From e54399dced7888a0a369b016bc42a181e1fbc254 Mon Sep 17 00:00:00 2001 From: elgekko Date: Thu, 2 Jun 2022 10:22:13 +0200 Subject: [PATCH] Rekurziv archivalas segedfolyamatok --- .../server/steps/ArchiveRecursive.java | 59 ++-- .../steps/CheckArchiveRecursiveStep.java | 267 ++++++++++++++++++ .../jobengine/server/steps/MXFCutterStep.java | 51 ++-- .../server/steps/SafeDeleteRecursiveStep.java | 70 +++-- 4 files changed, 366 insertions(+), 81 deletions(-) create mode 100644 server/user.jobengine.executors/src/user/jobengine/server/steps/CheckArchiveRecursiveStep.java diff --git a/server/user.jobengine.executors/src/user/jobengine/server/steps/ArchiveRecursive.java b/server/user.jobengine.executors/src/user/jobengine/server/steps/ArchiveRecursive.java index 2c90137a..cee331ea 100644 --- a/server/user.jobengine.executors/src/user/jobengine/server/steps/ArchiveRecursive.java +++ b/server/user.jobengine.executors/src/user/jobengine/server/steps/ArchiveRecursive.java @@ -169,7 +169,7 @@ public class ArchiveRecursive extends JobStep implements FileVisitor { } } if (json != null) { - logger.info(getSessionMarker(), json.toString()); + // logger.info(getSessionMarker(), json.toString()); result = ArchiveItem.fromJSONObject(json); } @@ -238,25 +238,34 @@ public class ArchiveRecursive extends JobStep implements FileVisitor { } private boolean processPathItem(Path mediaPath) throws Exception { - - boolean includeContains = includeList.contains(mediaPath.toString()); - if (!includeContains) { - logger.info(getSessionMarker(), "Not on whitelist {}, skipping", mediaPath); + if (limit != 0 && submitted == limit) { + logger.info(getSessionMarker(), "Limit reached {}, canceling", limit); return false; } String fileName = mediaPath.getFileName().toString(); if (fileName.startsWith(".") || fileName.endsWith(".nomd")) - return false; - - if (limit != 0 && submitted == limit) { - logger.info(getSessionMarker(), "Limit reached {}, canceling", limit); - return false; - } + return true; File mediaFile = mediaPath.toFile(); if (mediaFile.isDirectory()) { - return false; + return true; + } + + if (EscortFiles.isMediaCatched(mediaPath)) { + // logger.info(getSessionMarker(), "Skipping already catched {}", mediaPath); + return true; + } + + if (handleArchiveConflict(mediaPath)) { + logger.info(getSessionMarker(), "Skipping archive db already contains {}", mediaPath); + return true; + } + + boolean includeContains = includeList.contains(mediaPath.toString()); + if (!includeContains) { + logger.info(getSessionMarker(), "Not on whitelist {}, skipping", mediaPath); + return true; } // @@ -275,30 +284,22 @@ public class ArchiveRecursive extends JobStep implements FileVisitor { // } // } - if (EscortFiles.isMediaCatched(mediaPath)) { - // logger.info(getSessionMarker(), "Skipping already catched {}", mediaPath); - return false; - } - - if (handleArchiveConflict(mediaPath)) { - logger.info(getSessionMarker(), "Skipping archive db already contains {}", mediaPath); - return false; - } - if (!canReadMediaInfo(mediaPath)) { logger.info(getSessionMarker(), "Skipping cant read mediainfo {}", mediaPath); - return false; + return true; } ArchiveItem archiveItem = createArchiveItem(mediaPath); - Path nomdFile = Paths.get(mediaPath.toString() + ".nomd"); - if (archiveItem == null && !Files.exists(nomdFile)) { + if (archiveItem == null) { // Message msg = new ParameterizedMessage("Nincs metaadat!"); // logger.info(new MediaCubeMarker("vasary@elgekko.net,muszak@mediavivantis.hu", // "Értesítés problémás " + mediaPath.getFileName().toString() + " archiválásról"), msg); - Files.createFile(nomdFile); - return false; + + Path nomdFile = Paths.get(mediaPath.toString() + ".nomd"); + if (!Files.exists(nomdFile)) + Files.createFile(nomdFile); + return true; } logger.info(getSessionMarker(), "Archiving {}", mediaPath); @@ -333,8 +334,10 @@ public class ArchiveRecursive extends JobStep implements FileVisitor { public FileVisitResult visitFile(Path filePath, BasicFileAttributes paramBasicFileAttributes) throws IOException { // logger.info(getSessionMarker(), "Will checked {}", filePath); try { - if (!processPathItem(filePath)) + if (!processPathItem(filePath)) { + logger.info(getSessionMarker(), "processPathItem returned false, terminating"); return FileVisitResult.TERMINATE; + } } catch (Exception e) { logger.catching(e); } diff --git a/server/user.jobengine.executors/src/user/jobengine/server/steps/CheckArchiveRecursiveStep.java b/server/user.jobengine.executors/src/user/jobengine/server/steps/CheckArchiveRecursiveStep.java new file mode 100644 index 00000000..5e1328ad --- /dev/null +++ b/server/user.jobengine.executors/src/user/jobengine/server/steps/CheckArchiveRecursiveStep.java @@ -0,0 +1,267 @@ +package user.jobengine.server.steps; + +import java.io.File; +import java.io.IOException; +import java.nio.file.DirectoryStream; +import java.nio.file.FileVisitResult; +import java.nio.file.FileVisitor; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.nio.file.attribute.BasicFileAttributes; +import java.text.SimpleDateFormat; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.Date; +import java.util.LinkedHashSet; +import java.util.List; +import java.util.Set; + +import org.apache.commons.io.FileUtils; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.apache.logging.log4j.Marker; +import org.apache.logging.log4j.MarkerManager; + +import com.ibm.nosql.json.api.BasicDBObject; + +import user.commons.IEntityBase; +import user.commons.RemoteFile; +import user.commons.StoreUri; +import user.commons.remotestore.RemoteStoreProtocol; +import user.jobengine.db.Media; +import user.jobengine.db.MediaFile; +import user.jobengine.db.MediaFileDAO; +import user.jobengine.db.Store; +import user.jobengine.server.steps.shared.EscortFiles; +import user.mediacube.metadata.interfaces.IMetadata; +import user.mediacube.metadata.interfaces.IMetadataListOptions; +import user.mediacube.metadata.interfaces.IMetadataProvider; +import user.mediacube.metadata.interfaces.IMetadataProviderFactory; +import user.mediacube.metadata.interfaces.MetadataProviderType; + +public class CheckArchiveRecursiveStep extends JobStep implements FileVisitor { + private static final Logger logger = LogManager.getLogger(); + private Marker csvMarker = MarkerManager.getMarker("CHECK-ARCHIVE-RECURSIVE-CSV"); + + private static final String KILLDATEEXT = ".killdate"; + private SimpleDateFormat df = new SimpleDateFormat("yyyy.MM.dd HH:mm:ss"); + + private List skipPathNames = Arrays.asList("!ARCHIVALAS_ALATT", EscortFiles.STATUSFOLDER, EscortFiles.CONFLICTFOLDER); + private Set includeList; + + @StepEntry + public Object[] execute(String sourcePath) throws Exception { + logger.info(getSessionMarker(), "Starting in {}", sourcePath); + logger.info(csvMarker, "{};{};{};{};{};{};{};{};{};{};{};{};{};{};{};", "Napló időbélyeg", "Neve", "Elérése", "Mérete", "Létrehozva", "Módosítva", + "Archiválva (MC)", "Archiválva (TSM)", "Méret (TSM)", ".catched", ".killdate", ".nomd", "Archivált (MC)", "Archivált (TSM)", "Méret egyezés"); + + try { + + String location = "/opt/test-mediacube/archive-recursive-files.txt"; + includeList = loadWhiteList(location); + + Files.walkFileTree(Paths.get(sourcePath), this); + } catch (Exception e) { + logger.error(getSessionMarker(), "Az '{}' mappa elérése sikertelen. A rendszer hibaüzenete: {}", sourcePath, e.getMessage()); + } + return null; + } + + @Override + public FileVisitResult postVisitDirectory(Path paramT, IOException paramIOException) throws IOException { + return FileVisitResult.CONTINUE; + } + + @Override + public FileVisitResult preVisitDirectory(Path dir, BasicFileAttributes paramBasicFileAttributes) throws IOException { + Path dirName = dir.getFileName(); + + if (skipPathNames.contains(dirName.toString())) { + // logger.info(getSessionMarker(), "PreVisit skip {}", dir); + return FileVisitResult.SKIP_SUBTREE; + } // else + // logger.info(getSessionMarker(), "PreVisit {}", dir); + + return FileVisitResult.CONTINUE; + } + + private boolean processPathItem(Path mediaPath) throws Exception { + if (getJobRuntime().isWaitingCancel()) { + cancel(); + return false; + } + + BasicFileAttributes attr = Files.readAttributes(mediaPath, BasicFileAttributes.class); + File mediaFSFile = mediaPath.toFile(); + String fileName = mediaPath.getFileName().toString(); + + if (mediaFSFile.isDirectory()) + return true; + + if (fileName.startsWith(".") || fileName.endsWith(".nomd")) + return true; + + boolean includeContains = includeList.contains(mediaPath.toString()); + if (!includeContains) + return true; + + Path nomdFile = Paths.get(mediaPath.toString() + ".nomd"); + List killDateFiles = getKillDateFiles(mediaPath); + MediaFile mediaFile = getMediaFile(fileName); + + long size = mediaFSFile.length(); + Date lastModifiedDate = new Date(attr.lastModifiedTime().toMillis()); + // Date lastAccesDate = new Date(attr.lastAccessTime().toMillis()); + Date createDate = new Date(attr.creationTime().toMillis()); + boolean catchedExists = EscortFiles.isMediaCatched(mediaPath); + boolean noMDExists = nomdFile.toFile().exists(); + boolean killdateExists = killDateFiles.size() > 0; + boolean mcArchived = mediaFile != null; + boolean tsmArchived = false; + boolean sizeEquals = false; + Date mcArchivedDate = null; + long tsmSize = 0; + Date tsmBackupDate = null; + + if (mcArchived) { + // metadata + Media media = getManager().getMedia(mediaFile.getMediaId()); + mcArchivedDate = media.getCreated(); + + // tsm + String tsmFileName = mediaFile.getRelativePath(); + RemoteFile tsmFile = getTSMFile(tsmFileName); + tsmArchived = tsmFile != null; + if (tsmArchived) { + tsmSize = tsmFile.getSize(); + tsmBackupDate = getTSMBackupDate(tsmFileName); + sizeEquals = tsmSize == size; + } + } + + Date now = new Date(System.currentTimeMillis()); + logger.info(csvMarker, "{};{};{};{};{};{};{};{};{};{};{};{};{};{};{};", D(now), fileName, mediaPath.getParent(), size, D(createDate), + D(lastModifiedDate), D(mcArchivedDate), D(tsmBackupDate), tsmSize, YN(catchedExists), YN(killdateExists), YN(noMDExists), YN(mcArchived), + YN(tsmArchived), YN(sizeEquals)); + + logger.info(getSessionMarker(), "{} {}{}{}{}{}{}", mediaPath, YN(catchedExists), YN(killdateExists), YN(noMDExists), YN(mcArchived), YN(tsmArchived), + YN(sizeEquals)); + + return true; + } + + private Date getTSMBackupDate(String tsmFileName) throws Exception { + Date result = null; + IMetadataProviderFactory factory = getService(IMetadataProviderFactory.class); + if (factory == null) + return null; + IMetadataProvider provider = factory.getProvider(MetadataProviderType.TSM); + if (provider == null) + return null; + + IMetadataListOptions opt = provider.createOptions(new BasicDBObject("fileName", tsmFileName)); + + List tsmContents = provider.list(opt); + if (tsmContents != null && tsmContents.size() > 0) { + + for (IMetadata md : tsmContents) { + Date backupDate = md.asJSON().getDate("backupDate"); + if (result == null) + result = backupDate; + else { + if (backupDate.after(result)) + result = backupDate; + } + } + } + return result; + } + + private char YN(boolean value) { + return value ? 'Y' : 'N'; + } + + private String D(Date value) { + return value == null ? "" : df.format(value); + } + + @Override + public FileVisitResult visitFile(Path filePath, BasicFileAttributes paramBasicFileAttributes) throws IOException { + // logger.info(getSessionMarker(), "Will checked {}", filePath); + try { + if (!processPathItem(filePath)) + return FileVisitResult.TERMINATE; + } catch (Exception e) { + logger.catching(e); + } + return FileVisitResult.CONTINUE; + } + + @Override + public FileVisitResult visitFileFailed(Path filePath, IOException paramIOException) throws IOException { + logger.info("Error archive {}", filePath); + return FileVisitResult.CONTINUE; + } + + private List getKillDateFiles(Path filePath) { + String killDateFilePattern = String.format("%s.*%s", filePath.getFileName().toString(), KILLDATEEXT); + List result = new ArrayList<>(); + Path statusPath = null; + try { + statusPath = Paths.get(filePath.getParent().toString(), EscortFiles.STATUSFOLDER); + } catch (Exception e) { + logger.catching(e); + return null; + } + File statusPathFile = statusPath.toFile(); + if (statusPathFile.exists() && statusPathFile.isDirectory()) { + try (DirectoryStream stream = Files.newDirectoryStream(statusPath, killDateFilePattern)) { + stream.forEach(p -> result.add(p)); + } catch (Exception e) { + logger.catching(e); + } + } + Collections.sort(result); + return result; + } + + MediaFile getMediaFile(String fileName) { + MediaFile result = null; + MediaFileDAO mfDAO = (MediaFileDAO) getManager().getBaseDAO(MediaFile.class); + List mfList = mfDAO.getByHouseId(fileName); + if (mfList != null && mfList.size() == 1) + result = (MediaFile) mfList.get(0); + return result; + } + + RemoteFile getTSMFile(String mcFileName) { + RemoteFile result = null; + Store tsmStore = getManager().getSystemStore(false); + StoreUri tsmStoreUri = tsmStore.getSourceStoreUri(RemoteStoreProtocol.TSM); + try { + result = tsmStoreUri.getRemoteFile(mcFileName); + } catch (Exception e) { + logger.error(e); + } finally { + tsmStoreUri.cleanUp(); + } + return result; + } + + private Set loadWhiteList(String location) throws IOException { + Set result = new LinkedHashSet<>(); + Path path = Paths.get(location); + List lines = FileUtils.readLines(path.toFile()); + + for (String line : lines) { + String[] tokens = line.trim().split("\t"); + if (tokens.length == 0) + continue; + result.add(tokens[0]); + } + return result; + } + +} diff --git a/server/user.jobengine.executors/src/user/jobengine/server/steps/MXFCutterStep.java b/server/user.jobengine.executors/src/user/jobengine/server/steps/MXFCutterStep.java index 8c094d80..f1209d9d 100644 --- a/server/user.jobengine.executors/src/user/jobengine/server/steps/MXFCutterStep.java +++ b/server/user.jobengine.executors/src/user/jobengine/server/steps/MXFCutterStep.java @@ -5,11 +5,8 @@ import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.apache.logging.log4j.Marker; -import user.commons.RemoteFile; import user.commons.StoreUri; import user.commons.configuration.SystemConfiguration; -import user.commons.remotestore.IProgressEventListener; -import user.commons.remotestore.ProgressEvent; import user.commons.remotestore.RemoteStoreProtocol; //import user.jobengine.db.Media; import user.jobengine.db.ArchivedMedia; @@ -17,7 +14,6 @@ import user.jobengine.db.IItemManager; import user.jobengine.db.Store; import user.jobengine.server.IJobEngine; import user.jobengine.server.IJobRuntime; -import user.jobengine.server.steps.shared.EscortFiles; public class MXFCutterStep extends JobStep { private static final String TARGETNAMEPATTERN = "-ARCH-%s"; @@ -56,24 +52,35 @@ public class MXFCutterStep extends JobStep { this.nexioPassword = nexioPassword; marker = jobRuntime.getSessionMarker(); - if (useNexioTarget && archivedMedia.getTcIn() != null && archivedMedia.getTcOut() != null) { - setAndCheck(archivedMedia, houseId, targetPath, useNexioTarget, jobEngine); - - final IJobRuntime runtime = jobRuntime; - sourceFileName = houseId + TARGETNAMEPATTERN; - tempSourceUri.addProgressListener(new IProgressEventListener() { - @Override - public void progressChanged(ProgressEvent evt) { - runtime.incrementProgress(evt.getProgress()); - } - }); - - RemoteFile result = tempSourceUri.transferFrom(tempTargetUri, sourceFileName, sourceFileName); - - EscortFiles.setNEXIOKillDate(killDateDays, houseId, nexioAgency, tempTargetUri); - - logger.info("A {} videó kivágva {}s - {}s", sourceFileName, archivedMedia.getTcIn(), archivedMedia.getTcOut()); - } +// if (!useNexioTarget) { +// String targetFileName = String.format(targetNamePattern, sourceFileName); +// MediaArea mediaArea = new MediaArea(Paths.get(targetPath)); +// try { +// mediaArea.process(); +// logger.info(getSessionMarker(), mediaArea.getInform().toPrettyString("")); +// } catch (Exception e) { +// logger.error(getSessionMarker(), "Can't analyze input {}, skipping. System message is: {}", e.getMessage()); +// } +// } + +// if (useNexioTarget && archivedMedia.getTcIn() != null && archivedMedia.getTcOut() != null) { +// setAndCheck(archivedMedia, houseId, targetPath, useNexioTarget, jobEngine); +// +// final IJobRuntime runtime = jobRuntime; +// sourceFileName = houseId + TARGETNAMEPATTERN; +// tempSourceUri.addProgressListener(new IProgressEventListener() { +// @Override +// public void progressChanged(ProgressEvent evt) { +// runtime.incrementProgress(evt.getProgress()); +// } +// }); +// +// RemoteFile result = tempSourceUri.transferFrom(tempTargetUri, sourceFileName, sourceFileName); +// +// EscortFiles.setNEXIOKillDate(killDateDays, houseId, nexioAgency, tempTargetUri); +// +// logger.info("A {} videó kivágva {}s - {}s", sourceFileName, archivedMedia.getTcIn(), archivedMedia.getTcOut()); +// } return null; } diff --git a/server/user.jobengine.executors/src/user/jobengine/server/steps/SafeDeleteRecursiveStep.java b/server/user.jobengine.executors/src/user/jobengine/server/steps/SafeDeleteRecursiveStep.java index 6092a231..596a05a6 100644 --- a/server/user.jobengine.executors/src/user/jobengine/server/steps/SafeDeleteRecursiveStep.java +++ b/server/user.jobengine.executors/src/user/jobengine/server/steps/SafeDeleteRecursiveStep.java @@ -62,8 +62,7 @@ public class SafeDeleteRecursiveStep extends JobStep implements FileVisitor skipPathNames = Arrays.asList("!ARCHIVALAS_ALATT", EscortFiles.STATUSFOLDER, - EscortFiles.CONFLICTFOLDER); + private List skipPathNames = Arrays.asList("!ARCHIVALAS_ALATT", EscortFiles.STATUSFOLDER, EscortFiles.CONFLICTFOLDER); private Set includeList; private boolean checkArchiveItem(Path mediaPath, ArchiveItem archiveItem, Item item, Media media) { @@ -128,18 +127,20 @@ public class SafeDeleteRecursiveStep extends JobStep implements FileVisitor loadWhiteList(String location) throws IOException { + Set result = new LinkedHashSet<>(); + Path path = Paths.get(location); + List lines = FileUtils.readLines(path.toFile()); + + for (String line : lines) { + String[] tokens = line.trim().split("\t"); + if (tokens.length == 0) + continue; + result.add(tokens[0]); + } + return result; + } + } -- 2.54.0