From 5175c83a33d3252f95395a6f39478633f2c7c6c7 Mon Sep 17 00:00:00 2001 From: "vasary.daniel" Date: Mon, 9 Aug 2021 09:17:04 +0000 Subject: [PATCH] git-tfs-id: [http://tfs.userrendszerhaz.hu:8080/tfs/DefaultCollection]$/MediaCube;C32288 --- .../server/steps/ArchiveRecursive.java | 179 +++++++++++------- 1 file changed, 106 insertions(+), 73 deletions(-) diff --git a/server/user.jobengine.executors/src/user/jobengine/server/steps/ArchiveRecursive.java b/server/user.jobengine.executors/src/user/jobengine/server/steps/ArchiveRecursive.java index 6a580e0f..bdef82c0 100644 --- a/server/user.jobengine.executors/src/user/jobengine/server/steps/ArchiveRecursive.java +++ b/server/user.jobengine.executors/src/user/jobengine/server/steps/ArchiveRecursive.java @@ -10,18 +10,23 @@ import java.nio.file.Paths; import java.nio.file.attribute.BasicFileAttributes; import java.text.SimpleDateFormat; import java.util.Arrays; +import java.util.HashSet; import java.util.List; import java.util.Map; +import java.util.Set; import org.apache.commons.io.FilenameUtils; import org.apache.commons.lang.StringUtils; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; +import org.apache.logging.log4j.message.Message; +import org.apache.logging.log4j.message.ParameterizedMessage; import com.ibm.nosql.json.api.BasicDBObject; import user.commons.JobStatus; import user.commons.ListUtils; +import user.commons.MediaCubeMarker; import user.commons.mediatool.MediaInfo; import user.jobengine.server.IJobEngine; import user.jobengine.server.IJobRuntime; @@ -47,10 +52,12 @@ public class ArchiveRecursive extends JobStep implements FileVisitor { private static final String KILL_DATE_DAYS = "killDateDays"; private SimpleDateFormat df = new SimpleDateFormat("yyyy.MM.dd"); - private List skipPathNames = Arrays.asList("!ARCHIVALAS_ALATT", "VASAROLT", EscortFiles.STATUSFOLDER, EscortFiles.CONFLICTFOLDER); + private List skipPathNames = Arrays.asList("!ARCHIVALAS_ALATT", EscortFiles.STATUSFOLDER, + EscortFiles.CONFLICTFOLDER); private int limit; private int submitted; private int killDateDays; + private boolean disableProxy; private boolean canReadMediaInfo(Path mediaFilePath) { boolean result = false; @@ -64,21 +71,32 @@ public class ArchiveRecursive extends JobStep implements FileVisitor { return result; } - private void checkArchiveItem(ArchiveItem archiveItem, Path mediaPath) throws Exception { + private void checkArchiveItem(ArchiveItem archiveItem) throws Exception { if (archiveItem == null) - throw new Exception(mediaPath + " has no metadata specified."); + throw new Exception("No metadata specified."); if (StringUtils.isBlank(archiveItem.getItemHouseId())) - throw new Exception(mediaPath + " has no Item HouseID specified in metadata."); + throw new Exception("No Item HouseID specified in metadata."); if (StringUtils.isBlank(archiveItem.getItemTitle())) - throw new Exception(mediaPath + " has no Item Title specified in metadata."); + throw new Exception("No Item Title specified in metadata."); if (StringUtils.isBlank(archiveItem.getMediaHouseId())) - throw new Exception(mediaPath + " has no Media HouseID specified in metadata."); + throw new Exception("No Media HouseID specified in metadata."); if (StringUtils.isBlank(archiveItem.getMediaTitle())) - throw new Exception(mediaPath + " has no Media Title specified in metadata."); + throw new Exception("No Media Title specified in metadata."); + } + + private String archiveItemJSON(ArchiveItem result) { + BasicDBObject obj = new BasicDBObject(); + obj.put("itemHouseId", result.getItemHouseId()); + obj.put("itemTitle", result.getItemTitle()); + obj.put("mediaHouseId", result.getMediaHouseId()); + obj.put("mediaTitle", result.getMediaTitle()); + obj.put("mediaDescription", result.getMediaDescription()); + obj.put("mediaType", result.getMediaType()); + return obj.toPrettyString(""); } private ArchiveItem createArchiveItem(Path filePath) throws Exception { @@ -88,45 +106,55 @@ public class ArchiveRecursive extends JobStep implements FileVisitor { String mediaHouseId = FilenameUtils.removeExtension(fileName); try { result = getPlanAirMetadata(mediaHouseId); - } catch (Exception e) { - logger.error("PlanAir metadata error", e); - //nem latja a drivert pl. + if (result != null) { + result.setMediaFile(filePath.toString()); + // 210617 proxy keszites tiltasa + result.setDisableProxy(disableProxy); + } + + logger.info(getSessionMarker(), "PlanAir query done for {}", filePath); + } catch (Exception e) { + logger.error(getSessionMarker(), "PlanAir metadata error", e); + // nem latja a drivert pl. throw e; } - if (result == null) { - result = new ArchiveItem(); - BasicFileAttributes attr = Files.readAttributes(filePath, BasicFileAttributes.class); - result.setItemHouseId(df.format(attr.lastModifiedTime().toMillis())); - result.setItemTitle(filePath.getParent().toString()); - result.setMediaHouseId(mediaHouseId); - result.setMediaTitle(fileName); - //TODO kesobb kivenni - //result.setMediaDescription("/OMARCHIVE"); - result.setMediaType("Generic"); - } +//210616 Ha nincs metaadat, nem archivalunk +// if (result == null) { +// result = new ArchiveItem(); +// BasicFileAttributes attr = Files.readAttributes(filePath, BasicFileAttributes.class); +// result.setItemHouseId(df.format(attr.lastModifiedTime().toMillis())); +// result.setItemTitle(filePath.getParent().toString()); +// result.setMediaHouseId(mediaHouseId); +// result.setMediaTitle(fileName); +// // TODO kesobb kivenni +// // result.setMediaDescription("/OMARCHIVE"); +// result.setMediaType("Generic"); +// result.setMediaFile(filePath.toString()); +// } - result.setMediaFile(filePath.toString()); return result; } @StepEntry - public Object[] execute(String sourcePath, int killDateDays, int limit) throws Exception { + public Object[] execute(String sourcePath, int killDateDays, int limit, boolean disableProxy) throws Exception { this.killDateDays = killDateDays; this.limit = limit; + this.disableProxy = disableProxy; + logger.info(getSessionMarker(), "Starting in {}", sourcePath); try { if (getJobRuntime().forkPrepare()) { Files.walkFileTree(Paths.get(sourcePath), this); } } catch (Exception e) { - logger.error(getSessionMarker(), "Az '{}' mappa elérése sikertelen. A rendszer hibaüzenete: {}", sourcePath, e.getMessage()); + logger.error(getSessionMarker(), "Az '{}' mappa elérése sikertelen. A rendszer hibaüzenete: {}", sourcePath, + e.getMessage()); } finally { if (submitted > 0) getJobRuntime().forkWaitComplete(); else getJobRuntime().cancelForkPrepare(); - } return null; } @@ -140,7 +168,12 @@ public class ArchiveRecursive extends JobStep implements FileVisitor { List data = null; IMetadataProviderFactory factory = getService(IMetadataProviderFactory.class); + if (factory == null) + logger.info(getSessionMarker(), "IMetadataProviderFactory is null"); + IMetadataProvider planairProvider = factory.getProvider(MetadataProviderType.PLANAIR); + if (planairProvider == null) + logger.info(getSessionMarker(), "IMetadataProvider is null"); ArchiveItem result = null; data = planairProvider.list(opt); @@ -176,36 +209,7 @@ public class ArchiveRecursive extends JobStep implements FileVisitor { if (getManager().isMediaFileExists(sourceFileName)) { EscortFiles.createMediaCatch(mediaPath); result = true; - - // try { - // Path parent = sourcePath.getParent(); - // Path conflictPath = Paths.get(parent.toString(), CONFLICT); - // File folder = conflictPath.toFile(); - // if (!folder.exists() || !folder.isDirectory()) { - // Set perms = PosixFilePermissions.fromString("rwxrwxrwx"); - // FileAttribute> attr = PosixFilePermissions.asFileAttribute(perms); - // try { - // Files.createDirectories(conflictPath, attr); - // } catch (Exception e) { - // try { - // Files.createDirectory(conflictPath); - // } catch (Exception e1) { - // logger.catching(e); - // throw e; - // } - // } - // } - // - // Path conflictFile = Paths.get(conflictPath.toString(), sourceFileName); - // if (conflictFile.toFile().exists()) - // conflictFile = Paths.get(conflictPath.toString(), sourceFileName + (new Date()).getTime()); - // Files.move(sourcePath, conflictFile); - // result = true; - // } catch (Exception e1) { - // logger.error(getSessionMarker(), "Hiba az '{}' állomány mappába mozgatásakor. A rendszer üzenete: {}", CONFLICT, e1.getMessage()); - // } } - return result; } @@ -215,46 +219,71 @@ public class ArchiveRecursive extends JobStep implements FileVisitor { } @Override - public FileVisitResult preVisitDirectory(Path dir, BasicFileAttributes paramBasicFileAttributes) throws IOException { + public FileVisitResult preVisitDirectory(Path dir, BasicFileAttributes paramBasicFileAttributes) + throws IOException { Path dirName = dir.getFileName(); - logger.info("PreVisit {}", dir); - if (skipPathNames.contains(dirName.toString())) + if (skipPathNames.contains(dirName.toString())) { + logger.info(getSessionMarker(), "PreVisit skip {}", dir); return FileVisitResult.SKIP_SUBTREE; - else - logger.info("PreVisit {}", dir); + } else + logger.info(getSessionMarker(), "PreVisit {}", dir); return FileVisitResult.CONTINUE; } private boolean processPathItem(Path mediaPath) throws Exception { - if (submitted == limit) + if (limit != 0 && submitted == limit) { + logger.info(getSessionMarker(), "Limit reached {}, canceling", limit); + return false; + } + + String fileName = mediaPath.getFileName().toString(); + if (fileName.startsWith(".") || fileName.endsWith(".nomd")) return false; - // if (!"Transfered".equals(mediaPath.getParent().getFileName().toString())) - // return false; + logger.info(getSessionMarker(), "Processing {}", mediaPath); - if (mediaPath.getFileName().toString().startsWith(".")) + File mediaFile = mediaPath.toFile(); + if (mediaFile.isDirectory()) { + //logger.info(getSessionMarker(), "Skipping directory {}", mediaPath); return false; + } - File mediaFile = mediaPath.toFile(); - if (mediaFile.isDirectory()) + if (EscortFiles.isMediaCatched(mediaPath)) { + //logger.info(getSessionMarker(), "Skipping already catched {}", mediaPath); return false; + } - if (EscortFiles.isMediaCatched(mediaPath)) + Path nomdFile = Paths.get(mediaPath.toString() + ".nomd"); + + if (nomdFile.toFile().exists()) { + //logger.info(getSessionMarker(), "Skipping nomd file exists {}", nomdFile); return false; + } - if (handleArchiveConflict(mediaPath)) + if (handleArchiveConflict(mediaPath)) { + logger.info(getSessionMarker(), "Skipping archive db already contains {}", mediaPath); return false; + } - if (!canReadMediaInfo(mediaPath)) + if (!canReadMediaInfo(mediaPath)) { + logger.info(getSessionMarker(), "Skipping cant read mediainfo {}", mediaPath); return false; + } ArchiveItem archiveItem = createArchiveItem(mediaPath); + if (archiveItem == null) { + Message msg = new ParameterizedMessage("Nincs metaadat!"); + logger.info(new MediaCubeMarker("vasary@elgekko.net,muszak@mediavivantis.hu", + "Értesítés problémás " + mediaPath.getFileName().toString() + " archiválásról"), msg); + Files.createFile(nomdFile); + return false; + } + try { - checkArchiveItem(archiveItem, mediaPath); - submitted++; + checkArchiveItem(archiveItem); Map parameters = ListUtils.asMap(ARCHIVE_ITEM, archiveItem, KILL_DATE_DAYS, killDateDays); IJobRuntime runtime = getEngine().submit(getJobRuntime(), e -> { @@ -265,9 +294,13 @@ public class ArchiveRecursive extends JobStep implements FileVisitor { throw new Exception("Submit returned null runtime"); runtime.setRelated(mediaPath.toString()); EscortFiles.createMediaCatch(mediaPath); + String metadata = archiveItemJSON(archiveItem); + EscortFiles.createMetadata(mediaPath.getParent().toString(), mediaPath.getFileName().toString(), metadata); + submitted++; } catch (Exception e) { - String fileName = new File(archiveItem.getMediaFile()).getName(); - logger.error(getSessionMarker(), "Az '{}' állomány archiválási kísérlete sikertelen. A rendszer üzenete: {}", fileName, e.getMessage()); + logger.error(getSessionMarker(), + "Az '{}' állomány archiválási kísérlete sikertelen. A rendszer üzenete: {}", mediaPath, + e.getMessage()); } return true; @@ -275,7 +308,7 @@ public class ArchiveRecursive extends JobStep implements FileVisitor { @Override public FileVisitResult visitFile(Path filePath, BasicFileAttributes paramBasicFileAttributes) throws IOException { - //logger.info("Will archived {}", filePath); + // logger.info(getSessionMarker(), "Will checked {}", filePath); try { processPathItem(filePath); } catch (Exception e) { -- 2.54.0