From 53ac878ca149c42bfc7b415ee1f3565d655f3207 Mon Sep 17 00:00:00 2001 From: "vasary.daniel" Date: Mon, 23 Aug 2021 15:04:34 +0000 Subject: [PATCH] git-tfs-id: [http://tfs.userrendszerhaz.hu:8080/tfs/DefaultCollection]$/MediaCube;C32321 --- .../mediacube/executors/tests/SmallTests.java | 13 +++ .../server/steps/SafeDeleteRecursive.java | 87 +++++++++++++------ 2 files changed, 75 insertions(+), 25 deletions(-) diff --git a/server/hu.user.mediacube.executors.tests/src/hu/user/mediacube/executors/tests/SmallTests.java b/server/hu.user.mediacube.executors.tests/src/hu/user/mediacube/executors/tests/SmallTests.java index 3c11fab1..bf52e648 100644 --- a/server/hu.user.mediacube.executors.tests/src/hu/user/mediacube/executors/tests/SmallTests.java +++ b/server/hu.user.mediacube.executors.tests/src/hu/user/mediacube/executors/tests/SmallTests.java @@ -25,6 +25,7 @@ import java.time.format.DateTimeFormatter; import java.time.temporal.ChronoUnit; import java.util.ArrayList; import java.util.Arrays; +import java.util.Calendar; import java.util.Collection; import java.util.Collections; import java.util.Date; @@ -1443,4 +1444,16 @@ public class SmallTests { } } + + @Test + public void test999997() throws Exception { + Calendar cal = Calendar.getInstance(); + cal.set(Calendar.HOUR_OF_DAY, 0); + cal.set(Calendar.MINUTE, 0); + cal.set(Calendar.SECOND, 0); + cal.set(Calendar.MILLISECOND, 0); + + SimpleDateFormat df = new SimpleDateFormat("yyyy.MM.dd HH:mm:ss"); + System.out.println(df.format(cal.getTime())); + } } diff --git a/server/user.jobengine.executors/src/user/jobengine/server/steps/SafeDeleteRecursive.java b/server/user.jobengine.executors/src/user/jobengine/server/steps/SafeDeleteRecursive.java index 42c1bf9b..2ad105e9 100644 --- a/server/user.jobengine.executors/src/user/jobengine/server/steps/SafeDeleteRecursive.java +++ b/server/user.jobengine.executors/src/user/jobengine/server/steps/SafeDeleteRecursive.java @@ -22,13 +22,14 @@ import org.apache.commons.io.FileUtils; import org.apache.commons.io.FilenameUtils; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; +import org.apache.logging.log4j.Marker; +import org.apache.logging.log4j.MarkerManager; import com.ibm.nosql.json.api.BasicDBObject; import user.commons.IEntityBase; import user.commons.RemoteFile; import user.commons.StoreUri; -import user.commons.mediatool.MediaInfo; import user.commons.remotestore.RemoteStoreProtocol; import user.jobengine.db.Item; import user.jobengine.db.Media; @@ -46,6 +47,8 @@ import user.mediacube.metadata.interfaces.PlanAirMetadataListOptions; public class SafeDeleteRecursive extends JobStep implements FileVisitor { private static final Logger logger = LogManager.getLogger(); + private Marker csvMarker = MarkerManager.getMarker("SAFE-DELETE-RECURSIVE-CSV"); + private static final String ITEM_TITLE = "itemTitle"; private static final String ITEM_HOUSEID = "itemHouseId"; private static final String MEDIA_HOUSEID = "mediaHouseId"; @@ -53,54 +56,45 @@ public class SafeDeleteRecursive extends JobStep implements FileVisitor { private static final String MEDIA_DESCRIPTION = "mediaDescription"; private static final String MEDIA_TYPE = "mediaType"; private static final String KILLDATEEXT = ".killdate"; + private static final String EWC2EXT = ".ewc2"; + private static final String XMPEXT = ".xmp"; + private SimpleDateFormat df = new SimpleDateFormat("yyyy.MM.dd HH:mm:ss"); private List skipPathNames = Arrays.asList("!ARCHIVALAS_ALATT", EscortFiles.STATUSFOLDER, EscortFiles.CONFLICTFOLDER); private Set includeList; - private boolean canReadMediaInfo(Path mediaFilePath) { - boolean result = false; - try { - MediaInfo mi = new MediaInfo(mediaFilePath); - mi.process(); - result = true; - } catch (Exception e) { - logger.warn(getSessionMarker(), e.getMessage()); - } - return result; - } - - private boolean checkArchiveItem(ArchiveItem archiveItem, Item item, Media media) { + private boolean checkArchiveItem(Path mediaPath, ArchiveItem archiveItem, Item item, Media media) { if (archiveItem == null) return false; String itemHouseId = archiveItem.getItemHouseId() == null ? "" : archiveItem.getItemHouseId(); if (!itemHouseId.equals(item.getHouseId())) { - logger.error(getSessionMarker(), "ItemHouseId"); + logger.error(getSessionMarker(), "{} ItemHouseId mismatch", mediaPath); return false; } String itemTitle = archiveItem.getItemTitle() == null ? "" : archiveItem.getItemTitle(); if (!itemTitle.equals(item.getTitle())) { - logger.error(getSessionMarker(), "ItemTitle"); + logger.error(getSessionMarker(), "{} ItemTitle mismatch", mediaPath); return false; } String mediaHouseId = archiveItem.getMediaHouseId() == null ? "" : archiveItem.getMediaHouseId(); if (!mediaHouseId.equals(media.getHouseId())) { - logger.error(getSessionMarker(), "MediaHouseId"); + logger.error(getSessionMarker(), "{} MediaHouseId mismatch", mediaPath); return false; } String mediaTitle = archiveItem.getMediaTitle() == null ? "" : archiveItem.getMediaTitle(); if (!mediaTitle.equals(media.getTitle())) { - logger.error(getSessionMarker(), "MediaTitle"); + logger.error(getSessionMarker(), "{} MediaTitle mismatch", mediaPath); return false; } String mediaDescription = archiveItem.getMediaDescription() == null ? "" : archiveItem.getMediaDescription(); String storedMediaDescription = media.getDescription() == null ? "" : media.getDescription(); if (!mediaDescription.equals(storedMediaDescription)) { - logger.error(getSessionMarker(), "MediaDescription"); + logger.error(getSessionMarker(), "{} MediaDescription mismatch", mediaPath); return false; } @@ -128,10 +122,10 @@ public class SafeDeleteRecursive extends JobStep implements FileVisitor { @StepEntry public Object[] execute(String sourcePath) throws Exception { logger.info(getSessionMarker(), "Starting in {}", sourcePath); - logger.info(getSessionMarker(), "{};{};{};{};{};{};{};{};{};{};{};{};{};{};{};{};{};{};", "Napló időbélyeg", - "Neve", "Elérése", "Mérete", "Létrehozva", "Módosítva", "Archiválva (MC)", "Archiválva (TSM)", - "Méret (TSM)", ".catched", ".killdate", ".nomd", "Archivált (MC)", "Archivált (TSM)", - "Metaadat egyezés", "Méret egyezés", "Törölhető", "Kezelendő"); + logger.info(csvMarker, "{};{};{};{};{};{};{};{};{};{};{};{};{};{};{};{};{};{};", "Napló időbélyeg", "Neve", + "Elérése", "Mérete", "Létrehozva", "Módosítva", "Archiválva (MC)", "Archiválva (TSM)", "Méret (TSM)", + ".catched", ".killdate", ".nomd", "Archivált (MC)", "Archivált (TSM)", "Metaadat egyezés", + "Méret egyezés", "Törölhető", "Kezelendő"); try { String location = "/opt/test-mediacube/file_list_original.txt"; @@ -207,6 +201,42 @@ public class SafeDeleteRecursive extends JobStep implements FileVisitor { return FileVisitResult.CONTINUE; } + private boolean removeFile(Path filePath) { + boolean result = false; + try { + File file = filePath.toFile(); + if (file.exists()) { + logger.info(getSessionMarker(), "Törlés {}", filePath); + result = true; + //result = file.delete(); + } + } catch (Exception e) { + logger.error(getSessionMarker(), "A {} fájl nem törölhető. A rendszer hibaüzenete: {}", filePath, + e.getMessage()); + } + return result; + } + + private boolean removeFiles(Path filePath, List killDateFiles) { + if (!removeFile(filePath)) + return false; + + removeFile(Paths.get(filePath.toString() + EWC2EXT)); + removeFile(Paths.get(filePath.toString() + XMPEXT)); + removeFile(Paths.get(filePath.getParent().toString(), EscortFiles.STATUSFOLDER, + filePath.getFileName().toString() + EscortFiles.DOT_CATCHED)); + removeFile(Paths.get(filePath.getParent().toString(), EscortFiles.STATUSFOLDER, + filePath.getFileName().toString() + EscortFiles.DOT_JSON)); + + boolean result = true; + for (Path killDateFile : killDateFiles) { + if (!removeFile(killDateFile)) + result = false; + } + + return result; + } + private boolean processPathItem(Path mediaPath) throws Exception { if (getJobRuntime().isWaitingCancel()) { cancel(); @@ -250,7 +280,7 @@ public class SafeDeleteRecursive extends JobStep implements FileVisitor { Media media = getManager().getMedia(mediaFile.getMediaId()); mcArchivedDate = media.getCreated(); Item item = getManager().getItem(media.getItemId()); - metadataEquals = checkArchiveItem(archiveItem, item, media); + metadataEquals = checkArchiveItem(mediaPath, archiveItem, item, media); //tsm String tsmFileName = mediaFile.getRelativePath(); @@ -266,11 +296,18 @@ public class SafeDeleteRecursive extends JobStep implements FileVisitor { boolean canDelete = !noMDExists && catchedExists && killdateExists && mcArchived && tsmArchived && metadataEquals; Date now = new Date(System.currentTimeMillis()); - logger.info(getSessionMarker(), "{};{};{};{};{};{};{};{};{};{};{};{};{};{};{};{};{};{};", D(now), fileName, + logger.info(csvMarker, "{};{};{};{};{};{};{};{};{};{};{};{};{};{};{};{};{};{};", D(now), fileName, mediaPath.getParent(), size, D(createDate), D(lastModifiedDate), D(mcArchivedDate), D(tsmBackupDate), tsmSize, YN(catchedExists), YN(killdateExists), YN(noMDExists), YN(mcArchived), YN(tsmArchived), YN(metadataEquals), YN(sizeEquals), YN(canDelete), YN(includeContains)); + logger.info(getSessionMarker(), "{} {}{}{}{}{}{}{}{}{}", mediaPath, YN(canDelete), YN(catchedExists), + YN(killdateExists), YN(noMDExists), YN(mcArchived), YN(tsmArchived), YN(metadataEquals), YN(sizeEquals), + YN(includeContains)); + + if (canDelete) + removeFiles(mediaPath, killDateFiles); + return true; } -- 2.54.0