Implemented CSV logging, before final test
authorelgekko <vasary@elgekko.net>
Mon, 11 Sep 2023 12:39:52 +0000 (14:39 +0200)
committerelgekko <vasary@elgekko.net>
Mon, 11 Sep 2023 12:39:52 +0000 (14:39 +0200)
22 files changed:
.gitignore [new file with mode: 0644]
mc-intergator/.idea/modules.xml [new file with mode: 0644]
mc-intergator/mc-db/target/mc-db-0.0.1-SNAPSHOT.jar [deleted file]
mc-intergator/mc-integrator.iml [new file with mode: 0644]
mc-intergator/mc-safe-delete/README.md [new file with mode: 0644]
mc-intergator/mc-safe-delete/pom.xml
mc-intergator/mc-safe-delete/src/main/java/hu/user/mediacube/integration/safedelete/ArchiveFileStatus.java
mc-intergator/mc-safe-delete/src/main/java/hu/user/mediacube/integration/safedelete/FileOperations.java
mc-intergator/mc-safe-delete/src/main/java/hu/user/mediacube/integration/safedelete/SafeDeleteCommand.java
mc-intergator/mc-safe-delete/src/main/java/hu/user/mediacube/integration/safedelete/SafeDeleteProperties.java
mc-intergator/mc-safe-delete/src/main/java/hu/user/mediacube/integration/safedelete/logging/MarkerFilter.java [new file with mode: 0644]
mc-intergator/mc-safe-delete/src/main/java/hu/user/mediacube/integration/safedelete/services/ArchiveStatusService.java [new file with mode: 0644]
mc-intergator/mc-safe-delete/src/main/java/hu/user/mediacube/integration/safedelete/services/StatusFileService.java [new file with mode: 0644]
mc-intergator/mc-safe-delete/src/main/java/hu/user/mediacube/integration/safedelete/verifier/MediaCubeDatabaseService.java
mc-intergator/mc-safe-delete/src/main/java/hu/user/mediacube/integration/safedelete/verifier/TSMService.java
mc-intergator/mc-safe-delete/src/main/resources/application.yaml
mc-intergator/mc-safe-delete/src/main/resources/logback.xml
mc-intergator/mc-safe-delete/start-mc-safe-delete.sh [new file with mode: 0644]
mc-intergator/mc-safe-delete/target/classes/application.yaml [deleted file]
mc-intergator/mc-safe-delete/target/classes/logback.xml [deleted file]
mc-intergator/mc-safe-delete/target/maven-archiver/pom.properties [deleted file]
mc-intergator/mc-safe-delete/target/maven-status/maven-compiler-plugin/compile/default-compile/inputFiles.lst [deleted file]

diff --git a/.gitignore b/.gitignore
new file mode 100644 (file)
index 0000000..232ccd1
--- /dev/null
@@ -0,0 +1,2 @@
+target/
+logs/
diff --git a/mc-intergator/.idea/modules.xml b/mc-intergator/.idea/modules.xml
new file mode 100644 (file)
index 0000000..14c76e4
--- /dev/null
@@ -0,0 +1,8 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<project version="4">
+  <component name="ProjectModuleManager">
+    <modules>
+      <module fileurl="file://$PROJECT_DIR$/mc-integrator.iml" filepath="$PROJECT_DIR$/mc-integrator.iml" />
+    </modules>
+  </component>
+</project>
\ No newline at end of file
diff --git a/mc-intergator/mc-db/target/mc-db-0.0.1-SNAPSHOT.jar b/mc-intergator/mc-db/target/mc-db-0.0.1-SNAPSHOT.jar
deleted file mode 100644 (file)
index 09991a5..0000000
Binary files a/mc-intergator/mc-db/target/mc-db-0.0.1-SNAPSHOT.jar and /dev/null differ
diff --git a/mc-intergator/mc-integrator.iml b/mc-intergator/mc-integrator.iml
new file mode 100644 (file)
index 0000000..e0b14e3
--- /dev/null
@@ -0,0 +1,8 @@
+<?xml version="1.0" encoding="UTF-8"?>
+<module version="4">
+  <component name="AdditionalModuleElements">
+    <content url="file://$MODULE_DIR$" dumb="true">
+      <excludeFolder url="file://$MODULE_DIR$/logs" />
+    </content>
+  </component>
+</module>
\ No newline at end of file
diff --git a/mc-intergator/mc-safe-delete/README.md b/mc-intergator/mc-safe-delete/README.md
new file mode 100644 (file)
index 0000000..22a0615
--- /dev/null
@@ -0,0 +1,34 @@
+A programot az mc-safe-dalete.sh parancsfájllal lehet elindítani.
+A parancsfájlban használt kapcsolók, amik a program futását befolyásolják:
+-f
+Hiánya esetén nincs fizikai fájl törlés. Alkalmazása esetén van !
+-d
+A viszgálandó mappa teljes elérési útja. A .STATUS mappát ez alatt keresi.
+
+A program kilistázza a .STATUS mappa alatt található, .killdate végződésű fájlokat.
+A .killdate fájlokból képzett fájlnév megléte azt jelenti a programnak, hogy ezekkel a fájlokkal foglalkozhat törlés
+vizsgálat szempontjából.
+
+Ezután végighalad a bemenetként kapott mappa fájljain, és amennyiben az előző lista tartalmazza a nevet, akkor:
+
+1. Nulla méretű fájl esetén bejátszó ismétlésről van szó: törli a fájlt és annak státusz fájljait. Ebben az esetben
+   nincs visszaellenőrzés, mert tényleges fájl archiválás nem történik.
+2. Nullától különböző méretű fájl esetén, amennyiben létezik a MediaCube adatbázisban metaadat bejegyzés,
+   továbbá a TSM-ből a program által visszatöltött fájl mérete és a fájl tartalmából képzett MD5 hash megegyezik:
+   törli a fájlt és státusz fájljait.
+
+Időzített indítás az operációs rendszer ütemezőjével oldható meg.
+A program addig fut, míg a beállításban engedélyezett óraszám le nem telik, vagy míg el nem fogy a mappa tartalma.
+Erről a max-execution-hours beállítás rendelkezik. -1 érték esetén nincs limit, 0 esetén egy fájlt dolgoz fel, 0-nál
+nagyobb érték esetén óraszámként értemezi.
+Az eltelt idő ellenőrzését minden fájl után vizsgálja, tehát ha lejárat előtt 5 perccel elkezd feldolgozni egy
+állományt és közben lejár az idő, akkor a feldolgozás még végigmegy.
+
+A futási beállításokat az application.yaml fájlból olvassa fel.
+
+A naplózási beállításokat a logback.xml fájlból olvassa fel.
+A program napi bontású emberi olvasásra alkalmas szöveges naplófájlt hoz létre a logs mappában.
+A program napi bontású, CSV formátumú naplófájlt hoz létre a kezelt fájl adatok számára a logs mappában.
+Minden elindítás során az első bejegyzés a fejléc lesz:
+"timestamp;fileName;fileSize;creationTime;lastAccessTime;lastModifiedTime;metadataOnly;metadataExists;tsmFileExists;
+fileSizeEquals;hashEquals;canDelete;deleteSuccess;statusCleanupSuccess"
\ No newline at end of file
index 94a98192d4fa35c767c1ada48b4d20e559257bcb..ae3c8722fdf6be0c4fda300f006f1d96d1eb644d 100644 (file)
@@ -3,7 +3,7 @@
          xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 https://maven.apache.org/xsd/maven-4.0.0.xsd">
     <modelVersion>4.0.0</modelVersion>
     <artifactId>mc-safe-delete</artifactId>
-    <version>0.0.1-SNAPSHOT</version>
+    <version>0.0.2-SNAPSHOT</version>
     <parent>
         <groupId>hu.user.mediacube</groupId>
         <artifactId>mc-integrator</artifactId>
index 2666d373e1ba6955f2abaff2a7a77b3dce8a8c47..9a19ff4f99329a219644e98a5439f8543372512c 100644 (file)
@@ -4,12 +4,24 @@ import lombok.Builder;
 import lombok.Getter;
 import lombok.Setter;
 
+import java.nio.file.attribute.FileTime;
+
 @Getter
 @Setter
 @Builder
 public class ArchiveFileStatus {
-    
-    private String name;
+
+    private String fileName;
+
+    private long fileSize;
+
+    private FileTime creationTime;
+
+    private FileTime lastAccessTime;
+
+    private FileTime lastModifiedTime;
+
+    private boolean isMetadataOnly;
 
     private boolean metadataExists;
 
@@ -19,4 +31,10 @@ public class ArchiveFileStatus {
 
     private boolean hashEquals;
 
+    private boolean canDelete;
+
+    private boolean deleteSuccess;
+
+    private boolean statusCleanupSuccess;
+
 }
index 4e089d3774648d6f754f7a98eba2556f11b7e1a2..ad5be295747fa741915b708f9ca8b8b0266de051 100644 (file)
@@ -2,21 +2,37 @@ package hu.user.mediacube.integration.safedelete;
 
 import lombok.extern.log4j.Log4j2;
 
+import java.nio.file.DirectoryStream;
 import java.nio.file.Files;
 import java.nio.file.Path;
 import java.nio.file.Paths;
 
 @Log4j2
 public class FileOperations {
+    public static final String ERROR_FILENAME_PREFIX = "ERROR-";
+    public static final String KILLDATE_POSTFIX = ".killdate";
+    public static final String STATUS = "/.STATUS";
 
-    static public void silentDelete(Path path) {
+    static public Path getStatusPath(Path inputPath) {
+        Path result = null;
+        if (Files.isDirectory(inputPath)) {
+            result = Paths.get(inputPath.toAbsolutePath().toString(), STATUS);
+        } else {
+            result = Paths.get(inputPath.getParent().toAbsolutePath().toString(), STATUS);
+        }
+        return result;
+    }
+
+    static public boolean silentDelete(Path path) {
+        boolean result = false;
         try {
             Files.delete(path);
             log.info("File deleted: {}", path);
+            result = true;
         } catch (Exception e) {
             log.catching(e);
         }
-
+        return result;
     }
 
     static public void silentRename(Path source, String prefix) {
@@ -29,4 +45,16 @@ public class FileOperations {
             log.catching(e);
         }
     }
+
+    public static DirectoryStream.Filter<Path> filterOnlyFiles() {
+        return entry -> !Files.isDirectory(entry) && !entry.getFileName().toString().startsWith(ERROR_FILENAME_PREFIX);
+    }
+
+    public static DirectoryStream.Filter<Path> filterKillDateFiles() {
+        return entry -> !Files.isDirectory(entry) && entry.getFileName().toString().endsWith(KILLDATE_POSTFIX);
+    }
+
+    public static DirectoryStream.Filter<Path> filterRelatedFiles(String originalFileName) {
+        return entry -> !Files.isDirectory(entry) && entry.getFileName().toString().startsWith(originalFileName);
+    }
 }
index baa96ad2b570ff9f290b4ceecf1a1e4da74dba01..f625cf2197e2ea1dcd189a5e3a0f1b9c272c8bae 100644 (file)
@@ -1,5 +1,7 @@
 package hu.user.mediacube.integration.safedelete;
 
+import hu.user.mediacube.integration.safedelete.services.ArchiveStatusService;
+import hu.user.mediacube.integration.safedelete.services.StatusFileService;
 import hu.user.mediacube.integration.safedelete.verifier.MediaCubeDatabaseService;
 import hu.user.mediacube.integration.safedelete.verifier.TSMService;
 import lombok.extern.log4j.Log4j2;
@@ -13,13 +15,14 @@ import java.nio.file.Path;
 import java.nio.file.Paths;
 import java.time.Duration;
 import java.time.Instant;
+import java.time.LocalDate;
 import java.time.format.DateTimeFormatter;
+import java.util.Set;
+import java.util.TimeZone;
 
 @Component
 @Log4j2
 public class SafeDeleteCommand {
-    private static final String ERROR_FILENAME_PREFIX = "ERROR-";
-
     @Autowired
     private SafeDeleteProperties safeDeleteProperties;
 
@@ -29,48 +32,83 @@ public class SafeDeleteCommand {
     @Autowired
     private TSMService tsmService;
 
-    private final DateTimeFormatter formatter = DateTimeFormatter.ofPattern("yyyyMMddHHmmss");
+    @Autowired
+    private ArchiveStatusService archivestatusService;
+
+    @Autowired
+    private StatusFileService statusFileService;
+
+    private final DateTimeFormatter formatter = DateTimeFormatter.ofPattern("yyyyMMddHHmmss").withZone(TimeZone.getDefault().toZoneId());
 
     public void processDirectory(String directory, boolean forceDeleteArchived) throws IOException {
+        archivestatusService.logHeader();
         final Instant start = Instant.now();
         log.info("Check directory: {}", directory);
         log.info("Delete already archived: {}", forceDeleteArchived);
         Path inputPath = Paths.get(directory);
-        try (DirectoryStream<Path> stream = Files.newDirectoryStream(inputPath)) {
+
+        Set<String> fileNamesWithKillDates = statusFileService.loadFileNamesWithKillDates(inputPath);
+
+        try (DirectoryStream<Path> stream = Files.newDirectoryStream(inputPath, FileOperations.filterOnlyFiles())) {
             for (Path filePath : stream) {
-                if (Files.isDirectory(filePath) || filePath.getFileName().toString().startsWith(ERROR_FILENAME_PREFIX)) {
-                    continue;
+                ArchiveFileStatus status = archivestatusService.createStatus(filePath);
+
+                try {
+                    process(filePath, status, fileNamesWithKillDates, forceDeleteArchived);
+                } catch (Exception e) {
+                    log.error(e.getMessage());
+                    FileOperations.silentRename(filePath, FileOperations.ERROR_FILENAME_PREFIX + formatter.format(LocalDate.now()));
+                } finally {
+                    archivestatusService.logStatus(status);
                 }
-                processInputFile(filePath, forceDeleteArchived);
+
                 if (safeDeleteProperties.getMaxExecutionHours() > -1
                         && safeDeleteProperties.getMaxExecutionHours() <= Duration.between(start, Instant.now()).toHours()
                 ) {
-                    log.info("Reached maximum execution duration");
+                    log.info("Reached maximum enabled execution duration {} hours(s)", safeDeleteProperties.getMaxExecutionHours());
                     break;
                 }
             }
         }
     }
 
-    private void processInputFile(Path source, boolean forceDeleteArchived) {
-        ArchiveFileStatus status = ArchiveFileStatus.builder().name(source.getFileName().toString()).build();
-        String fileName = source.getFileName().toString();
-
-        try {
-            log.info("Check MediaCube metadata for {}", source);
-            mediaCubeDatabaseService.verify(fileName, status);
-
-            log.info("Check TSM file for {}", source);
-            tsmService.verify(source, status);
-
-            if (forceDeleteArchived && status.isFileSizeEquals() && status.isHashEquals()) {
-                FileOperations.silentDelete(source);
+    private void process(Path filePath, ArchiveFileStatus status, Set<String> fileNamesWithKillDates, boolean forceDeleteArchived) throws Exception {
+        boolean canDelete = fileNamesWithKillDates.contains(status.getFileName());
+        if (canDelete) {
+            if (status.getFileSize() == 0) {
+                processEmptyInputFile(filePath, status, forceDeleteArchived);
+            } else {
+                processInputFile(filePath, status, forceDeleteArchived);
             }
+        } else {
+            log.info("File {} not processed yet by archive system, .killdate status file is missing", filePath.toString());
+        }
+    }
+
+    private void processEmptyInputFile(Path source, ArchiveFileStatus status, boolean forceDeleteArchived) throws Exception {
+        status.setMetadataOnly(true);
+        status.setCanDelete(true);
+        if (forceDeleteArchived) {
+            deleteArchivedFileAndStatusFiles(source, status);
+        }
+    }
 
-        } catch (Exception e) {
-            log.catching(e);
-            FileOperations.silentRename(source, ERROR_FILENAME_PREFIX);
+    private void processInputFile(Path source, ArchiveFileStatus status, boolean forceDeleteArchived) throws Exception {
+        log.info("Check MediaCube metadata for {}", source);
+        mediaCubeDatabaseService.verify(source, status);
+        log.info("Check TSM file for {}", source);
+        tsmService.verify(source, status);
+        status.setCanDelete(status.isFileSizeEquals() && status.isHashEquals());
+        if (forceDeleteArchived && status.isCanDelete()) {
+            deleteArchivedFileAndStatusFiles(source, status);
         }
     }
+
+    private void deleteArchivedFileAndStatusFiles(Path source, ArchiveFileStatus status) throws IOException {
+        String fileName = source.getFileName().toString();
+        Path statusPath = FileOperations.getStatusPath(source);
+        status.setStatusCleanupSuccess(statusFileService.cleanupStatusFiles(statusPath, fileName));
+        status.setDeleteSuccess(FileOperations.silentDelete(source));
+    }
 }
 
index 1d4dfa70ac2a03e28daee5f026c3a385d5a86341..18434919cdcbbfa2cfa403880bf1df59dc8ae5f5 100644 (file)
@@ -8,22 +8,28 @@ import org.springframework.context.annotation.Configuration;
 @Configuration
 @Getter
 @Setter
-@ConfigurationProperties(prefix = "tsm")
+@ConfigurationProperties(prefix = "safe-delete")
 public class SafeDeleteProperties {
-    private String nodeName;
+    private int maxExecutionHours;
+    private Tsm tsm;
 
-    private String fsName;
+    @Getter
+    @Setter
+    public static class Tsm {
+        private String nodeName;
 
-    private String hlName;
+        private String fsName;
 
-    private String delimiter;
+        private String hlName;
 
-    private String user;
+        private String delimiter;
 
-    private String password;
+        private String user;
 
-    private int bufferSize;
+        private String password;
+
+        private int bufferSize;
+    }
 
-    private int maxExecutionHours;
 
 }
diff --git a/mc-intergator/mc-safe-delete/src/main/java/hu/user/mediacube/integration/safedelete/logging/MarkerFilter.java b/mc-intergator/mc-safe-delete/src/main/java/hu/user/mediacube/integration/safedelete/logging/MarkerFilter.java
new file mode 100644 (file)
index 0000000..8300152
--- /dev/null
@@ -0,0 +1,58 @@
+package hu.user.mediacube.integration.safedelete.logging;
+
+import ch.qos.logback.classic.spi.ILoggingEvent;
+import ch.qos.logback.core.filter.AbstractMatcherFilter;
+import ch.qos.logback.core.spi.FilterReply;
+import org.apache.logging.log4j.Marker;
+import org.apache.logging.log4j.MarkerManager;
+
+/**
+ * @author yoshiori_shoji
+ */
+public class MarkerFilter extends AbstractMatcherFilter<ILoggingEvent> {
+
+    Marker markerToMatch;
+
+    /*
+     * (non-Javadoc)
+     *
+     * @see ch.qos.logback.core.filter.Filter#decide(java.lang.Object)
+     */
+    @Override
+    public FilterReply decide(ILoggingEvent event) {
+        if (!isStarted()) {
+            return FilterReply.NEUTRAL;
+        }
+        org.slf4j.Marker marker = event.getMarker();
+        if (marker == null) {
+            return onMismatch;
+        }
+
+        if (markerToMatch.toString().equals(marker.toString())) {
+            return onMatch;
+        } else {
+            return onMismatch;
+        }
+    }
+
+    /**
+     * The marker to match in the event.
+     *
+     * @param markerToMatch
+     */
+    public void setMarker(String markerStr) {
+        markerToMatch = MarkerManager.getMarker(markerStr);
+    }
+
+    /*
+     * (non-Javadoc)
+     *
+     * @see ch.qos.logback.core.filter.Filter#start()
+     */
+    @Override
+    public void start() {
+        if (this.markerToMatch != null) {
+            super.start();
+        }
+    }
+}
\ No newline at end of file
diff --git a/mc-intergator/mc-safe-delete/src/main/java/hu/user/mediacube/integration/safedelete/services/ArchiveStatusService.java b/mc-intergator/mc-safe-delete/src/main/java/hu/user/mediacube/integration/safedelete/services/ArchiveStatusService.java
new file mode 100644 (file)
index 0000000..20dc79a
--- /dev/null
@@ -0,0 +1,53 @@
+package hu.user.mediacube.integration.safedelete.services;
+
+import hu.user.mediacube.integration.safedelete.ArchiveFileStatus;
+import lombok.extern.log4j.Log4j2;
+import org.apache.logging.log4j.Marker;
+import org.apache.logging.log4j.MarkerManager;
+import org.springframework.stereotype.Service;
+
+import java.io.IOException;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.attribute.BasicFileAttributes;
+
+@Log4j2
+@Service
+public class ArchiveStatusService {
+    private final Marker CSV_MARKER = MarkerManager.getMarker("CSV_MARKER");
+
+    public ArchiveFileStatus createStatus(Path filePath) throws IOException {
+        String fileName = filePath.getFileName().toString();
+        long length = filePath.toFile().length();
+        BasicFileAttributes attr = Files.readAttributes(filePath, BasicFileAttributes.class);
+
+        return ArchiveFileStatus.builder()
+                .fileName(fileName)
+                .fileSize(length)
+                .creationTime(attr.creationTime())
+                .creationTime(attr.lastAccessTime())
+                .creationTime(attr.lastModifiedTime())
+                .build();
+    }
+
+    public void logHeader() {
+        log.info(CSV_MARKER, "timestamp;fileName;fileSize;creationTime;lastAccessTime;lastModifiedTime;metadataOnly;metadataExists;tsmFileExists;fileSizeEquals;hashEquals;canDelete;deleteSuccess;statusCleanupSuccess");
+    }
+
+    public void logStatus(ArchiveFileStatus status) {
+        log.info(CSV_MARKER, "{};{};{};{};{};{};{};{};{};{};{};{};{}",
+                status.getFileName(),
+                status.getFileSize(),
+                status.getCreationTime(),
+                status.getLastAccessTime(),
+                status.getLastModifiedTime(),
+                status.isMetadataOnly(),
+                status.isMetadataExists(),
+                status.isTsmFileExists(),
+                status.isFileSizeEquals(),
+                status.isHashEquals(),
+                status.isCanDelete(),
+                status.isDeleteSuccess(),
+                status.isStatusCleanupSuccess());
+    }
+}
diff --git a/mc-intergator/mc-safe-delete/src/main/java/hu/user/mediacube/integration/safedelete/services/StatusFileService.java b/mc-intergator/mc-safe-delete/src/main/java/hu/user/mediacube/integration/safedelete/services/StatusFileService.java
new file mode 100644 (file)
index 0000000..7235266
--- /dev/null
@@ -0,0 +1,47 @@
+package hu.user.mediacube.integration.safedelete.services;
+
+import hu.user.mediacube.integration.safedelete.FileOperations;
+import lombok.extern.log4j.Log4j2;
+import org.springframework.stereotype.Service;
+
+import java.io.IOException;
+import java.nio.file.DirectoryStream;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.util.HashSet;
+import java.util.Set;
+
+@Log4j2
+@Service
+public class StatusFileService {
+
+    public Set<String> loadFileNamesWithKillDates(Path inputPath) throws IOException {
+        Set<String> result = new HashSet<>();
+        Path statusPath = FileOperations.getStatusPath(inputPath);
+        if (Files.exists(statusPath) && Files.isDirectory(statusPath)) {
+            try (DirectoryStream<Path> stream = Files.newDirectoryStream(statusPath, FileOperations.filterKillDateFiles())) {
+                stream.forEach(file -> {
+                    String[] tokens = file.getFileName().toString().split("\\.");
+                    if (tokens.length > 1) {
+                        result.add(tokens[0] + "." + tokens[1]);
+                    }
+                });
+            }
+        }
+        return result;
+    }
+
+
+    public boolean cleanupStatusFiles(Path statusPath, String fileName) throws IOException {
+        boolean[] statusCleanupSuccess = {true};
+        try (DirectoryStream<Path> stream = Files.newDirectoryStream(statusPath, FileOperations.filterRelatedFiles(fileName))) {
+            stream.forEach(file -> {
+                if (!FileOperations.silentDelete(file)) {
+                    statusCleanupSuccess[0] = false;
+                }
+            });
+        }
+        return statusCleanupSuccess[0];
+    }
+
+}
index 1741e18f5e8f98be517d020fcc552cfb47fd9544..545b4c5c42064e125d9b2ae4b1aeaa7e8a45abea 100644 (file)
@@ -7,6 +7,7 @@ import lombok.extern.log4j.Log4j2;
 import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.stereotype.Service;
 
+import java.nio.file.Path;
 import java.util.List;
 
 @Service
@@ -15,8 +16,8 @@ public class MediaCubeDatabaseService {
     @Autowired
     MediaCubeRecordMapper mediaCubeRecordMapper;
 
-    public void verify(String fileName, ArchiveFileStatus status) throws Exception {
-        List<MediaCubeRecord> dbRecords = mediaCubeRecordMapper.getByFileName(fileName);
+    public void verify(Path source, ArchiveFileStatus status) throws Exception {
+        List<MediaCubeRecord> dbRecords = mediaCubeRecordMapper.getByFileName(source.getFileName().toString());
         if (dbRecords.size() != 1) {
             throw new Exception(String.format("Database record count mismatch. Expected 1 found %d", dbRecords.size()));
         }
index eed9471cd9126b217b72d44d5634d75dac7ca756..367e149e289479c9068b678fe8e7eec719087102 100644 (file)
@@ -31,12 +31,12 @@ public class TSMService {
         Path result = null;
         TSMClient server = null;
         try {
-            server = new TSMClient(safeDeleteProperties.getNodeName());
-            server.connect(safeDeleteProperties.getUser(), safeDeleteProperties.getPassword());
-            server.setBufferSize(safeDeleteProperties.getBufferSize());
+            server = new TSMClient(safeDeleteProperties.getTsm().getNodeName());
+            server.connect(safeDeleteProperties.getTsm().getUser(), safeDeleteProperties.getTsm().getPassword());
+            server.setBufferSize(safeDeleteProperties.getTsm().getBufferSize());
 
-            TSMBackupFileObject backupFileObject = server.getActiveBackupFileObject(safeDeleteProperties.getFsName(),
-                    safeDeleteProperties.getHlName(), safeDeleteProperties.getDelimiter() + source.getFileName().toString());
+            TSMBackupFileObject backupFileObject = server.getActiveBackupFileObject(safeDeleteProperties.getTsm().getFsName(),
+                    safeDeleteProperties.getTsm().getHlName(), safeDeleteProperties.getTsm().getDelimiter() + source.getFileName().toString());
             if (backupFileObject != null) {
                 result = Paths.get(source.getParent().toAbsolutePath().toString(),
                         source.getFileName().toString() + UUID.randomUUID());
index 86dbc4a3c8ed70edb92ab0d7631ead29325bde53..56854758cf5b54814f01277fb97b775f5a62cfc6 100644 (file)
@@ -14,11 +14,13 @@ spring:
       pool-name: pool-mediacube
       maximum-pool-size: 5
       minimum-idle: 1
-tsm:
-  user: mediacube
-  password: password
-  buffer-size: 65536
-  delimiter: /
-  node-name: mc-safe-delete
-  fs-name: /JOBENGINE
-  hl-name: /JOBENGINE
+safe-delete:
+  max-execution-hours: 2
+  tsm:
+    user: mediacube
+    password: password
+    buffer-size: 65536
+    delimiter: /
+    node-name: mc-safe-delete
+    fs-name: /JOBENGINE
+    hl-name: /JOBENGINE
index 01730548ef174786cd7b3474b441f143449d0b75..78af067579913c8f2eb9b5bdd7209f8db0dd4633 100644 (file)
@@ -1,5 +1,56 @@
 <?xml version="1.0" encoding="UTF-8"?>
 <configuration>
     <include resource="org/springframework/boot/logging/logback/base.xml"/>
-    <logger name="org.springframework" level="ERROR"/>
+    <property name="LOGS" value="./logs"/>
+
+    <appender name="Console" class="ch.qos.logback.core.ConsoleAppender">
+        <layout class="ch.qos.logback.classic.PatternLayout">
+            <Pattern>
+                %black(%d{ISO8601}) %highlight(%-5level) [%blue(%t)] %yellow(%C{1.}): %msg%n%throwable
+            </Pattern>
+        </layout>
+    </appender>
+
+    <appender name="RollingFile" class="ch.qos.logback.core.rolling.RollingFileAppender">
+        <filter class="hu.user.mediacube.integration.safedelete.logging.MarkerFilter">
+            <marker>CSV_MARKER</marker>
+            <onMatch>DENY</onMatch>
+            <onMismatch>ACCEPT</onMismatch>
+        </filter>
+        <file>${LOGS}/mc-safe-delete.log</file>
+        <encoder class="ch.qos.logback.classic.encoder.PatternLayoutEncoder">
+            <Pattern>%d %p %C{1.} [%t] %m%n</Pattern>
+        </encoder>
+        <rollingPolicy class="ch.qos.logback.core.rolling.TimeBasedRollingPolicy">
+            <fileNamePattern>${LOGS}/archived/mc-safe-delete-%d{yyyy-MM-dd}.log.zip</fileNamePattern>
+        </rollingPolicy>
+    </appender>
+
+    <appender name="RollingCSVFile" class="ch.qos.logback.core.rolling.RollingFileAppender">
+        <filter class="hu.user.mediacube.integration.safedelete.logging.MarkerFilter">
+            <marker>CSV_MARKER</marker>
+            <onMatch>ACCEPT</onMatch>
+            <onMismatch>DENY</onMismatch>
+        </filter>
+        <file>${LOGS}/mc-safe-delete.csv</file>
+        <encoder class="ch.qos.logback.classic.encoder.PatternLayoutEncoder">
+            <Pattern>%d{ISO8601};%m%n</Pattern>
+        </encoder>
+        <rollingPolicy class="ch.qos.logback.core.rolling.TimeBasedRollingPolicy">
+            <fileNamePattern>${LOGS}/archived/mc-safe-delete-%d{yyyy-MM-dd}.csv.zip</fileNamePattern>
+        </rollingPolicy>
+    </appender>
+
+    <root level="info">
+        <appender-ref ref="RollingFile"/>
+        <appender-ref ref="RollingCSVFile"/>
+        <appender-ref ref="Console"/>
+    </root>
+
+    <logger name="hu.user.mediacube" level="trace" additivity="false">
+        <appender-ref ref="RollingFile"/>
+        <appender-ref ref="RollingCSVFile"/>
+        <appender-ref ref="Console"/>
+    </logger>
+
 </configuration>
\ No newline at end of file
diff --git a/mc-intergator/mc-safe-delete/start-mc-safe-delete.sh b/mc-intergator/mc-safe-delete/start-mc-safe-delete.sh
new file mode 100644 (file)
index 0000000..36cda61
--- /dev/null
@@ -0,0 +1,17 @@
+export DSMI_CONFIG=/opt/mediacube/settings/dsm.opt
+export DSMI_DIR=/opt/tivoli/tsm/client/api/bin64
+export DSMI_LOG=/opt/mediacube/log
+export DSM_DIR=/opt/tivoli/tsm/client/api/bin64
+export LD_LIBRARY_PATH=/opt/tivoli/tsm/client/ba/bin:/opt/tivoli/tsm/client/api/bin64:/opt/mediacube
+export LIBPATH=/opt/tivoli/tsm/client/ba/bin:/opt/tivoli/tsm/client/api/bin64
+export SHLIB_PATH=/opt/tivoli/tsm/client/ba/bin:/opt/tivoli/tsm/client/api/bin64
+export PATH=/usr/lib/jvm/java-1.8.0-openjdk-1.8.0.102-4.b14.el7.x86_64/bin:$PATH
+java \
+-DMC-SAFE-DELETE \
+-Dfile.encoding=UTF-8 \
+-Xms1024m \
+-Xmx2048m \
+-jar mc-safe-delete-0.0.1-SNAPSHOT.jar \
+-d /mnt/ISILON/ARCHIVE/TEST/ \
+-f
+
diff --git a/mc-intergator/mc-safe-delete/target/classes/application.yaml b/mc-intergator/mc-safe-delete/target/classes/application.yaml
deleted file mode 100644 (file)
index 86dbc4a..0000000
+++ /dev/null
@@ -1,24 +0,0 @@
-spring:
-  main:
-    banner-mode: off
-  output:
-    ansi:
-      enabled: always
-  datasource:
-    url: jdbc:db2://10.10.1.27:50000/mc:retrieveMessagesFromServerOnGetMessage=true;
-    username: db2admin
-    password: password
-    driver-class-name: com.ibm.db2.jcc.DB2Driver
-    type: com.zaxxer.hikari.HikariDataSource
-    hikari:
-      pool-name: pool-mediacube
-      maximum-pool-size: 5
-      minimum-idle: 1
-tsm:
-  user: mediacube
-  password: password
-  buffer-size: 65536
-  delimiter: /
-  node-name: mc-safe-delete
-  fs-name: /JOBENGINE
-  hl-name: /JOBENGINE
diff --git a/mc-intergator/mc-safe-delete/target/classes/logback.xml b/mc-intergator/mc-safe-delete/target/classes/logback.xml
deleted file mode 100644 (file)
index 0173054..0000000
+++ /dev/null
@@ -1,5 +0,0 @@
-<?xml version="1.0" encoding="UTF-8"?>
-<configuration>
-    <include resource="org/springframework/boot/logging/logback/base.xml"/>
-    <logger name="org.springframework" level="ERROR"/>
-</configuration>
\ No newline at end of file
diff --git a/mc-intergator/mc-safe-delete/target/maven-archiver/pom.properties b/mc-intergator/mc-safe-delete/target/maven-archiver/pom.properties
deleted file mode 100644 (file)
index 9306887..0000000
+++ /dev/null
@@ -1,3 +0,0 @@
-version=0.0.1-SNAPSHOT
-groupId=hu.user.mediacube
-artifactId=mc-safe-delete
diff --git a/mc-intergator/mc-safe-delete/target/maven-status/maven-compiler-plugin/compile/default-compile/inputFiles.lst b/mc-intergator/mc-safe-delete/target/maven-status/maven-compiler-plugin/compile/default-compile/inputFiles.lst
deleted file mode 100644 (file)
index d7a812e..0000000
+++ /dev/null
@@ -1,9 +0,0 @@
-C:\work\user\mediacube\mc-intergator\mc-safe-delete\src\main\java\hu\user\mediacube\integration\safedelete\SafeDeleteMainEntry.java
-C:\work\user\mediacube\mc-intergator\mc-safe-delete\src\main\java\hu\user\mediacube\integration\safedelete\SafeDeleteProperties.java
-C:\work\user\mediacube\mc-intergator\mc-safe-delete\src\main\java\hu\user\mediacube\integration\safedelete\ArchiveFileStatus.java
-C:\work\user\mediacube\mc-intergator\mc-safe-delete\src\main\java\hu\user\mediacube\integration\safedelete\db\MediaCubeRecord.java
-C:\work\user\mediacube\mc-intergator\mc-safe-delete\src\main\java\hu\user\mediacube\integration\safedelete\db\MediaCubeRecordMapper.java
-C:\work\user\mediacube\mc-intergator\mc-safe-delete\src\main\java\hu\user\mediacube\integration\safedelete\verifier\MediaCubeDatabaseService.java
-C:\work\user\mediacube\mc-intergator\mc-safe-delete\src\main\java\hu\user\mediacube\integration\safedelete\verifier\TSMService.java
-C:\work\user\mediacube\mc-intergator\mc-safe-delete\src\main\java\hu\user\mediacube\integration\safedelete\FileOperations.java
-C:\work\user\mediacube\mc-intergator\mc-safe-delete\src\main\java\hu\user\mediacube\integration\safedelete\SafeDeleteCommand.java