From: elgekko Date: Mon, 11 Sep 2023 12:39:52 +0000 (+0200) Subject: Implemented CSV logging, before final test X-Git-Url: http://git.useribm.hu/?a=commitdiff_plain;h=9c096dc5956e0ea882a86623d8d665bb05460868;p=mediacube.git Implemented CSV logging, before final test --- diff --git a/.gitignore b/.gitignore new file mode 100644 index 00000000..232ccd1d --- /dev/null +++ b/.gitignore @@ -0,0 +1,2 @@ +target/ +logs/ diff --git a/mc-intergator/.idea/modules.xml b/mc-intergator/.idea/modules.xml new file mode 100644 index 00000000..14c76e49 --- /dev/null +++ b/mc-intergator/.idea/modules.xml @@ -0,0 +1,8 @@ + + + + + + + + \ No newline at end of file diff --git a/mc-intergator/mc-db/target/mc-db-0.0.1-SNAPSHOT.jar b/mc-intergator/mc-db/target/mc-db-0.0.1-SNAPSHOT.jar deleted file mode 100644 index 09991a5f..00000000 Binary files a/mc-intergator/mc-db/target/mc-db-0.0.1-SNAPSHOT.jar and /dev/null differ diff --git a/mc-intergator/mc-integrator.iml b/mc-intergator/mc-integrator.iml new file mode 100644 index 00000000..e0b14e33 --- /dev/null +++ b/mc-intergator/mc-integrator.iml @@ -0,0 +1,8 @@ + + + + + + + + \ No newline at end of file diff --git a/mc-intergator/mc-safe-delete/README.md b/mc-intergator/mc-safe-delete/README.md new file mode 100644 index 00000000..22a06153 --- /dev/null +++ b/mc-intergator/mc-safe-delete/README.md @@ -0,0 +1,34 @@ +A programot az mc-safe-dalete.sh parancsfájllal lehet elindítani. +A parancsfájlban használt kapcsolók, amik a program futását befolyásolják: +-f +Hiánya esetén nincs fizikai fájl törlés. Alkalmazása esetén van ! +-d +A viszgálandó mappa teljes elérési útja. A .STATUS mappát ez alatt keresi. + +A program kilistázza a .STATUS mappa alatt található, .killdate végződésű fájlokat. +A .killdate fájlokból képzett fájlnév megléte azt jelenti a programnak, hogy ezekkel a fájlokkal foglalkozhat törlés +vizsgálat szempontjából. + +Ezután végighalad a bemenetként kapott mappa fájljain, és amennyiben az előző lista tartalmazza a nevet, akkor: + +1. Nulla méretű fájl esetén bejátszó ismétlésről van szó: törli a fájlt és annak státusz fájljait. Ebben az esetben + nincs visszaellenőrzés, mert tényleges fájl archiválás nem történik. +2. Nullától különböző méretű fájl esetén, amennyiben létezik a MediaCube adatbázisban metaadat bejegyzés, + továbbá a TSM-ből a program által visszatöltött fájl mérete és a fájl tartalmából képzett MD5 hash megegyezik: + törli a fájlt és státusz fájljait. + +Időzített indítás az operációs rendszer ütemezőjével oldható meg. +A program addig fut, míg a beállításban engedélyezett óraszám le nem telik, vagy míg el nem fogy a mappa tartalma. +Erről a max-execution-hours beállítás rendelkezik. -1 érték esetén nincs limit, 0 esetén egy fájlt dolgoz fel, 0-nál +nagyobb érték esetén óraszámként értemezi. +Az eltelt idő ellenőrzését minden fájl után vizsgálja, tehát ha lejárat előtt 5 perccel elkezd feldolgozni egy +állományt és közben lejár az idő, akkor a feldolgozás még végigmegy. + +A futási beállításokat az application.yaml fájlból olvassa fel. + +A naplózási beállításokat a logback.xml fájlból olvassa fel. +A program napi bontású emberi olvasásra alkalmas szöveges naplófájlt hoz létre a logs mappában. +A program napi bontású, CSV formátumú naplófájlt hoz létre a kezelt fájl adatok számára a logs mappában. +Minden elindítás során az első bejegyzés a fejléc lesz: +"timestamp;fileName;fileSize;creationTime;lastAccessTime;lastModifiedTime;metadataOnly;metadataExists;tsmFileExists; +fileSizeEquals;hashEquals;canDelete;deleteSuccess;statusCleanupSuccess" \ No newline at end of file diff --git a/mc-intergator/mc-safe-delete/pom.xml b/mc-intergator/mc-safe-delete/pom.xml index 94a98192..ae3c8722 100644 --- a/mc-intergator/mc-safe-delete/pom.xml +++ b/mc-intergator/mc-safe-delete/pom.xml @@ -3,7 +3,7 @@ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 https://maven.apache.org/xsd/maven-4.0.0.xsd"> 4.0.0 mc-safe-delete - 0.0.1-SNAPSHOT + 0.0.2-SNAPSHOT hu.user.mediacube mc-integrator diff --git a/mc-intergator/mc-safe-delete/src/main/java/hu/user/mediacube/integration/safedelete/ArchiveFileStatus.java b/mc-intergator/mc-safe-delete/src/main/java/hu/user/mediacube/integration/safedelete/ArchiveFileStatus.java index 2666d373..9a19ff4f 100644 --- a/mc-intergator/mc-safe-delete/src/main/java/hu/user/mediacube/integration/safedelete/ArchiveFileStatus.java +++ b/mc-intergator/mc-safe-delete/src/main/java/hu/user/mediacube/integration/safedelete/ArchiveFileStatus.java @@ -4,12 +4,24 @@ import lombok.Builder; import lombok.Getter; import lombok.Setter; +import java.nio.file.attribute.FileTime; + @Getter @Setter @Builder public class ArchiveFileStatus { - - private String name; + + private String fileName; + + private long fileSize; + + private FileTime creationTime; + + private FileTime lastAccessTime; + + private FileTime lastModifiedTime; + + private boolean isMetadataOnly; private boolean metadataExists; @@ -19,4 +31,10 @@ public class ArchiveFileStatus { private boolean hashEquals; + private boolean canDelete; + + private boolean deleteSuccess; + + private boolean statusCleanupSuccess; + } diff --git a/mc-intergator/mc-safe-delete/src/main/java/hu/user/mediacube/integration/safedelete/FileOperations.java b/mc-intergator/mc-safe-delete/src/main/java/hu/user/mediacube/integration/safedelete/FileOperations.java index 4e089d37..ad5be295 100644 --- a/mc-intergator/mc-safe-delete/src/main/java/hu/user/mediacube/integration/safedelete/FileOperations.java +++ b/mc-intergator/mc-safe-delete/src/main/java/hu/user/mediacube/integration/safedelete/FileOperations.java @@ -2,21 +2,37 @@ package hu.user.mediacube.integration.safedelete; import lombok.extern.log4j.Log4j2; +import java.nio.file.DirectoryStream; import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.Paths; @Log4j2 public class FileOperations { + public static final String ERROR_FILENAME_PREFIX = "ERROR-"; + public static final String KILLDATE_POSTFIX = ".killdate"; + public static final String STATUS = "/.STATUS"; - static public void silentDelete(Path path) { + static public Path getStatusPath(Path inputPath) { + Path result = null; + if (Files.isDirectory(inputPath)) { + result = Paths.get(inputPath.toAbsolutePath().toString(), STATUS); + } else { + result = Paths.get(inputPath.getParent().toAbsolutePath().toString(), STATUS); + } + return result; + } + + static public boolean silentDelete(Path path) { + boolean result = false; try { Files.delete(path); log.info("File deleted: {}", path); + result = true; } catch (Exception e) { log.catching(e); } - + return result; } static public void silentRename(Path source, String prefix) { @@ -29,4 +45,16 @@ public class FileOperations { log.catching(e); } } + + public static DirectoryStream.Filter filterOnlyFiles() { + return entry -> !Files.isDirectory(entry) && !entry.getFileName().toString().startsWith(ERROR_FILENAME_PREFIX); + } + + public static DirectoryStream.Filter filterKillDateFiles() { + return entry -> !Files.isDirectory(entry) && entry.getFileName().toString().endsWith(KILLDATE_POSTFIX); + } + + public static DirectoryStream.Filter filterRelatedFiles(String originalFileName) { + return entry -> !Files.isDirectory(entry) && entry.getFileName().toString().startsWith(originalFileName); + } } diff --git a/mc-intergator/mc-safe-delete/src/main/java/hu/user/mediacube/integration/safedelete/SafeDeleteCommand.java b/mc-intergator/mc-safe-delete/src/main/java/hu/user/mediacube/integration/safedelete/SafeDeleteCommand.java index baa96ad2..f625cf21 100644 --- a/mc-intergator/mc-safe-delete/src/main/java/hu/user/mediacube/integration/safedelete/SafeDeleteCommand.java +++ b/mc-intergator/mc-safe-delete/src/main/java/hu/user/mediacube/integration/safedelete/SafeDeleteCommand.java @@ -1,5 +1,7 @@ package hu.user.mediacube.integration.safedelete; +import hu.user.mediacube.integration.safedelete.services.ArchiveStatusService; +import hu.user.mediacube.integration.safedelete.services.StatusFileService; import hu.user.mediacube.integration.safedelete.verifier.MediaCubeDatabaseService; import hu.user.mediacube.integration.safedelete.verifier.TSMService; import lombok.extern.log4j.Log4j2; @@ -13,13 +15,14 @@ import java.nio.file.Path; import java.nio.file.Paths; import java.time.Duration; import java.time.Instant; +import java.time.LocalDate; import java.time.format.DateTimeFormatter; +import java.util.Set; +import java.util.TimeZone; @Component @Log4j2 public class SafeDeleteCommand { - private static final String ERROR_FILENAME_PREFIX = "ERROR-"; - @Autowired private SafeDeleteProperties safeDeleteProperties; @@ -29,48 +32,83 @@ public class SafeDeleteCommand { @Autowired private TSMService tsmService; - private final DateTimeFormatter formatter = DateTimeFormatter.ofPattern("yyyyMMddHHmmss"); + @Autowired + private ArchiveStatusService archivestatusService; + + @Autowired + private StatusFileService statusFileService; + + private final DateTimeFormatter formatter = DateTimeFormatter.ofPattern("yyyyMMddHHmmss").withZone(TimeZone.getDefault().toZoneId()); public void processDirectory(String directory, boolean forceDeleteArchived) throws IOException { + archivestatusService.logHeader(); final Instant start = Instant.now(); log.info("Check directory: {}", directory); log.info("Delete already archived: {}", forceDeleteArchived); Path inputPath = Paths.get(directory); - try (DirectoryStream stream = Files.newDirectoryStream(inputPath)) { + + Set fileNamesWithKillDates = statusFileService.loadFileNamesWithKillDates(inputPath); + + try (DirectoryStream stream = Files.newDirectoryStream(inputPath, FileOperations.filterOnlyFiles())) { for (Path filePath : stream) { - if (Files.isDirectory(filePath) || filePath.getFileName().toString().startsWith(ERROR_FILENAME_PREFIX)) { - continue; + ArchiveFileStatus status = archivestatusService.createStatus(filePath); + + try { + process(filePath, status, fileNamesWithKillDates, forceDeleteArchived); + } catch (Exception e) { + log.error(e.getMessage()); + FileOperations.silentRename(filePath, FileOperations.ERROR_FILENAME_PREFIX + formatter.format(LocalDate.now())); + } finally { + archivestatusService.logStatus(status); } - processInputFile(filePath, forceDeleteArchived); + if (safeDeleteProperties.getMaxExecutionHours() > -1 && safeDeleteProperties.getMaxExecutionHours() <= Duration.between(start, Instant.now()).toHours() ) { - log.info("Reached maximum execution duration"); + log.info("Reached maximum enabled execution duration {} hours(s)", safeDeleteProperties.getMaxExecutionHours()); break; } } } } - private void processInputFile(Path source, boolean forceDeleteArchived) { - ArchiveFileStatus status = ArchiveFileStatus.builder().name(source.getFileName().toString()).build(); - String fileName = source.getFileName().toString(); - - try { - log.info("Check MediaCube metadata for {}", source); - mediaCubeDatabaseService.verify(fileName, status); - - log.info("Check TSM file for {}", source); - tsmService.verify(source, status); - - if (forceDeleteArchived && status.isFileSizeEquals() && status.isHashEquals()) { - FileOperations.silentDelete(source); + private void process(Path filePath, ArchiveFileStatus status, Set fileNamesWithKillDates, boolean forceDeleteArchived) throws Exception { + boolean canDelete = fileNamesWithKillDates.contains(status.getFileName()); + if (canDelete) { + if (status.getFileSize() == 0) { + processEmptyInputFile(filePath, status, forceDeleteArchived); + } else { + processInputFile(filePath, status, forceDeleteArchived); } + } else { + log.info("File {} not processed yet by archive system, .killdate status file is missing", filePath.toString()); + } + } + + private void processEmptyInputFile(Path source, ArchiveFileStatus status, boolean forceDeleteArchived) throws Exception { + status.setMetadataOnly(true); + status.setCanDelete(true); + if (forceDeleteArchived) { + deleteArchivedFileAndStatusFiles(source, status); + } + } - } catch (Exception e) { - log.catching(e); - FileOperations.silentRename(source, ERROR_FILENAME_PREFIX); + private void processInputFile(Path source, ArchiveFileStatus status, boolean forceDeleteArchived) throws Exception { + log.info("Check MediaCube metadata for {}", source); + mediaCubeDatabaseService.verify(source, status); + log.info("Check TSM file for {}", source); + tsmService.verify(source, status); + status.setCanDelete(status.isFileSizeEquals() && status.isHashEquals()); + if (forceDeleteArchived && status.isCanDelete()) { + deleteArchivedFileAndStatusFiles(source, status); } } + + private void deleteArchivedFileAndStatusFiles(Path source, ArchiveFileStatus status) throws IOException { + String fileName = source.getFileName().toString(); + Path statusPath = FileOperations.getStatusPath(source); + status.setStatusCleanupSuccess(statusFileService.cleanupStatusFiles(statusPath, fileName)); + status.setDeleteSuccess(FileOperations.silentDelete(source)); + } } diff --git a/mc-intergator/mc-safe-delete/src/main/java/hu/user/mediacube/integration/safedelete/SafeDeleteProperties.java b/mc-intergator/mc-safe-delete/src/main/java/hu/user/mediacube/integration/safedelete/SafeDeleteProperties.java index 1d4dfa70..18434919 100644 --- a/mc-intergator/mc-safe-delete/src/main/java/hu/user/mediacube/integration/safedelete/SafeDeleteProperties.java +++ b/mc-intergator/mc-safe-delete/src/main/java/hu/user/mediacube/integration/safedelete/SafeDeleteProperties.java @@ -8,22 +8,28 @@ import org.springframework.context.annotation.Configuration; @Configuration @Getter @Setter -@ConfigurationProperties(prefix = "tsm") +@ConfigurationProperties(prefix = "safe-delete") public class SafeDeleteProperties { - private String nodeName; + private int maxExecutionHours; + private Tsm tsm; - private String fsName; + @Getter + @Setter + public static class Tsm { + private String nodeName; - private String hlName; + private String fsName; - private String delimiter; + private String hlName; - private String user; + private String delimiter; - private String password; + private String user; - private int bufferSize; + private String password; + + private int bufferSize; + } - private int maxExecutionHours; } diff --git a/mc-intergator/mc-safe-delete/src/main/java/hu/user/mediacube/integration/safedelete/logging/MarkerFilter.java b/mc-intergator/mc-safe-delete/src/main/java/hu/user/mediacube/integration/safedelete/logging/MarkerFilter.java new file mode 100644 index 00000000..83001520 --- /dev/null +++ b/mc-intergator/mc-safe-delete/src/main/java/hu/user/mediacube/integration/safedelete/logging/MarkerFilter.java @@ -0,0 +1,58 @@ +package hu.user.mediacube.integration.safedelete.logging; + +import ch.qos.logback.classic.spi.ILoggingEvent; +import ch.qos.logback.core.filter.AbstractMatcherFilter; +import ch.qos.logback.core.spi.FilterReply; +import org.apache.logging.log4j.Marker; +import org.apache.logging.log4j.MarkerManager; + +/** + * @author yoshiori_shoji + */ +public class MarkerFilter extends AbstractMatcherFilter { + + Marker markerToMatch; + + /* + * (non-Javadoc) + * + * @see ch.qos.logback.core.filter.Filter#decide(java.lang.Object) + */ + @Override + public FilterReply decide(ILoggingEvent event) { + if (!isStarted()) { + return FilterReply.NEUTRAL; + } + org.slf4j.Marker marker = event.getMarker(); + if (marker == null) { + return onMismatch; + } + + if (markerToMatch.toString().equals(marker.toString())) { + return onMatch; + } else { + return onMismatch; + } + } + + /** + * The marker to match in the event. + * + * @param markerToMatch + */ + public void setMarker(String markerStr) { + markerToMatch = MarkerManager.getMarker(markerStr); + } + + /* + * (non-Javadoc) + * + * @see ch.qos.logback.core.filter.Filter#start() + */ + @Override + public void start() { + if (this.markerToMatch != null) { + super.start(); + } + } +} \ No newline at end of file diff --git a/mc-intergator/mc-safe-delete/src/main/java/hu/user/mediacube/integration/safedelete/services/ArchiveStatusService.java b/mc-intergator/mc-safe-delete/src/main/java/hu/user/mediacube/integration/safedelete/services/ArchiveStatusService.java new file mode 100644 index 00000000..20dc79a4 --- /dev/null +++ b/mc-intergator/mc-safe-delete/src/main/java/hu/user/mediacube/integration/safedelete/services/ArchiveStatusService.java @@ -0,0 +1,53 @@ +package hu.user.mediacube.integration.safedelete.services; + +import hu.user.mediacube.integration.safedelete.ArchiveFileStatus; +import lombok.extern.log4j.Log4j2; +import org.apache.logging.log4j.Marker; +import org.apache.logging.log4j.MarkerManager; +import org.springframework.stereotype.Service; + +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.attribute.BasicFileAttributes; + +@Log4j2 +@Service +public class ArchiveStatusService { + private final Marker CSV_MARKER = MarkerManager.getMarker("CSV_MARKER"); + + public ArchiveFileStatus createStatus(Path filePath) throws IOException { + String fileName = filePath.getFileName().toString(); + long length = filePath.toFile().length(); + BasicFileAttributes attr = Files.readAttributes(filePath, BasicFileAttributes.class); + + return ArchiveFileStatus.builder() + .fileName(fileName) + .fileSize(length) + .creationTime(attr.creationTime()) + .creationTime(attr.lastAccessTime()) + .creationTime(attr.lastModifiedTime()) + .build(); + } + + public void logHeader() { + log.info(CSV_MARKER, "timestamp;fileName;fileSize;creationTime;lastAccessTime;lastModifiedTime;metadataOnly;metadataExists;tsmFileExists;fileSizeEquals;hashEquals;canDelete;deleteSuccess;statusCleanupSuccess"); + } + + public void logStatus(ArchiveFileStatus status) { + log.info(CSV_MARKER, "{};{};{};{};{};{};{};{};{};{};{};{};{}", + status.getFileName(), + status.getFileSize(), + status.getCreationTime(), + status.getLastAccessTime(), + status.getLastModifiedTime(), + status.isMetadataOnly(), + status.isMetadataExists(), + status.isTsmFileExists(), + status.isFileSizeEquals(), + status.isHashEquals(), + status.isCanDelete(), + status.isDeleteSuccess(), + status.isStatusCleanupSuccess()); + } +} diff --git a/mc-intergator/mc-safe-delete/src/main/java/hu/user/mediacube/integration/safedelete/services/StatusFileService.java b/mc-intergator/mc-safe-delete/src/main/java/hu/user/mediacube/integration/safedelete/services/StatusFileService.java new file mode 100644 index 00000000..7235266d --- /dev/null +++ b/mc-intergator/mc-safe-delete/src/main/java/hu/user/mediacube/integration/safedelete/services/StatusFileService.java @@ -0,0 +1,47 @@ +package hu.user.mediacube.integration.safedelete.services; + +import hu.user.mediacube.integration.safedelete.FileOperations; +import lombok.extern.log4j.Log4j2; +import org.springframework.stereotype.Service; + +import java.io.IOException; +import java.nio.file.DirectoryStream; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.HashSet; +import java.util.Set; + +@Log4j2 +@Service +public class StatusFileService { + + public Set loadFileNamesWithKillDates(Path inputPath) throws IOException { + Set result = new HashSet<>(); + Path statusPath = FileOperations.getStatusPath(inputPath); + if (Files.exists(statusPath) && Files.isDirectory(statusPath)) { + try (DirectoryStream stream = Files.newDirectoryStream(statusPath, FileOperations.filterKillDateFiles())) { + stream.forEach(file -> { + String[] tokens = file.getFileName().toString().split("\\."); + if (tokens.length > 1) { + result.add(tokens[0] + "." + tokens[1]); + } + }); + } + } + return result; + } + + + public boolean cleanupStatusFiles(Path statusPath, String fileName) throws IOException { + boolean[] statusCleanupSuccess = {true}; + try (DirectoryStream stream = Files.newDirectoryStream(statusPath, FileOperations.filterRelatedFiles(fileName))) { + stream.forEach(file -> { + if (!FileOperations.silentDelete(file)) { + statusCleanupSuccess[0] = false; + } + }); + } + return statusCleanupSuccess[0]; + } + +} diff --git a/mc-intergator/mc-safe-delete/src/main/java/hu/user/mediacube/integration/safedelete/verifier/MediaCubeDatabaseService.java b/mc-intergator/mc-safe-delete/src/main/java/hu/user/mediacube/integration/safedelete/verifier/MediaCubeDatabaseService.java index 1741e18f..545b4c5c 100644 --- a/mc-intergator/mc-safe-delete/src/main/java/hu/user/mediacube/integration/safedelete/verifier/MediaCubeDatabaseService.java +++ b/mc-intergator/mc-safe-delete/src/main/java/hu/user/mediacube/integration/safedelete/verifier/MediaCubeDatabaseService.java @@ -7,6 +7,7 @@ import lombok.extern.log4j.Log4j2; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Service; +import java.nio.file.Path; import java.util.List; @Service @@ -15,8 +16,8 @@ public class MediaCubeDatabaseService { @Autowired MediaCubeRecordMapper mediaCubeRecordMapper; - public void verify(String fileName, ArchiveFileStatus status) throws Exception { - List dbRecords = mediaCubeRecordMapper.getByFileName(fileName); + public void verify(Path source, ArchiveFileStatus status) throws Exception { + List dbRecords = mediaCubeRecordMapper.getByFileName(source.getFileName().toString()); if (dbRecords.size() != 1) { throw new Exception(String.format("Database record count mismatch. Expected 1 found %d", dbRecords.size())); } diff --git a/mc-intergator/mc-safe-delete/src/main/java/hu/user/mediacube/integration/safedelete/verifier/TSMService.java b/mc-intergator/mc-safe-delete/src/main/java/hu/user/mediacube/integration/safedelete/verifier/TSMService.java index eed9471c..367e149e 100644 --- a/mc-intergator/mc-safe-delete/src/main/java/hu/user/mediacube/integration/safedelete/verifier/TSMService.java +++ b/mc-intergator/mc-safe-delete/src/main/java/hu/user/mediacube/integration/safedelete/verifier/TSMService.java @@ -31,12 +31,12 @@ public class TSMService { Path result = null; TSMClient server = null; try { - server = new TSMClient(safeDeleteProperties.getNodeName()); - server.connect(safeDeleteProperties.getUser(), safeDeleteProperties.getPassword()); - server.setBufferSize(safeDeleteProperties.getBufferSize()); + server = new TSMClient(safeDeleteProperties.getTsm().getNodeName()); + server.connect(safeDeleteProperties.getTsm().getUser(), safeDeleteProperties.getTsm().getPassword()); + server.setBufferSize(safeDeleteProperties.getTsm().getBufferSize()); - TSMBackupFileObject backupFileObject = server.getActiveBackupFileObject(safeDeleteProperties.getFsName(), - safeDeleteProperties.getHlName(), safeDeleteProperties.getDelimiter() + source.getFileName().toString()); + TSMBackupFileObject backupFileObject = server.getActiveBackupFileObject(safeDeleteProperties.getTsm().getFsName(), + safeDeleteProperties.getTsm().getHlName(), safeDeleteProperties.getTsm().getDelimiter() + source.getFileName().toString()); if (backupFileObject != null) { result = Paths.get(source.getParent().toAbsolutePath().toString(), source.getFileName().toString() + UUID.randomUUID()); diff --git a/mc-intergator/mc-safe-delete/src/main/resources/application.yaml b/mc-intergator/mc-safe-delete/src/main/resources/application.yaml index 86dbc4a3..56854758 100644 --- a/mc-intergator/mc-safe-delete/src/main/resources/application.yaml +++ b/mc-intergator/mc-safe-delete/src/main/resources/application.yaml @@ -14,11 +14,13 @@ spring: pool-name: pool-mediacube maximum-pool-size: 5 minimum-idle: 1 -tsm: - user: mediacube - password: password - buffer-size: 65536 - delimiter: / - node-name: mc-safe-delete - fs-name: /JOBENGINE - hl-name: /JOBENGINE +safe-delete: + max-execution-hours: 2 + tsm: + user: mediacube + password: password + buffer-size: 65536 + delimiter: / + node-name: mc-safe-delete + fs-name: /JOBENGINE + hl-name: /JOBENGINE diff --git a/mc-intergator/mc-safe-delete/src/main/resources/logback.xml b/mc-intergator/mc-safe-delete/src/main/resources/logback.xml index 01730548..78af0675 100644 --- a/mc-intergator/mc-safe-delete/src/main/resources/logback.xml +++ b/mc-intergator/mc-safe-delete/src/main/resources/logback.xml @@ -1,5 +1,56 @@ - + + + + + + %black(%d{ISO8601}) %highlight(%-5level) [%blue(%t)] %yellow(%C{1.}): %msg%n%throwable + + + + + + + CSV_MARKER + DENY + ACCEPT + + ${LOGS}/mc-safe-delete.log + + %d %p %C{1.} [%t] %m%n + + + ${LOGS}/archived/mc-safe-delete-%d{yyyy-MM-dd}.log.zip + + + + + + CSV_MARKER + ACCEPT + DENY + + ${LOGS}/mc-safe-delete.csv + + %d{ISO8601};%m%n + + + ${LOGS}/archived/mc-safe-delete-%d{yyyy-MM-dd}.csv.zip + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/mc-intergator/mc-safe-delete/start-mc-safe-delete.sh b/mc-intergator/mc-safe-delete/start-mc-safe-delete.sh new file mode 100644 index 00000000..36cda617 --- /dev/null +++ b/mc-intergator/mc-safe-delete/start-mc-safe-delete.sh @@ -0,0 +1,17 @@ +export DSMI_CONFIG=/opt/mediacube/settings/dsm.opt +export DSMI_DIR=/opt/tivoli/tsm/client/api/bin64 +export DSMI_LOG=/opt/mediacube/log +export DSM_DIR=/opt/tivoli/tsm/client/api/bin64 +export LD_LIBRARY_PATH=/opt/tivoli/tsm/client/ba/bin:/opt/tivoli/tsm/client/api/bin64:/opt/mediacube +export LIBPATH=/opt/tivoli/tsm/client/ba/bin:/opt/tivoli/tsm/client/api/bin64 +export SHLIB_PATH=/opt/tivoli/tsm/client/ba/bin:/opt/tivoli/tsm/client/api/bin64 +export PATH=/usr/lib/jvm/java-1.8.0-openjdk-1.8.0.102-4.b14.el7.x86_64/bin:$PATH +java \ +-DMC-SAFE-DELETE \ +-Dfile.encoding=UTF-8 \ +-Xms1024m \ +-Xmx2048m \ +-jar mc-safe-delete-0.0.1-SNAPSHOT.jar \ +-d /mnt/ISILON/ARCHIVE/TEST/ \ +-f + diff --git a/mc-intergator/mc-safe-delete/target/classes/application.yaml b/mc-intergator/mc-safe-delete/target/classes/application.yaml deleted file mode 100644 index 86dbc4a3..00000000 --- a/mc-intergator/mc-safe-delete/target/classes/application.yaml +++ /dev/null @@ -1,24 +0,0 @@ -spring: - main: - banner-mode: off - output: - ansi: - enabled: always - datasource: - url: jdbc:db2://10.10.1.27:50000/mc:retrieveMessagesFromServerOnGetMessage=true; - username: db2admin - password: password - driver-class-name: com.ibm.db2.jcc.DB2Driver - type: com.zaxxer.hikari.HikariDataSource - hikari: - pool-name: pool-mediacube - maximum-pool-size: 5 - minimum-idle: 1 -tsm: - user: mediacube - password: password - buffer-size: 65536 - delimiter: / - node-name: mc-safe-delete - fs-name: /JOBENGINE - hl-name: /JOBENGINE diff --git a/mc-intergator/mc-safe-delete/target/classes/logback.xml b/mc-intergator/mc-safe-delete/target/classes/logback.xml deleted file mode 100644 index 01730548..00000000 --- a/mc-intergator/mc-safe-delete/target/classes/logback.xml +++ /dev/null @@ -1,5 +0,0 @@ - - - - - \ No newline at end of file diff --git a/mc-intergator/mc-safe-delete/target/maven-archiver/pom.properties b/mc-intergator/mc-safe-delete/target/maven-archiver/pom.properties deleted file mode 100644 index 9306887f..00000000 --- a/mc-intergator/mc-safe-delete/target/maven-archiver/pom.properties +++ /dev/null @@ -1,3 +0,0 @@ -version=0.0.1-SNAPSHOT -groupId=hu.user.mediacube -artifactId=mc-safe-delete diff --git a/mc-intergator/mc-safe-delete/target/maven-status/maven-compiler-plugin/compile/default-compile/inputFiles.lst b/mc-intergator/mc-safe-delete/target/maven-status/maven-compiler-plugin/compile/default-compile/inputFiles.lst deleted file mode 100644 index d7a812e5..00000000 --- a/mc-intergator/mc-safe-delete/target/maven-status/maven-compiler-plugin/compile/default-compile/inputFiles.lst +++ /dev/null @@ -1,9 +0,0 @@ -C:\work\user\mediacube\mc-intergator\mc-safe-delete\src\main\java\hu\user\mediacube\integration\safedelete\SafeDeleteMainEntry.java -C:\work\user\mediacube\mc-intergator\mc-safe-delete\src\main\java\hu\user\mediacube\integration\safedelete\SafeDeleteProperties.java -C:\work\user\mediacube\mc-intergator\mc-safe-delete\src\main\java\hu\user\mediacube\integration\safedelete\ArchiveFileStatus.java -C:\work\user\mediacube\mc-intergator\mc-safe-delete\src\main\java\hu\user\mediacube\integration\safedelete\db\MediaCubeRecord.java -C:\work\user\mediacube\mc-intergator\mc-safe-delete\src\main\java\hu\user\mediacube\integration\safedelete\db\MediaCubeRecordMapper.java -C:\work\user\mediacube\mc-intergator\mc-safe-delete\src\main\java\hu\user\mediacube\integration\safedelete\verifier\MediaCubeDatabaseService.java -C:\work\user\mediacube\mc-intergator\mc-safe-delete\src\main\java\hu\user\mediacube\integration\safedelete\verifier\TSMService.java -C:\work\user\mediacube\mc-intergator\mc-safe-delete\src\main\java\hu\user\mediacube\integration\safedelete\FileOperations.java -C:\work\user\mediacube\mc-intergator\mc-safe-delete\src\main\java\hu\user\mediacube\integration\safedelete\SafeDeleteCommand.java