From: elgekko Date: Wed, 6 Sep 2023 14:05:55 +0000 (+0200) Subject: Implemented without CSV logging, tested X-Git-Url: http://git.useribm.hu/?a=commitdiff_plain;h=a02043515d50f65a8fc48468cb005a16f53d1cf8;p=mediacube.git Implemented without CSV logging, tested --- diff --git a/mc-intergator/mc-db/pom.xml b/mc-intergator/mc-db/pom.xml index 5dd22938..6db71928 100644 --- a/mc-intergator/mc-db/pom.xml +++ b/mc-intergator/mc-db/pom.xml @@ -49,27 +49,16 @@ - - - - - - - - - - - - org.mybatis.spring.boot - mybatis-spring-boot-starter - 2.1.1 - com.ibm.db2 jcc 11.5.8.0 - + + org.mybatis.spring.boot + mybatis-spring-boot-starter + 2.1.1 + org.springframework.boot spring-boot-starter-test @@ -81,7 +70,5 @@ - - \ No newline at end of file diff --git a/mc-intergator/mc-db/target/maven-status/maven-compiler-plugin/compile/default-compile/createdFiles.lst b/mc-intergator/mc-db/target/maven-status/maven-compiler-plugin/compile/default-compile/createdFiles.lst index aae6faef..0fa053fd 100644 --- a/mc-intergator/mc-db/target/maven-status/maven-compiler-plugin/compile/default-compile/createdFiles.lst +++ b/mc-intergator/mc-db/target/maven-status/maven-compiler-plugin/compile/default-compile/createdFiles.lst @@ -28,6 +28,7 @@ hu\user\mediacube\rdb\model\ItemtypeExample$GeneratedCriteria.class hu\user\mediacube\rdb\model\StoreExample$Criterion.class hu\user\mediacube\rdb\model\StoreExample$Criteria.class hu\user\mediacube\rdb\model\StoreuriExample$Criteria.class +hu\user\mediacube\rdb\MediaCubeMapper.class hu\user\mediacube\rdb\model\FiletypeExample$GeneratedCriteria.class hu\user\mediacube\rdb\model\Mediafile.class hu\user\mediacube\rdb\model\MediaExample$Criterion.class diff --git a/mc-intergator/mc-db/target/mc-db-0.0.1-SNAPSHOT.jar b/mc-intergator/mc-db/target/mc-db-0.0.1-SNAPSHOT.jar index fe0a84e5..09991a5f 100644 Binary files a/mc-intergator/mc-db/target/mc-db-0.0.1-SNAPSHOT.jar and b/mc-intergator/mc-db/target/mc-db-0.0.1-SNAPSHOT.jar differ diff --git a/mc-intergator/mc-safe-delete/pom.xml b/mc-intergator/mc-safe-delete/pom.xml index 9dfad920..94a98192 100644 --- a/mc-intergator/mc-safe-delete/pom.xml +++ b/mc-intergator/mc-safe-delete/pom.xml @@ -10,16 +10,16 @@ 0.0.1-SNAPSHOT + + hu.user.tsm + tsm-clientapi + 1.0.0-SNAPSHOT + hu.user.mediacube mc-db 0.0.1-SNAPSHOT - - hu.user.tsm - tsm-clientapi - 1.0-SNAPSHOT - commons-codec commons-codec diff --git a/mc-intergator/mc-safe-delete/src/main/java/hu/user/mediacube/integration/safedelete/ArchiveFileStatus.java b/mc-intergator/mc-safe-delete/src/main/java/hu/user/mediacube/integration/safedelete/ArchiveFileStatus.java index 5d4b1394..2666d373 100644 --- a/mc-intergator/mc-safe-delete/src/main/java/hu/user/mediacube/integration/safedelete/ArchiveFileStatus.java +++ b/mc-intergator/mc-safe-delete/src/main/java/hu/user/mediacube/integration/safedelete/ArchiveFileStatus.java @@ -8,6 +8,7 @@ import lombok.Setter; @Setter @Builder public class ArchiveFileStatus { + private String name; private boolean metadataExists; diff --git a/mc-intergator/mc-safe-delete/src/main/java/hu/user/mediacube/integration/safedelete/FileOperations.java b/mc-intergator/mc-safe-delete/src/main/java/hu/user/mediacube/integration/safedelete/FileOperations.java index 6dd1b539..4e089d37 100644 --- a/mc-intergator/mc-safe-delete/src/main/java/hu/user/mediacube/integration/safedelete/FileOperations.java +++ b/mc-intergator/mc-safe-delete/src/main/java/hu/user/mediacube/integration/safedelete/FileOperations.java @@ -12,6 +12,7 @@ public class FileOperations { static public void silentDelete(Path path) { try { Files.delete(path); + log.info("File deleted: {}", path); } catch (Exception e) { log.catching(e); } @@ -23,6 +24,7 @@ public class FileOperations { String fileName = String.format("%s%s", prefix, source.getFileName().toString()); Path target = Paths.get(source.getParent().toAbsolutePath().toString(), fileName); Files.move(source, target); + log.info("File renamed to: {}", target); } catch (Exception e) { log.catching(e); } diff --git a/mc-intergator/mc-safe-delete/src/main/java/hu/user/mediacube/integration/safedelete/SafeDeleteCommand.java b/mc-intergator/mc-safe-delete/src/main/java/hu/user/mediacube/integration/safedelete/SafeDeleteCommand.java index fee217ff..baa96ad2 100644 --- a/mc-intergator/mc-safe-delete/src/main/java/hu/user/mediacube/integration/safedelete/SafeDeleteCommand.java +++ b/mc-intergator/mc-safe-delete/src/main/java/hu/user/mediacube/integration/safedelete/SafeDeleteCommand.java @@ -11,6 +11,9 @@ import java.nio.file.DirectoryStream; import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.Paths; +import java.time.Duration; +import java.time.Instant; +import java.time.format.DateTimeFormatter; @Component @Log4j2 @@ -18,13 +21,20 @@ public class SafeDeleteCommand { private static final String ERROR_FILENAME_PREFIX = "ERROR-"; @Autowired - MediaCubeDatabaseService mediaCubeDatabaseService; + private SafeDeleteProperties safeDeleteProperties; @Autowired - TSMService tsmService; + private MediaCubeDatabaseService mediaCubeDatabaseService; + + @Autowired + private TSMService tsmService; + + private final DateTimeFormatter formatter = DateTimeFormatter.ofPattern("yyyyMMddHHmmss"); public void processDirectory(String directory, boolean forceDeleteArchived) throws IOException { - log.info("Check directory '{} ' and delete already archived is {}", directory, forceDeleteArchived); + final Instant start = Instant.now(); + log.info("Check directory: {}", directory); + log.info("Delete already archived: {}", forceDeleteArchived); Path inputPath = Paths.get(directory); try (DirectoryStream stream = Files.newDirectoryStream(inputPath)) { for (Path filePath : stream) { @@ -32,6 +42,12 @@ public class SafeDeleteCommand { continue; } processInputFile(filePath, forceDeleteArchived); + if (safeDeleteProperties.getMaxExecutionHours() > -1 + && safeDeleteProperties.getMaxExecutionHours() <= Duration.between(start, Instant.now()).toHours() + ) { + log.info("Reached maximum execution duration"); + break; + } } } } @@ -41,10 +57,10 @@ public class SafeDeleteCommand { String fileName = source.getFileName().toString(); try { - log.debug("Check MediaCube metadata for {}", source); + log.info("Check MediaCube metadata for {}", source); mediaCubeDatabaseService.verify(fileName, status); - log.debug("Check TSM file for {}", source); + log.info("Check TSM file for {}", source); tsmService.verify(source, status); if (forceDeleteArchived && status.isFileSizeEquals() && status.isHashEquals()) { diff --git a/mc-intergator/mc-safe-delete/src/main/java/hu/user/mediacube/integration/safedelete/SafeDeleteMainEntry.java b/mc-intergator/mc-safe-delete/src/main/java/hu/user/mediacube/integration/safedelete/SafeDeleteMainEntry.java index 9d1b5520..fce8b340 100644 --- a/mc-intergator/mc-safe-delete/src/main/java/hu/user/mediacube/integration/safedelete/SafeDeleteMainEntry.java +++ b/mc-intergator/mc-safe-delete/src/main/java/hu/user/mediacube/integration/safedelete/SafeDeleteMainEntry.java @@ -1,22 +1,21 @@ package hu.user.mediacube.integration.safedelete; import lombok.extern.log4j.Log4j2; -import org.apache.commons.cli.CommandLine; -import org.apache.commons.cli.CommandLineParser; -import org.apache.commons.cli.DefaultParser; -import org.apache.commons.cli.Options; +import org.apache.commons.cli.*; import org.mybatis.spring.annotation.MapperScan; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.boot.CommandLineRunner; import org.springframework.boot.SpringApplication; import org.springframework.boot.autoconfigure.SpringBootApplication; +import org.springframework.context.annotation.ComponentScan; @Log4j2 -@SpringBootApplication() +@SpringBootApplication +@ComponentScan({"hu.user.mediacube", "user.tsm.client"}) @MapperScan({"hu.user.mediacube.rdb", "hu.user.mediacube.integration.safedelete.db"}) public class SafeDeleteMainEntry implements CommandLineRunner { @Autowired - SafeDeleteCommand safeDeleteCommand; + private SafeDeleteCommand safeDeleteCommand; public static void main(String[] args) { SpringApplication.run(SafeDeleteMainEntry.class, args); @@ -26,18 +25,22 @@ public class SafeDeleteMainEntry implements CommandLineRunner { public void run(String... args) throws Exception { Options options = new Options(); options.addRequiredOption("d", "directory", true, "source directory"); - options.addOption("f", "force-delete", false, "force delete archived"); + options.addOption("f", "force-delete", false, "force delete archived file"); + options.addOption("h", "help", false, "show help"); CommandLineParser parser = new DefaultParser(); - CommandLine cmd = null; + CommandLine cmd; try { - cmd = parser.parse(options, args); safeDeleteCommand.processDirectory(cmd.getOptionValue("d"), cmd.hasOption("f")); } catch (Exception e) { log.error(e.getMessage()); - log.info("Usage: mc-safe-delete -d 'directory path' -f true/false"); + showHelp(options); } - //if (cmd.hasOption("d") + } + + private static void showHelp(Options options) { + HelpFormatter formatter = new HelpFormatter(); + formatter.printHelp("mc-safe-delete", options, true); } } diff --git a/mc-intergator/mc-safe-delete/src/main/java/hu/user/mediacube/integration/safedelete/SafeDeleteProperties.java b/mc-intergator/mc-safe-delete/src/main/java/hu/user/mediacube/integration/safedelete/SafeDeleteProperties.java index dec25000..1d4dfa70 100644 --- a/mc-intergator/mc-safe-delete/src/main/java/hu/user/mediacube/integration/safedelete/SafeDeleteProperties.java +++ b/mc-intergator/mc-safe-delete/src/main/java/hu/user/mediacube/integration/safedelete/SafeDeleteProperties.java @@ -22,6 +22,8 @@ public class SafeDeleteProperties { private String password; - private int port; + private int bufferSize; + + private int maxExecutionHours; } diff --git a/mc-intergator/mc-safe-delete/src/main/java/hu/user/mediacube/integration/safedelete/verifier/MediaCubeDatabaseService.java b/mc-intergator/mc-safe-delete/src/main/java/hu/user/mediacube/integration/safedelete/verifier/MediaCubeDatabaseService.java index 073a5b40..1741e18f 100644 --- a/mc-intergator/mc-safe-delete/src/main/java/hu/user/mediacube/integration/safedelete/verifier/MediaCubeDatabaseService.java +++ b/mc-intergator/mc-safe-delete/src/main/java/hu/user/mediacube/integration/safedelete/verifier/MediaCubeDatabaseService.java @@ -20,6 +20,7 @@ public class MediaCubeDatabaseService { if (dbRecords.size() != 1) { throw new Exception(String.format("Database record count mismatch. Expected 1 found %d", dbRecords.size())); } + log.info("Metadata check passed"); status.setMetadataExists(true); } diff --git a/mc-intergator/mc-safe-delete/src/main/java/hu/user/mediacube/integration/safedelete/verifier/TSMService.java b/mc-intergator/mc-safe-delete/src/main/java/hu/user/mediacube/integration/safedelete/verifier/TSMService.java index 17eaccca..eed9471c 100644 --- a/mc-intergator/mc-safe-delete/src/main/java/hu/user/mediacube/integration/safedelete/verifier/TSMService.java +++ b/mc-intergator/mc-safe-delete/src/main/java/hu/user/mediacube/integration/safedelete/verifier/TSMService.java @@ -17,6 +17,7 @@ import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.Paths; import java.util.Objects; +import java.util.UUID; @Service @Log4j2 @@ -26,24 +27,25 @@ public class TSMService { private static final String TMP_PREFIX = "TSM-RESTORED-"; - private Path restore(String fileName) throws Exception { + private Path restore(Path source) throws Exception { Path result = null; TSMClient server = null; try { server = new TSMClient(safeDeleteProperties.getNodeName()); server.connect(safeDeleteProperties.getUser(), safeDeleteProperties.getPassword()); - server.setBufferSize(safeDeleteProperties.getPort()); + server.setBufferSize(safeDeleteProperties.getBufferSize()); TSMBackupFileObject backupFileObject = server.getActiveBackupFileObject(safeDeleteProperties.getFsName(), - safeDeleteProperties.getHlName(), safeDeleteProperties.getDelimiter() + fileName); + safeDeleteProperties.getHlName(), safeDeleteProperties.getDelimiter() + source.getFileName().toString()); if (backupFileObject != null) { - Path tempDirectory = Files.createTempDirectory(String.format("%s-%s", TMP_PREFIX, fileName)); - backupFileObject.setLocalFilePath(tempDirectory.toAbsolutePath().toString()); + result = Paths.get(source.getParent().toAbsolutePath().toString(), + source.getFileName().toString() + UUID.randomUUID()); + backupFileObject.setLocalFilePath(result.toAbsolutePath().toString()); server.recieve(backupFileObject); - result = Paths.get(tempDirectory.toAbsolutePath().toString(), fileName); if (!result.toFile().exists()) { throw new Exception(String.format("Restored file not exists: %s ", result)); } + log.info("TSM file successfully restored to: {}", result); } } catch (Exception e) { @@ -59,12 +61,13 @@ public class TSMService { public void verify(Path source, ArchiveFileStatus status) throws Exception { Path restored = null; try { - restored = restore(source.getFileName().toString()); + restored = restore(source); long restoredLength = restored.toFile().length(); long originalLength = source.toFile().length(); if (originalLength != restoredLength) { throw new Exception(String.format("File size mismatch. Expected %d, found %d", originalLength, restoredLength)); } + log.info("File size check passed: {}", originalLength); status.setFileSizeEquals(true); String originalMD5 = createMD5Hash(source); @@ -72,13 +75,13 @@ public class TSMService { if (!StringUtils.equals(originalMD5, restoredMD5)) { throw new Exception(String.format("File MD5 hash mismatch. Expected %d, found %d", originalMD5, restoredMD5)); } + log.info("MD5 hash check passed: {}", originalMD5); status.setHashEquals(true); } catch (Exception e) { throw e; } finally { if (Objects.nonNull(restored)) { FileOperations.silentDelete(restored); - FileOperations.silentDelete(restored.getParent()); } } } diff --git a/mc-intergator/mc-safe-delete/src/main/resources/application.yaml b/mc-intergator/mc-safe-delete/src/main/resources/application.yaml index 61c3f074..86dbc4a3 100644 --- a/mc-intergator/mc-safe-delete/src/main/resources/application.yaml +++ b/mc-intergator/mc-safe-delete/src/main/resources/application.yaml @@ -5,19 +5,19 @@ spring: ansi: enabled: always datasource: - url: jdbc:db2://10.228.212.42:50000/mc:retrieveMessagesFromServerOnGetMessage=true; + url: jdbc:db2://10.10.1.27:50000/mc:retrieveMessagesFromServerOnGetMessage=true; username: db2admin password: password driver-class-name: com.ibm.db2.jcc.DB2Driver type: com.zaxxer.hikari.HikariDataSource hikari: pool-name: pool-mediacube - maximum-pool-size: 50 - minimum-idle: 10 + maximum-pool-size: 5 + minimum-idle: 1 tsm: - user: support - password: usrkft1q2 - port: 65536 + user: mediacube + password: password + buffer-size: 65536 delimiter: / node-name: mc-safe-delete fs-name: /JOBENGINE diff --git a/mc-intergator/mc-safe-delete/target/classes/application.yaml b/mc-intergator/mc-safe-delete/target/classes/application.yaml index 9363773e..86dbc4a3 100644 --- a/mc-intergator/mc-safe-delete/target/classes/application.yaml +++ b/mc-intergator/mc-safe-delete/target/classes/application.yaml @@ -5,15 +5,20 @@ spring: ansi: enabled: always datasource: - url: jdbc:db2://10.228.212.42:50000/mc:retrieveMessagesFromServerOnGetMessage=true; - username: blobtest - password: blobtest + url: jdbc:db2://10.10.1.27:50000/mc:retrieveMessagesFromServerOnGetMessage=true; + username: db2admin + password: password driver-class-name: com.ibm.db2.jcc.DB2Driver type: com.zaxxer.hikari.HikariDataSource hikari: pool-name: pool-mediacube - maximum-pool-size: 50 - minimum-idle: 10 + maximum-pool-size: 5 + minimum-idle: 1 tsm: - executable: c:\program files\tivoli\client\ba\bin\dsmadmc.exe - args: -id=support -password=userkft -TABdelimited "select LL_NAME from backups where NODE_NAME='PASANODE'" + user: mediacube + password: password + buffer-size: 65536 + delimiter: / + node-name: mc-safe-delete + fs-name: /JOBENGINE + hl-name: /JOBENGINE diff --git a/mc-intergator/mc-safe-delete/target/dumper-mc-0.0.1-SNAPSHOT.jar b/mc-intergator/mc-safe-delete/target/dumper-mc-0.0.1-SNAPSHOT.jar deleted file mode 100644 index 8810c3e6..00000000 Binary files a/mc-intergator/mc-safe-delete/target/dumper-mc-0.0.1-SNAPSHOT.jar and /dev/null differ diff --git a/mc-intergator/mc-safe-delete/target/dumper-mc-0.0.1-SNAPSHOT.jar.original b/mc-intergator/mc-safe-delete/target/dumper-mc-0.0.1-SNAPSHOT.jar.original deleted file mode 100644 index 996e0b37..00000000 Binary files a/mc-intergator/mc-safe-delete/target/dumper-mc-0.0.1-SNAPSHOT.jar.original and /dev/null differ diff --git a/mc-intergator/mc-safe-delete/target/maven-status/maven-compiler-plugin/compile/default-compile/inputFiles.lst b/mc-intergator/mc-safe-delete/target/maven-status/maven-compiler-plugin/compile/default-compile/inputFiles.lst index b764c0f3..d7a812e5 100644 --- a/mc-intergator/mc-safe-delete/target/maven-status/maven-compiler-plugin/compile/default-compile/inputFiles.lst +++ b/mc-intergator/mc-safe-delete/target/maven-status/maven-compiler-plugin/compile/default-compile/inputFiles.lst @@ -1,7 +1,9 @@ C:\work\user\mediacube\mc-intergator\mc-safe-delete\src\main\java\hu\user\mediacube\integration\safedelete\SafeDeleteMainEntry.java +C:\work\user\mediacube\mc-intergator\mc-safe-delete\src\main\java\hu\user\mediacube\integration\safedelete\SafeDeleteProperties.java C:\work\user\mediacube\mc-intergator\mc-safe-delete\src\main\java\hu\user\mediacube\integration\safedelete\ArchiveFileStatus.java C:\work\user\mediacube\mc-intergator\mc-safe-delete\src\main\java\hu\user\mediacube\integration\safedelete\db\MediaCubeRecord.java C:\work\user\mediacube\mc-intergator\mc-safe-delete\src\main\java\hu\user\mediacube\integration\safedelete\db\MediaCubeRecordMapper.java C:\work\user\mediacube\mc-intergator\mc-safe-delete\src\main\java\hu\user\mediacube\integration\safedelete\verifier\MediaCubeDatabaseService.java C:\work\user\mediacube\mc-intergator\mc-safe-delete\src\main\java\hu\user\mediacube\integration\safedelete\verifier\TSMService.java +C:\work\user\mediacube\mc-intergator\mc-safe-delete\src\main\java\hu\user\mediacube\integration\safedelete\FileOperations.java C:\work\user\mediacube\mc-intergator\mc-safe-delete\src\main\java\hu\user\mediacube\integration\safedelete\SafeDeleteCommand.java diff --git a/mc-intergator/pom.xml b/mc-intergator/pom.xml index 4a52436b..ee7d33df 100644 --- a/mc-intergator/pom.xml +++ b/mc-intergator/pom.xml @@ -18,8 +18,8 @@ - mc-db tsm-clientapi + mc-db integration-amc dumper-mc mc-safe-delete diff --git a/mc-intergator/spring-shell.log b/mc-intergator/spring-shell.log deleted file mode 100644 index e69de29b..00000000 diff --git a/mc-intergator/tsm-clientapi/pom.xml b/mc-intergator/tsm-clientapi/pom.xml index 00e6de3c..15102f51 100644 --- a/mc-intergator/tsm-clientapi/pom.xml +++ b/mc-intergator/tsm-clientapi/pom.xml @@ -6,41 +6,31 @@ hu.user.tsm tsm-clientapi - 1.0-SNAPSHOT + 1.0.0-SNAPSHOT hu.user.mediacube mc-integrator 0.0.1-SNAPSHOT - - - org.springframework.boot - spring-boot-autoconfigure - - - org.springframework.boot - spring-boot-starter-actuator - - - - - - org.springframework.boot - spring-boot-maven-plugin - - - org.apache.maven.plugins - maven-jar-plugin - - - - development - ${project.url} - value - - - - - - + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/mc-intergator/tsm-clientapi/src/main/java/user/tsm/client/Main.java b/mc-intergator/tsm-clientapi/src/main/java/user/tsm/client/Main.java deleted file mode 100644 index 4172bdc6..00000000 --- a/mc-intergator/tsm-clientapi/src/main/java/user/tsm/client/Main.java +++ /dev/null @@ -1,57 +0,0 @@ -package user.tsm.client; - -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import org.springframework.beans.factory.annotation.Value; -import org.springframework.boot.CommandLineRunner; -import org.springframework.boot.SpringApplication; -import org.springframework.boot.autoconfigure.SpringBootApplication; -import org.springframework.context.ApplicationContext; - -@SpringBootApplication() -public class Main implements CommandLineRunner { - private static final Logger logger = LoggerFactory.getLogger("TSMClient"); - @Value("${tsm.hlname}") - private String hlname; - @Value("${tsm.separator}") - private String separator; - - public static void main(String[] args) { - ApplicationContext ctx = SpringApplication.run(Main.class, args); - } - - @Override - public void run(String... args) throws Exception { - TSMClient server = null; - - for (int i = 0; i < 100; i++) { - try { - logger.info("Try connect"); - server = new TSMClient("PASANODE"); - server.connect(null, null); - logger.info("Connected"); - TSMBackupFileObject[] result = null; - result = server.queryActiveBackupObjects("NEXIO", hlname, "\\" + args[0]); - if (result != null) { - logger.info("Active objects count {}", result.length); - for (TSMBackupFileObject o : result) { - logger.info("Size {}", o.getSizeEstimate()); - } - } -// result = server.queryInActiveBackupObjects("NEXIO", hlname, "\\" + args[0]); -// if (result != null) { -// logger.info("Inactive objects count {}", result.length); -// for (TSMBackupFileObject o : result) { -// logger.info("Size {}", o.getSizeEstimate()); -// } -// } - } catch (Exception e) { - logger.error("Message: {}", e.getMessage()); - } finally { - if ((server != null) && (server.isConnected())) - server.disconnect(); - } - } - - } -}