</plugins>
</build>
<dependencies>
- <!-- <dependency>-->
- <!-- <groupId>org.springframework.data</groupId>-->
- <!-- <artifactId>spring-data-mongodb-parent</artifactId>-->
- <!-- <version>${spring.version}</version>-->
- <!-- <type>pom</type>-->
- <!-- </dependency>-->
- <!-- <dependency>-->
- <!-- <groupId>org.springframework.data</groupId>-->
- <!-- <artifactId>spring-data-mongodb</artifactId>-->
- <!-- </dependency>-->
- <dependency>
- <groupId>org.mybatis.spring.boot</groupId>
- <artifactId>mybatis-spring-boot-starter</artifactId>
- <version>2.1.1</version>
- </dependency>
<dependency>
<groupId>com.ibm.db2</groupId>
<artifactId>jcc</artifactId>
<version>11.5.8.0</version>
</dependency>
-
+ <dependency>
+ <groupId>org.mybatis.spring.boot</groupId>
+ <artifactId>mybatis-spring-boot-starter</artifactId>
+ <version>2.1.1</version>
+ </dependency>
<dependency>
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-starter-test</artifactId>
</exclusion>
</exclusions>
</dependency>
-
</dependencies>
-
</project>
\ No newline at end of file
hu\user\mediacube\rdb\model\StoreExample$Criterion.class
hu\user\mediacube\rdb\model\StoreExample$Criteria.class
hu\user\mediacube\rdb\model\StoreuriExample$Criteria.class
+hu\user\mediacube\rdb\MediaCubeMapper.class
hu\user\mediacube\rdb\model\FiletypeExample$GeneratedCriteria.class
hu\user\mediacube\rdb\model\Mediafile.class
hu\user\mediacube\rdb\model\MediaExample$Criterion.class
<version>0.0.1-SNAPSHOT</version>
</parent>
<dependencies>
+ <dependency>
+ <groupId>hu.user.tsm</groupId>
+ <artifactId>tsm-clientapi</artifactId>
+ <version>1.0.0-SNAPSHOT</version>
+ </dependency>
<dependency>
<groupId>hu.user.mediacube</groupId>
<artifactId>mc-db</artifactId>
<version>0.0.1-SNAPSHOT</version>
</dependency>
- <dependency>
- <groupId>hu.user.tsm</groupId>
- <artifactId>tsm-clientapi</artifactId>
- <version>1.0-SNAPSHOT</version>
- </dependency>
<dependency>
<groupId>commons-codec</groupId>
<artifactId>commons-codec</artifactId>
@Setter
@Builder
public class ArchiveFileStatus {
+
private String name;
private boolean metadataExists;
static public void silentDelete(Path path) {
try {
Files.delete(path);
+ log.info("File deleted: {}", path);
} catch (Exception e) {
log.catching(e);
}
String fileName = String.format("%s%s", prefix, source.getFileName().toString());
Path target = Paths.get(source.getParent().toAbsolutePath().toString(), fileName);
Files.move(source, target);
+ log.info("File renamed to: {}", target);
} catch (Exception e) {
log.catching(e);
}
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
+import java.time.Duration;
+import java.time.Instant;
+import java.time.format.DateTimeFormatter;
@Component
@Log4j2
private static final String ERROR_FILENAME_PREFIX = "ERROR-";
@Autowired
- MediaCubeDatabaseService mediaCubeDatabaseService;
+ private SafeDeleteProperties safeDeleteProperties;
@Autowired
- TSMService tsmService;
+ private MediaCubeDatabaseService mediaCubeDatabaseService;
+
+ @Autowired
+ private TSMService tsmService;
+
+ private final DateTimeFormatter formatter = DateTimeFormatter.ofPattern("yyyyMMddHHmmss");
public void processDirectory(String directory, boolean forceDeleteArchived) throws IOException {
- log.info("Check directory '{} ' and delete already archived is {}", directory, forceDeleteArchived);
+ final Instant start = Instant.now();
+ log.info("Check directory: {}", directory);
+ log.info("Delete already archived: {}", forceDeleteArchived);
Path inputPath = Paths.get(directory);
try (DirectoryStream<Path> stream = Files.newDirectoryStream(inputPath)) {
for (Path filePath : stream) {
continue;
}
processInputFile(filePath, forceDeleteArchived);
+ if (safeDeleteProperties.getMaxExecutionHours() > -1
+ && safeDeleteProperties.getMaxExecutionHours() <= Duration.between(start, Instant.now()).toHours()
+ ) {
+ log.info("Reached maximum execution duration");
+ break;
+ }
}
}
}
String fileName = source.getFileName().toString();
try {
- log.debug("Check MediaCube metadata for {}", source);
+ log.info("Check MediaCube metadata for {}", source);
mediaCubeDatabaseService.verify(fileName, status);
- log.debug("Check TSM file for {}", source);
+ log.info("Check TSM file for {}", source);
tsmService.verify(source, status);
if (forceDeleteArchived && status.isFileSizeEquals() && status.isHashEquals()) {
package hu.user.mediacube.integration.safedelete;
import lombok.extern.log4j.Log4j2;
-import org.apache.commons.cli.CommandLine;
-import org.apache.commons.cli.CommandLineParser;
-import org.apache.commons.cli.DefaultParser;
-import org.apache.commons.cli.Options;
+import org.apache.commons.cli.*;
import org.mybatis.spring.annotation.MapperScan;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.boot.CommandLineRunner;
import org.springframework.boot.SpringApplication;
import org.springframework.boot.autoconfigure.SpringBootApplication;
+import org.springframework.context.annotation.ComponentScan;
@Log4j2
-@SpringBootApplication()
+@SpringBootApplication
+@ComponentScan({"hu.user.mediacube", "user.tsm.client"})
@MapperScan({"hu.user.mediacube.rdb", "hu.user.mediacube.integration.safedelete.db"})
public class SafeDeleteMainEntry implements CommandLineRunner {
@Autowired
- SafeDeleteCommand safeDeleteCommand;
+ private SafeDeleteCommand safeDeleteCommand;
public static void main(String[] args) {
SpringApplication.run(SafeDeleteMainEntry.class, args);
public void run(String... args) throws Exception {
Options options = new Options();
options.addRequiredOption("d", "directory", true, "source directory");
- options.addOption("f", "force-delete", false, "force delete archived");
+ options.addOption("f", "force-delete", false, "force delete archived file");
+ options.addOption("h", "help", false, "show help");
CommandLineParser parser = new DefaultParser();
- CommandLine cmd = null;
+ CommandLine cmd;
try {
-
cmd = parser.parse(options, args);
safeDeleteCommand.processDirectory(cmd.getOptionValue("d"), cmd.hasOption("f"));
} catch (Exception e) {
log.error(e.getMessage());
- log.info("Usage: mc-safe-delete -d 'directory path' -f true/false");
+ showHelp(options);
}
- //if (cmd.hasOption("d")
+ }
+
+ private static void showHelp(Options options) {
+ HelpFormatter formatter = new HelpFormatter();
+ formatter.printHelp("mc-safe-delete", options, true);
}
}
private String password;
- private int port;
+ private int bufferSize;
+
+ private int maxExecutionHours;
}
if (dbRecords.size() != 1) {
throw new Exception(String.format("Database record count mismatch. Expected 1 found %d", dbRecords.size()));
}
+ log.info("Metadata check passed");
status.setMetadataExists(true);
}
import java.nio.file.Path;
import java.nio.file.Paths;
import java.util.Objects;
+import java.util.UUID;
@Service
@Log4j2
private static final String TMP_PREFIX = "TSM-RESTORED-";
- private Path restore(String fileName) throws Exception {
+ private Path restore(Path source) throws Exception {
Path result = null;
TSMClient server = null;
try {
server = new TSMClient(safeDeleteProperties.getNodeName());
server.connect(safeDeleteProperties.getUser(), safeDeleteProperties.getPassword());
- server.setBufferSize(safeDeleteProperties.getPort());
+ server.setBufferSize(safeDeleteProperties.getBufferSize());
TSMBackupFileObject backupFileObject = server.getActiveBackupFileObject(safeDeleteProperties.getFsName(),
- safeDeleteProperties.getHlName(), safeDeleteProperties.getDelimiter() + fileName);
+ safeDeleteProperties.getHlName(), safeDeleteProperties.getDelimiter() + source.getFileName().toString());
if (backupFileObject != null) {
- Path tempDirectory = Files.createTempDirectory(String.format("%s-%s", TMP_PREFIX, fileName));
- backupFileObject.setLocalFilePath(tempDirectory.toAbsolutePath().toString());
+ result = Paths.get(source.getParent().toAbsolutePath().toString(),
+ source.getFileName().toString() + UUID.randomUUID());
+ backupFileObject.setLocalFilePath(result.toAbsolutePath().toString());
server.recieve(backupFileObject);
- result = Paths.get(tempDirectory.toAbsolutePath().toString(), fileName);
if (!result.toFile().exists()) {
throw new Exception(String.format("Restored file not exists: %s ", result));
}
+ log.info("TSM file successfully restored to: {}", result);
}
} catch (Exception e) {
public void verify(Path source, ArchiveFileStatus status) throws Exception {
Path restored = null;
try {
- restored = restore(source.getFileName().toString());
+ restored = restore(source);
long restoredLength = restored.toFile().length();
long originalLength = source.toFile().length();
if (originalLength != restoredLength) {
throw new Exception(String.format("File size mismatch. Expected %d, found %d", originalLength, restoredLength));
}
+ log.info("File size check passed: {}", originalLength);
status.setFileSizeEquals(true);
String originalMD5 = createMD5Hash(source);
if (!StringUtils.equals(originalMD5, restoredMD5)) {
throw new Exception(String.format("File MD5 hash mismatch. Expected %d, found %d", originalMD5, restoredMD5));
}
+ log.info("MD5 hash check passed: {}", originalMD5);
status.setHashEquals(true);
} catch (Exception e) {
throw e;
} finally {
if (Objects.nonNull(restored)) {
FileOperations.silentDelete(restored);
- FileOperations.silentDelete(restored.getParent());
}
}
}
ansi:
enabled: always
datasource:
- url: jdbc:db2://10.228.212.42:50000/mc:retrieveMessagesFromServerOnGetMessage=true;
+ url: jdbc:db2://10.10.1.27:50000/mc:retrieveMessagesFromServerOnGetMessage=true;
username: db2admin
password: password
driver-class-name: com.ibm.db2.jcc.DB2Driver
type: com.zaxxer.hikari.HikariDataSource
hikari:
pool-name: pool-mediacube
- maximum-pool-size: 50
- minimum-idle: 10
+ maximum-pool-size: 5
+ minimum-idle: 1
tsm:
- user: support
- password: usrkft1q2
- port: 65536
+ user: mediacube
+ password: password
+ buffer-size: 65536
delimiter: /
node-name: mc-safe-delete
fs-name: /JOBENGINE
ansi:
enabled: always
datasource:
- url: jdbc:db2://10.228.212.42:50000/mc:retrieveMessagesFromServerOnGetMessage=true;
- username: blobtest
- password: blobtest
+ url: jdbc:db2://10.10.1.27:50000/mc:retrieveMessagesFromServerOnGetMessage=true;
+ username: db2admin
+ password: password
driver-class-name: com.ibm.db2.jcc.DB2Driver
type: com.zaxxer.hikari.HikariDataSource
hikari:
pool-name: pool-mediacube
- maximum-pool-size: 50
- minimum-idle: 10
+ maximum-pool-size: 5
+ minimum-idle: 1
tsm:
- executable: c:\program files\tivoli\client\ba\bin\dsmadmc.exe
- args: -id=support -password=userkft -TABdelimited "select LL_NAME from backups where NODE_NAME='PASANODE'"
+ user: mediacube
+ password: password
+ buffer-size: 65536
+ delimiter: /
+ node-name: mc-safe-delete
+ fs-name: /JOBENGINE
+ hl-name: /JOBENGINE
C:\work\user\mediacube\mc-intergator\mc-safe-delete\src\main\java\hu\user\mediacube\integration\safedelete\SafeDeleteMainEntry.java
+C:\work\user\mediacube\mc-intergator\mc-safe-delete\src\main\java\hu\user\mediacube\integration\safedelete\SafeDeleteProperties.java
C:\work\user\mediacube\mc-intergator\mc-safe-delete\src\main\java\hu\user\mediacube\integration\safedelete\ArchiveFileStatus.java
C:\work\user\mediacube\mc-intergator\mc-safe-delete\src\main\java\hu\user\mediacube\integration\safedelete\db\MediaCubeRecord.java
C:\work\user\mediacube\mc-intergator\mc-safe-delete\src\main\java\hu\user\mediacube\integration\safedelete\db\MediaCubeRecordMapper.java
C:\work\user\mediacube\mc-intergator\mc-safe-delete\src\main\java\hu\user\mediacube\integration\safedelete\verifier\MediaCubeDatabaseService.java
C:\work\user\mediacube\mc-intergator\mc-safe-delete\src\main\java\hu\user\mediacube\integration\safedelete\verifier\TSMService.java
+C:\work\user\mediacube\mc-intergator\mc-safe-delete\src\main\java\hu\user\mediacube\integration\safedelete\FileOperations.java
C:\work\user\mediacube\mc-intergator\mc-safe-delete\src\main\java\hu\user\mediacube\integration\safedelete\SafeDeleteCommand.java
<relativePath/>
</parent>
<modules>
- <module>mc-db</module>
<module>tsm-clientapi</module>
+ <module>mc-db</module>
<module>integration-amc</module>
<module>dumper-mc</module>
<module>mc-safe-delete</module>
<groupId>hu.user.tsm</groupId>
<artifactId>tsm-clientapi</artifactId>
- <version>1.0-SNAPSHOT</version>
+ <version>1.0.0-SNAPSHOT</version>
<parent>
<groupId>hu.user.mediacube</groupId>
<artifactId>mc-integrator</artifactId>
<version>0.0.1-SNAPSHOT</version>
</parent>
- <dependencies>
- <dependency>
- <groupId>org.springframework.boot</groupId>
- <artifactId>spring-boot-autoconfigure</artifactId>
- </dependency>
- <dependency>
- <groupId>org.springframework.boot</groupId>
- <artifactId>spring-boot-starter-actuator</artifactId>
- </dependency>
- </dependencies>
- <build>
- <plugins>
- <plugin>
- <groupId>org.springframework.boot</groupId>
- <artifactId>spring-boot-maven-plugin</artifactId>
- </plugin>
- <plugin>
- <groupId>org.apache.maven.plugins</groupId>
- <artifactId>maven-jar-plugin</artifactId>
- <configuration>
- <archive>
- <manifestEntries>
- <mode>development</mode>
- <url>${project.url}</url>
- <key>value</key>
- </manifestEntries>
- </archive>
- </configuration>
- </plugin>
- </plugins>
- </build>
+ <!-- <build>-->
+ <!-- <plugins>-->
+ <!-- <plugin>-->
+ <!-- <groupId>org.springframework.boot</groupId>-->
+ <!-- <artifactId>spring-boot-maven-plugin</artifactId>-->
+ <!-- </plugin>-->
+ <!-- <plugin>-->
+ <!-- <groupId>org.apache.maven.plugins</groupId>-->
+ <!-- <artifactId>maven-jar-plugin</artifactId>-->
+ <!-- <configuration>-->
+ <!-- <archive>-->
+ <!-- <manifestEntries>-->
+ <!-- <mode>development</mode>-->
+ <!-- <url>${project.url}</url>-->
+ <!-- <key>value</key>-->
+ <!-- </manifestEntries>-->
+ <!-- </archive>-->
+ <!-- </configuration>-->
+ <!-- </plugin>-->
+ <!-- </plugins>-->
+ <!-- </build>-->
</project>
\ No newline at end of file
+++ /dev/null
-package user.tsm.client;
-
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
-import org.springframework.beans.factory.annotation.Value;
-import org.springframework.boot.CommandLineRunner;
-import org.springframework.boot.SpringApplication;
-import org.springframework.boot.autoconfigure.SpringBootApplication;
-import org.springframework.context.ApplicationContext;
-
-@SpringBootApplication()
-public class Main implements CommandLineRunner {
- private static final Logger logger = LoggerFactory.getLogger("TSMClient");
- @Value("${tsm.hlname}")
- private String hlname;
- @Value("${tsm.separator}")
- private String separator;
-
- public static void main(String[] args) {
- ApplicationContext ctx = SpringApplication.run(Main.class, args);
- }
-
- @Override
- public void run(String... args) throws Exception {
- TSMClient server = null;
-
- for (int i = 0; i < 100; i++) {
- try {
- logger.info("Try connect");
- server = new TSMClient("PASANODE");
- server.connect(null, null);
- logger.info("Connected");
- TSMBackupFileObject[] result = null;
- result = server.queryActiveBackupObjects("NEXIO", hlname, "\\" + args[0]);
- if (result != null) {
- logger.info("Active objects count {}", result.length);
- for (TSMBackupFileObject o : result) {
- logger.info("Size {}", o.getSizeEstimate());
- }
- }
-// result = server.queryInActiveBackupObjects("NEXIO", hlname, "\\" + args[0]);
-// if (result != null) {
-// logger.info("Inactive objects count {}", result.length);
-// for (TSMBackupFileObject o : result) {
-// logger.info("Size {}", o.getSizeEstimate());
-// }
-// }
- } catch (Exception e) {
- logger.error("Message: {}", e.getMessage());
- } finally {
- if ((server != null) && (server.isConnected()))
- server.disconnect();
- }
- }
-
- }
-}