From dfe0dba39e17d62846646fb4028cf8a776191c7a Mon Sep 17 00:00:00 2001 From: "vasary.daniel" Date: Thu, 5 May 2022 07:54:51 +0000 Subject: [PATCH] git-tfs-id: [http://tfs.userrendszerhaz.hu:8080/tfs/DefaultCollection]$/MediaCube;C33292 --- .../production/HIRTV/jobs/executors.xml | 2 +- .../production/HIRTV/jobs/schedules.json | 10 + .../ProjectCleanupMountedLocationStep.java | 235 ++++++++++++++++++ .../HIRTV/jobs/steps/shared/MetadataType.java | 5 + .../steps/shared/MetadataTypeDetector.java | 49 ++++ .../jobs/templates/delete-promo-materials.xml | 2 +- .../HIRTV/settings/application.yaml | 4 +- .../tests/DynamicStepsLoaderTests.java | 20 +- .../ProjectCleanupMountedLocationStep.java | 30 +-- 9 files changed, 330 insertions(+), 27 deletions(-) create mode 100644 server/-product/production/HIRTV/jobs/steps/ProjectCleanupMountedLocationStep.java create mode 100644 server/-product/production/HIRTV/jobs/steps/shared/MetadataType.java create mode 100644 server/-product/production/HIRTV/jobs/steps/shared/MetadataTypeDetector.java diff --git a/server/-product/production/HIRTV/jobs/executors.xml b/server/-product/production/HIRTV/jobs/executors.xml index e7efeaa6..b22f5689 100644 --- a/server/-product/production/HIRTV/jobs/executors.xml +++ b/server/-product/production/HIRTV/jobs/executors.xml @@ -16,7 +16,7 @@ - + diff --git a/server/-product/production/HIRTV/jobs/schedules.json b/server/-product/production/HIRTV/jobs/schedules.json index d6d68d4e..b31018c7 100644 --- a/server/-product/production/HIRTV/jobs/schedules.json +++ b/server/-product/production/HIRTV/jobs/schedules.json @@ -1,5 +1,15 @@ { "joblist": [ + { + "active": true, + "executeimmediate": false, + "name" : "Lejárt ISISLON/PROMO_NLE mappák törlése", + "template": "delete-promo-materials.xml", + "cronexpression": "0 0 6 * * ?", + "parameters": [ + {"name": "sourcePath", "value": "/mnt/ISILON/PROMO_NLE", "type": "java.lang.String"} + ] + }, { "template": "dummy-test-job.xml", "name": "dummy test job", diff --git a/server/-product/production/HIRTV/jobs/steps/ProjectCleanupMountedLocationStep.java b/server/-product/production/HIRTV/jobs/steps/ProjectCleanupMountedLocationStep.java new file mode 100644 index 00000000..ce9c3c09 --- /dev/null +++ b/server/-product/production/HIRTV/jobs/steps/ProjectCleanupMountedLocationStep.java @@ -0,0 +1,235 @@ +package user.jobengine.server.steps; + +import java.io.File; +import java.io.IOException; +import java.nio.file.DirectoryStream; +import java.nio.file.FileVisitResult; +import java.nio.file.FileVisitor; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.nio.file.SimpleFileVisitor; +import java.nio.file.attribute.BasicFileAttributes; +import java.text.ParseException; +import java.text.SimpleDateFormat; +import java.util.ArrayList; +import java.util.Collections; +import java.util.Date; +import java.util.List; + +import org.apache.commons.io.FileUtils; +import org.apache.commons.lang.StringUtils; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.apache.logging.log4j.Marker; + +import user.jobengine.server.IJobEngine; +import user.jobengine.server.IJobRuntime; +import user.jobengine.server.steps.shared.MetadataType; +import user.jobengine.server.steps.shared.MetadataTypeDetector; + +public class ProjectCleanupMountedLocationStep extends JobStep implements FileVisitor { + + private static final Logger logger = LogManager.getLogger(); + private static final String DATEFORMAT = "yyyyMMdd"; + private static final String DOT = "."; + private static final String STATUSFOLDER = ".STATUS"; + private static final String KILLDATEEXT = ".killdate"; + private Marker marker; + final int[] allCount = { 0 }; + final int[] currentCount = { 0 }; + private Path sourcePath; + private SimpleDateFormat dateFormat; + + private Date checkExpiration(List killDateFiles) { + Date killDate = null; + for (Path killDateFile : killDateFiles) { + Date currentKillDate = getKillDate(killDateFile); + if (currentKillDate == null) + continue; + if ((killDate != null && currentKillDate.after(killDate)) || killDate == null) + killDate = currentKillDate; + } + return new Date().after(killDate) ? killDate : null; + } + + @StepEntry + public Object[] execute(String sourceFolder, IJobEngine jobEngine, IJobRuntime jobRuntime) throws Exception { + marker = jobRuntime.getSessionMarker(); + sourcePath = Paths.get(sourceFolder); + DirectoryStream directoryStream = null; + if (StringUtils.isBlank(sourcePath.toString())) { + logger.error(marker, "A folyamat 'sourcePath' bemeneti paramétere üres."); + throw new NullPointerException( + "System is not configured properly, 'sourceFolder' input parameter missing."); + } + + if (!sourcePath.toFile().exists() || !sourcePath.toFile().isDirectory()) { + logger.error(marker, "A(z) {} mappa nem létezik.", sourceFolder); + throw new NullPointerException(String.format("Directory %s does not exist.", sourceFolder)); + } + try { + setProgress(1); + dateFormat = new SimpleDateFormat(DATEFORMAT); + Files.walkFileTree(sourcePath, new SimpleFileVisitor() { + @Override + public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) throws IOException { + allCount[0]++; + return super.visitFile(file, attrs); + } + }); + Files.walkFileTree(sourcePath, this); + } catch (Exception e) { + logger.catching(e); + logger.error(marker, "Az '{}' mappa elérése sikertelen. A rendszer hibaüzenete: {}", sourcePath, + e.getMessage()); + throw e; + } finally { + if (directoryStream != null) { + try { + directoryStream.close(); + } catch (IOException e) { + } + } + } + return null; + } + + private Date getKillDate(Path killDateFile) { + String fileName = killDateFile.getFileName().toString(); + int end = fileName.lastIndexOf(DOT); + if (end < 1) + return null; + int start = fileName.lastIndexOf(DOT, end - 1); + if (start < 0) + return null; + String strKillDate = fileName.substring(start + 1, end); + Date result = null; + if (StringUtils.isNumeric(strKillDate)) { + try { + result = dateFormat.parse(strKillDate); + } catch (ParseException e) { + logger.error(marker, + "A {} fájl 'killdate' állománya hibás formátumú, a {} karaktersorozat nem konvertálható dátummá.", + fileName, strKillDate); + return null; + } + } else + logger.error(marker, "A {} fájl 'killdate' állománya hibás formátumú, az dátum helyett ez áll: '{}'.", + fileName, strKillDate); + return result; + } + + private List getKillDateFiles(Path filePath) { + String killDateFilePattern = String.format("%s.*%s", filePath.getFileName().toString(), KILLDATEEXT); + List result = new ArrayList<>(); + Path statusPath = null; + try { + statusPath = Paths.get(filePath.getParent().toString(), STATUSFOLDER); + } catch (Exception e) { + logger.catching(e); + return null; + } + File statusPathFile = statusPath.toFile(); + if (statusPathFile.exists() && statusPathFile.isDirectory()) { + try (DirectoryStream stream = Files.newDirectoryStream(statusPath, killDateFilePattern)) { + stream.forEach(p -> result.add(p)); + } catch (Exception e) { + logger.catching(e); + } + } + Collections.sort(result); + return result; + } + + private Path getProjectRootPath(Path filePath) { + Path result = null; + if (filePath.getNameCount() > sourcePath.getNameCount()) { + String dir = filePath.getName(sourcePath.getNameCount()).toString(); + String[] tokens = dir.split("-"); + if (tokens.length != 0 && MetadataTypeDetector.GuessMetadataType(tokens[0]) == MetadataType.TrafficPromo) + result = Paths.get(sourcePath.toAbsolutePath().toString(), dir); + } + return result; + } + + @Override + public FileVisitResult postVisitDirectory(Path dir, IOException exc) throws IOException { + return FileVisitResult.CONTINUE; + } + + @Override + public FileVisitResult preVisitDirectory(Path dir, BasicFileAttributes attrs) throws IOException { + if (dir.equals(sourcePath)) + return FileVisitResult.CONTINUE; + + Path parent = getProjectRootPath(dir); + if (parent == null) { + System.out.println("Skipping " + dir); + return FileVisitResult.SKIP_SUBTREE; + } + + return FileVisitResult.CONTINUE; + } + + private boolean processPathItem(Path filePath) { + currentCount[0]++; + int progress = currentCount[0] * 100 / allCount[0]; + setProgress(progress); + + if (filePath.toFile().isDirectory() || !filePath.toString().toUpperCase().endsWith(".EZP")) + return false; + + // level check + if (filePath.getNameCount() != sourcePath.getNameCount() + 3) { + // logger.warn(marker, "A {} fájl elérési útja a várttól eltérő.", filePath); + return false; + } + + List killDateFiles = getKillDateFiles(filePath); + if (killDateFiles == null || killDateFiles.size() == 0) { + logger.warn(marker, "A {} fájlhoz nem található 'killdate' állomány.", filePath); + return false; + } + + if (killDateFiles.size() != 1) + logger.warn(marker, + "A {} fájlhoz több 'killdate' állomány található, a legújabb dátum határozza meg a törlés időpontját.", + filePath); + + Date killDate = checkExpiration(killDateFiles); + if (killDate == null) + return false; + + Path parent = getProjectRootPath(filePath); + if (parent != null) { + Path pathToDelete = Paths.get(sourcePath.toAbsolutePath().toString(), + filePath.getName(sourcePath.getNameCount()).toString()); + try { + FileUtils.deleteDirectory(pathToDelete.toFile()); + logger.info(marker, "A {} mappa sikeresen törlődött. Lejárat: {}", pathToDelete, + dateFormat.format(killDate)); + } catch (Exception e) { + logger.error(marker, "A {} mappa törlése nem lehetséges, a rendszer üzenete: {}", pathToDelete, + e.getMessage()); + logger.error(e); + } + } + // else + // logger.warn(marker, "A {} fájlhoz nem határozható meg a törlendő + // szülőkönyvtár.", filePath); + return true; + } + + @Override + public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) throws IOException { + processPathItem(file); + return FileVisitResult.CONTINUE; + } + + @Override + public FileVisitResult visitFileFailed(Path file, IOException e) throws IOException { + logger.error(marker, "A {} fájl nem érhető el. A rendszer hibaüzenete: {}", file.toString(), e.getMessage()); + return FileVisitResult.CONTINUE; + } +} diff --git a/server/-product/production/HIRTV/jobs/steps/shared/MetadataType.java b/server/-product/production/HIRTV/jobs/steps/shared/MetadataType.java new file mode 100644 index 00000000..0e27bf34 --- /dev/null +++ b/server/-product/production/HIRTV/jobs/steps/shared/MetadataType.java @@ -0,0 +1,5 @@ +package user.jobengine.server.steps.shared; + +public enum MetadataType { + TrafficMaterial, TrafficPromo, TrafficAD, OctopusStory, OctopusPlaceholder, Generic +} diff --git a/server/-product/production/HIRTV/jobs/steps/shared/MetadataTypeDetector.java b/server/-product/production/HIRTV/jobs/steps/shared/MetadataTypeDetector.java new file mode 100644 index 00000000..9e9e6be9 --- /dev/null +++ b/server/-product/production/HIRTV/jobs/steps/shared/MetadataTypeDetector.java @@ -0,0 +1,49 @@ +package user.jobengine.server.steps.shared; + +import org.apache.commons.lang.StringUtils; + +public class MetadataTypeDetector { + + private static final String HYPHEN = "-"; + private static final String DOT = "."; + + private static final String REGEXP_TRAFFICMATERIALID = "^M{1}[0-9]{6}[A-Z]{1}"; + private static final String REGEXP_TRAFFICADID = "^R{1}[0-9]{6}[A-Z]{1}"; + private static final String REGEXP_TRAFFICPROMOID = "^P{1}[0-9]{6}[A-Z]{1}"; + private static final String REGEXP_OCTOPUSSTORYID = "^[0-9]+"; + private static final String REGEXP_OCTOPUSPLACEHOLDERID = "^[0-9]+_[0-9]+"; + private static final String REGEXP_OCTOPUSPLACEHOLDERVERSIONEDID = "^[0-9]+_[0-9]+-[0-9]{3}"; + + public static MetadataType GuessMetadataType(String id) { + if (StringUtils.isBlank(id)) + return MetadataType.Generic; + if (id.matches(REGEXP_TRAFFICMATERIALID)) + return MetadataType.TrafficMaterial; + if (id.matches(REGEXP_TRAFFICPROMOID)) + return MetadataType.TrafficPromo; + if (id.matches(REGEXP_TRAFFICADID)) + return MetadataType.TrafficAD; + if (id.matches(REGEXP_OCTOPUSSTORYID)) + return MetadataType.OctopusStory; + if (id.matches(REGEXP_OCTOPUSPLACEHOLDERID)) + return MetadataType.OctopusPlaceholder; + if (id.matches(REGEXP_OCTOPUSPLACEHOLDERVERSIONEDID)) + return MetadataType.OctopusPlaceholder; + return MetadataType.Generic; + } + + public static String truncateExtension(String name) { + String result = name; + if (result != null && result.contains(DOT)) + result = result.substring(0, result.lastIndexOf(DOT)); + return result; + } + + public static String truncateVersion(String name) { + String result = name; + if (result != null && result.contains(HYPHEN)) + result = result.split(HYPHEN)[0]; + return result; + } + +} diff --git a/server/-product/production/HIRTV/jobs/templates/delete-promo-materials.xml b/server/-product/production/HIRTV/jobs/templates/delete-promo-materials.xml index f5df704c..38f5c42a 100644 --- a/server/-product/production/HIRTV/jobs/templates/delete-promo-materials.xml +++ b/server/-product/production/HIRTV/jobs/templates/delete-promo-materials.xml @@ -6,7 +6,7 @@ - + diff --git a/server/-product/production/HIRTV/settings/application.yaml b/server/-product/production/HIRTV/settings/application.yaml index 1acf04bf..60d934f8 100644 --- a/server/-product/production/HIRTV/settings/application.yaml +++ b/server/-product/production/HIRTV/settings/application.yaml @@ -1,6 +1,6 @@ datasource: mediacube: - url: jdbc:db2://10.10.1.27:50000/mc + url: jdbc:db2://localvm:50000/mc user: db2admin password: password external-indexer: false @@ -8,7 +8,7 @@ datasource: login-timeout: 3 #seconds pool-size: 10 mediacube-nosql: - url: jdbc:db2://10.10.1.27:50000/mc + url: jdbc:db2://localvm:50000/mc user: db2admin password: password schema: test diff --git a/server/hu.user.mediacube.executors.tests/src/hu/user/mediacube/executors/tests/DynamicStepsLoaderTests.java b/server/hu.user.mediacube.executors.tests/src/hu/user/mediacube/executors/tests/DynamicStepsLoaderTests.java index 37528a7e..6f3c54b3 100644 --- a/server/hu.user.mediacube.executors.tests/src/hu/user/mediacube/executors/tests/DynamicStepsLoaderTests.java +++ b/server/hu.user.mediacube.executors.tests/src/hu/user/mediacube/executors/tests/DynamicStepsLoaderTests.java @@ -19,8 +19,7 @@ public class DynamicStepsLoaderTests { @BeforeClass static public void beforeClass() { - sut = new DynamicStepsLoader( - "C:/work/user/mediacube/server/user.jobengine.executors/src/user/jobengine/server/steps"); + sut = new DynamicStepsLoader("C:/work/user/mediacube/server/user.jobengine.executors/src/user/jobengine/server/steps"); } @Test @@ -39,11 +38,26 @@ public class DynamicStepsLoaderTests { String name = f.getName().toString(); try { logger.info("Loading {}", name); - sut.loadClassFromSourceCode(classLoader, name); + Class clazz = sut.loadClassFromSourceCode(classLoader, name); + clazz.newInstance(); } catch (Exception e) { logger.error(e.getMessage()); throw e; } } } + + @Test + public void test2() throws Exception { + GroovyClassLoader classLoader = sut.createClassLoader(this.getClass().getClassLoader()); + String name = "ProjectCleanupMountedLocationStep.java"; + try { + logger.info("Loading {}", name); + Class clazz = sut.loadClassFromSourceCode(classLoader, name); + clazz.newInstance(); + } catch (Exception e) { + logger.error(e.getMessage()); + throw e; + } + } } diff --git a/server/user.jobengine.executors/src/user/jobengine/server/steps/ProjectCleanupMountedLocationStep.java b/server/user.jobengine.executors/src/user/jobengine/server/steps/ProjectCleanupMountedLocationStep.java index ce9c3c09..06d1cc65 100644 --- a/server/user.jobengine.executors/src/user/jobengine/server/steps/ProjectCleanupMountedLocationStep.java +++ b/server/user.jobengine.executors/src/user/jobengine/server/steps/ProjectCleanupMountedLocationStep.java @@ -36,8 +36,8 @@ public class ProjectCleanupMountedLocationStep extends JobStep implements FileVi private static final String STATUSFOLDER = ".STATUS"; private static final String KILLDATEEXT = ".killdate"; private Marker marker; - final int[] allCount = { 0 }; - final int[] currentCount = { 0 }; + final int[] allCount = new int[] { 0 }; + final int[] currentCount = new int[] { 0 }; private Path sourcePath; private SimpleDateFormat dateFormat; @@ -60,8 +60,7 @@ public class ProjectCleanupMountedLocationStep extends JobStep implements FileVi DirectoryStream directoryStream = null; if (StringUtils.isBlank(sourcePath.toString())) { logger.error(marker, "A folyamat 'sourcePath' bemeneti paramétere üres."); - throw new NullPointerException( - "System is not configured properly, 'sourceFolder' input parameter missing."); + throw new NullPointerException("System is not configured properly, 'sourceFolder' input parameter missing."); } if (!sourcePath.toFile().exists() || !sourcePath.toFile().isDirectory()) { @@ -81,8 +80,7 @@ public class ProjectCleanupMountedLocationStep extends JobStep implements FileVi Files.walkFileTree(sourcePath, this); } catch (Exception e) { logger.catching(e); - logger.error(marker, "Az '{}' mappa elérése sikertelen. A rendszer hibaüzenete: {}", sourcePath, - e.getMessage()); + logger.error(marker, "Az '{}' mappa elérése sikertelen. A rendszer hibaüzenete: {}", sourcePath, e.getMessage()); throw e; } finally { if (directoryStream != null) { @@ -109,14 +107,11 @@ public class ProjectCleanupMountedLocationStep extends JobStep implements FileVi try { result = dateFormat.parse(strKillDate); } catch (ParseException e) { - logger.error(marker, - "A {} fájl 'killdate' állománya hibás formátumú, a {} karaktersorozat nem konvertálható dátummá.", - fileName, strKillDate); + logger.error(marker, "A {} fájl 'killdate' állománya hibás formátumú, a {} karaktersorozat nem konvertálható dátummá.", fileName, strKillDate); return null; } } else - logger.error(marker, "A {} fájl 'killdate' állománya hibás formátumú, az dátum helyett ez áll: '{}'.", - fileName, strKillDate); + logger.error(marker, "A {} fájl 'killdate' állománya hibás formátumú, az dátum helyett ez áll: '{}'.", fileName, strKillDate); return result; } @@ -193,9 +188,7 @@ public class ProjectCleanupMountedLocationStep extends JobStep implements FileVi } if (killDateFiles.size() != 1) - logger.warn(marker, - "A {} fájlhoz több 'killdate' állomány található, a legújabb dátum határozza meg a törlés időpontját.", - filePath); + logger.warn(marker, "A {} fájlhoz több 'killdate' állomány található, a legújabb dátum határozza meg a törlés időpontját.", filePath); Date killDate = checkExpiration(killDateFiles); if (killDate == null) @@ -203,15 +196,12 @@ public class ProjectCleanupMountedLocationStep extends JobStep implements FileVi Path parent = getProjectRootPath(filePath); if (parent != null) { - Path pathToDelete = Paths.get(sourcePath.toAbsolutePath().toString(), - filePath.getName(sourcePath.getNameCount()).toString()); + Path pathToDelete = Paths.get(sourcePath.toAbsolutePath().toString(), filePath.getName(sourcePath.getNameCount()).toString()); try { FileUtils.deleteDirectory(pathToDelete.toFile()); - logger.info(marker, "A {} mappa sikeresen törlődött. Lejárat: {}", pathToDelete, - dateFormat.format(killDate)); + logger.info(marker, "A {} mappa sikeresen törlődött. Lejárat: {}", pathToDelete, dateFormat.format(killDate)); } catch (Exception e) { - logger.error(marker, "A {} mappa törlése nem lehetséges, a rendszer üzenete: {}", pathToDelete, - e.getMessage()); + logger.error(marker, "A {} mappa törlése nem lehetséges, a rendszer üzenete: {}", pathToDelete, e.getMessage()); logger.error(e); } } -- 2.54.0