From: vasary.daniel Date: Tue, 10 Aug 2021 13:51:16 +0000 (+0000) Subject: git-tfs-id: [http://tfs.userrendszerhaz.hu:8080/tfs/DefaultCollection]$/MediaCube... X-Git-Url: http://git.useribm.hu/?a=commitdiff_plain;h=2a737d26153f28d6958919cc6bd431663adac074;p=mediacube.git git-tfs-id: [tfs.userrendszerhaz.hu:8080/tfs/DefaultCollection]$/MediaCube;C32301 --- diff --git a/server/-product/production/MEDIAVIVANTIS/jobs/executors.xml b/server/-product/production/MEDIAVIVANTIS/jobs/executors.xml index 5ec9b71e..22205fbb 100644 --- a/server/-product/production/MEDIAVIVANTIS/jobs/executors.xml +++ b/server/-product/production/MEDIAVIVANTIS/jobs/executors.xml @@ -1,7 +1,12 @@ + + + + + - + diff --git a/server/-product/production/MEDIAVIVANTIS/jobs/schedules.json b/server/-product/production/MEDIAVIVANTIS/jobs/schedules.json index 9167bcfe..f9cae31a 100644 --- a/server/-product/production/MEDIAVIVANTIS/jobs/schedules.json +++ b/server/-product/production/MEDIAVIVANTIS/jobs/schedules.json @@ -1,6 +1,23 @@ {"joblist":[ { + "active": false, + "executeimmediate": false, + "name" : "Rekurzív törlés /mnt/POLC/FINISHED_SHOWS (DEMO)", + "template": "safe-delete-recursive.xml", + "cronexpression": "0 */1 * * * ?", + "parameters": [ + {"name": "sourcePath", "value": "/mnt/POLC/FINISHED_SHOWS", "type": "java.lang.String"}, + ] + }, + { + "template": "integration-test.xml" + }, + { "template": "remote-transcode.xml", + "name" : "Proxy generálás", + "active": true, + "executeimmediate": false, + "cronexpression": "0 */1 * * * ?", "parameters": [ {"name": "remoteServer", "value": "http://10.11.1.111:9080", "type": "java.lang.String"}, {"name": "transcodeTemplate", "value": "create-proxy-ffmpeg.xml", "type": "java.lang.String"}, @@ -11,35 +28,32 @@ ] }, { - "template": "test-fork-cancelable.xml", - "parameters": [ {"name": "itemID", "value": 1, "type": "java.lang.Long"} ] - }, - { - "active": true, - "executeimmediate": true, - "name" : "Proxy generálás", - "template": "sys-recreate-lowres.xml", - "cronexpression": "0 */1 * * * ?", + "template": "create-lowres-ondemand.xml", + "name" : "Manuális proxy generálás", "parameters": [ - {"name": "globalRetrievePath", "value": "file://10.11.1.90/data", "type": "java.lang.String"}, - {"name": "localRetrievePath", "value": "/mediacube/data", "type": "java.lang.String"}, - {"name": "localHiresPath", "value": "/mediacube/data", "type": "java.lang.String" }, - {"name": "globalHiresPath", "value": "\\\\10.11.1.90\\data", "type": "java.lang.String" }, - {"name": "localLowresPath", "value": "/mediacube/data/TRANSCODER_OUT", "type": "java.lang.String" }, - {"name": "transcoderAddress", "value": "http://10.11.1.111:65445/api/json/v1/", "type": "java.lang.String"}, - {"name": "transcoderTemplateName", "value": "MP4", "type": "java.lang.String"} + {"name": "remoteServer", "value": "http://10.11.1.111:9080", "type": "java.lang.String"}, + {"name": "transcodeTemplate", "value": "create-proxy-ffmpeg.xml", "type": "java.lang.String"}, + {"name": "jobName", "value": "Create proxy (slave process)", "type": "java.lang.String"}, + {"name": "profileName", "value": "proxy", "type": "java.lang.String"}, + {"name": "tempStoreName", "value": "LOCAL", "type": "java.lang.String"}, + {"name": "tempStoreProtocol", "value": "LOCAL", "type": "java.lang.String"} ] }, { + "template": "test-fork-cancelable.xml", + "parameters": [ {"name": "itemID", "value": 1, "type": "java.lang.Long"} ] + }, + { "active": false, "executeimmediate": false, - "name" : "Rekurzív archiválás a POLC/FINISHED_SHOWS mappából", + "name" : "Rekurzív archiválás a /mediacube/data/ARCHIVE-TEST", "template": "archive-recursive.xml", "cronexpression": "0 */1 * * * ?", "parameters": [ - {"name": "sourcePath", "value": "/mnt/POLC/FINISHED_SHOWS", "type": "java.lang.String"}, + {"name": "sourcePath", "value": "/mediacube/data/ARCHIVE-TEST", "type": "java.lang.String"}, {"name": "killDateDays", "value": 1, "type": "java.lang.Integer"}, - {"name": "limit", "value": 1, "type": "java.lang.Integer"} + {"name": "limit", "value": 20, "type": "java.lang.Integer"}, + {"name": "disableProxy", "value": true, "type": "java.lang.Boolean"} ] }, { @@ -99,7 +113,8 @@ "template": "delete-materials.xml", "cronexpression": "0 */10 * * * ?", "parameters": [ - {"name": "sourcePath", "value": "/mediacube/data/ARCHIVE", "type": "java.lang.String"} + {"name": "sourcePath", "value": "/mediacube/data/ARCHIVE", "type": "java.lang.String"}, + {"name": "skipArchiveCheck", "value": false, "type": "java.lang.Boolean"} ] }, { @@ -109,7 +124,8 @@ "template": "delete-materials.xml", "cronexpression": "0 */10 * * * ?", "parameters": [ - {"name": "sourcePath", "value": "/mediacube/data/OMARCHIVE", "type": "java.lang.String"} + {"name": "sourcePath", "value": "/mediacube/data/OMARCHIVE", "type": "java.lang.String"}, + {"name": "skipArchiveCheck", "value": false, "type": "java.lang.Boolean"} ] }, { @@ -119,7 +135,8 @@ "template": "delete-materials.xml", "cronexpression": "0 */10 * * * ?", "parameters": [ - {"name": "sourcePath", "value": "/mnt/PROMISE/ARCHIVE", "type": "java.lang.String"} + {"name": "sourcePath", "value": "/mnt/PROMISE/ARCHIVE", "type": "java.lang.String"}, + {"name": "skipArchiveCheck", "value": false, "type": "java.lang.Boolean"} ] }, { @@ -129,7 +146,8 @@ "template": "delete-materials.xml", "cronexpression": "0 * 11 * * ?", "parameters": [ - {"name": "sourcePath", "value": "/mnt/POLC/FINISHED_SHOWS", "type": "java.lang.String"} + {"name": "sourcePath", "value": "/mnt/POLC/FINISHED_SHOWS", "type": "java.lang.String"}, + {"name": "skipArchiveCheck", "value": false, "type": "java.lang.Boolean"} ] }, { @@ -166,20 +184,6 @@ "executeimmediate": false }, { - "template": "create-lowres-ondemand.xml", - "active": false, - "executeimmediate": false, - "parameters": [ - {"name": "globalRetrievePath", "value": "file://10.11.1.90/data", "type": "java.lang.String"}, - {"name": "localRetrievePath", "value": "/mediacube/data", "type": "java.lang.String"}, - {"name": "localHiresPath", "value": "/mediacube/data", "type": "java.lang.String" }, - {"name": "globalHiresPath", "value": "\\\\10.11.1.90\\data", "type": "java.lang.String" }, - {"name": "localLowresPath", "value": "/mediacube/data/TRANSCODER_OUT", "type": "java.lang.String" }, - {"name": "transcoderAddress", "value": "http://10.11.1.111:65445/api/json/v1/", "type": "java.lang.String"}, - {"name": "transcoderTemplateName", "value": "MP4", "type": "java.lang.String"} - ] - }, - { "active": false, "executeimmediate": false, "template": "cancelable.xml", diff --git a/server/-product/production/MEDIAVIVANTIS/jobs/steps/ArchiveMaterialSubmitStep.java b/server/-product/production/MEDIAVIVANTIS/jobs/steps/ArchiveMaterialSubmitStep.java index f11e3a52..c00a3e4c 100644 --- a/server/-product/production/MEDIAVIVANTIS/jobs/steps/ArchiveMaterialSubmitStep.java +++ b/server/-product/production/MEDIAVIVANTIS/jobs/steps/ArchiveMaterialSubmitStep.java @@ -31,9 +31,9 @@ public class ArchiveMaterialSubmitStep extends JobStep { try { IJobRuntime runtime = jobEngine.submit(jobRuntime, null, JOBTEMPLATE, ARCHIVE, ListUtils.asMap(ARCHIVE_ITEM, archiveItem, KILL_DATE_DAYS, killDateDays)); - int progress = (i + 1) * 100 / archiveList.size(); setProgress(progress); + //TODO kivezetni a submit hibaüzenetet if (runtime == null) throw new Exception("Submit returned null runtime"); } catch (Exception e) { diff --git a/server/-product/production/MEDIAVIVANTIS/jobs/steps/ArchiveRecursive.java b/server/-product/production/MEDIAVIVANTIS/jobs/steps/ArchiveRecursive.java index 6a65d5fa..ba6aba2c 100644 --- a/server/-product/production/MEDIAVIVANTIS/jobs/steps/ArchiveRecursive.java +++ b/server/-product/production/MEDIAVIVANTIS/jobs/steps/ArchiveRecursive.java @@ -10,21 +10,27 @@ import java.nio.file.Paths; import java.nio.file.attribute.BasicFileAttributes; import java.text.SimpleDateFormat; import java.util.Arrays; +import java.util.HashSet; import java.util.List; import java.util.Map; +import java.util.Set; import org.apache.commons.io.FilenameUtils; import org.apache.commons.lang.StringUtils; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; +import org.apache.logging.log4j.message.Message; +import org.apache.logging.log4j.message.ParameterizedMessage; import com.ibm.nosql.json.api.BasicDBObject; import user.commons.JobStatus; import user.commons.ListUtils; +import user.commons.MediaCubeMarker; import user.commons.mediatool.MediaInfo; import user.jobengine.server.IJobEngine; import user.jobengine.server.IJobRuntime; +import user.jobengine.server.steps.shared.EscortFiles; import user.mediacube.metadata.interfaces.IMetadata; import user.mediacube.metadata.interfaces.IMetadataProvider; import user.mediacube.metadata.interfaces.IMetadataProviderFactory; @@ -46,10 +52,12 @@ public class ArchiveRecursive extends JobStep implements FileVisitor { private static final String KILL_DATE_DAYS = "killDateDays"; private SimpleDateFormat df = new SimpleDateFormat("yyyy.MM.dd"); - private List skipPathNames = Arrays.asList("!ARCHIVALAS_ALATT", "VASAROLT", EscortFiles.STATUSFOLDER, EscortFiles.CONFLICTFOLDER); + private List skipPathNames = Arrays.asList("!ARCHIVALAS_ALATT", EscortFiles.STATUSFOLDER, + EscortFiles.CONFLICTFOLDER); private int limit; private int submitted; private int killDateDays; + private boolean disableProxy; private boolean canReadMediaInfo(Path mediaFilePath) { boolean result = false; @@ -63,21 +71,32 @@ public class ArchiveRecursive extends JobStep implements FileVisitor { return result; } - private void checkArchiveItem(ArchiveItem archiveItem, Path mediaPath) throws Exception { + private void checkArchiveItem(ArchiveItem archiveItem) throws Exception { if (archiveItem == null) - throw new Exception(mediaPath + " has no metadata specified."); + throw new Exception("No metadata specified."); if (StringUtils.isBlank(archiveItem.getItemHouseId())) - throw new Exception(mediaPath + " has no Item HouseID specified in metadata."); + throw new Exception("No Item HouseID specified in metadata."); if (StringUtils.isBlank(archiveItem.getItemTitle())) - throw new Exception(mediaPath + " has no Item Title specified in metadata."); + throw new Exception("No Item Title specified in metadata."); if (StringUtils.isBlank(archiveItem.getMediaHouseId())) - throw new Exception(mediaPath + " has no Media HouseID specified in metadata."); + throw new Exception("No Media HouseID specified in metadata."); if (StringUtils.isBlank(archiveItem.getMediaTitle())) - throw new Exception(mediaPath + " has no Media Title specified in metadata."); + throw new Exception("No Media Title specified in metadata."); + } + + private String archiveItemJSON(ArchiveItem result) { + BasicDBObject obj = new BasicDBObject(); + obj.put("itemHouseId", result.getItemHouseId()); + obj.put("itemTitle", result.getItemTitle()); + obj.put("mediaHouseId", result.getMediaHouseId()); + obj.put("mediaTitle", result.getMediaTitle()); + obj.put("mediaDescription", result.getMediaDescription()); + obj.put("mediaType", result.getMediaType()); + return obj.toPrettyString(""); } private ArchiveItem createArchiveItem(Path filePath) throws Exception { @@ -87,13 +106,21 @@ public class ArchiveRecursive extends JobStep implements FileVisitor { String mediaHouseId = FilenameUtils.removeExtension(fileName); try { result = getPlanAirMetadata(mediaHouseId); - } catch (Exception e) { - logger.error("PlanAir metadata error", e); - //nem latja a drivert pl. + if (result != null) { + result.setMediaFile(filePath.toString()); + // 210617 proxy keszites tiltasa + result.setDisableProxy(disableProxy); + } + + logger.info(getSessionMarker(), "PlanAir query done for {}", filePath); + } catch (Exception e) { + logger.error(getSessionMarker(), "PlanAir metadata error", e); + // nem latja a drivert pl. throw e; } +//210616 Ha nincs metaadat, nem archivalunk if (result == null) { result = new ArchiveItem(); BasicFileAttributes attr = Files.readAttributes(filePath, BasicFileAttributes.class); @@ -101,31 +128,32 @@ public class ArchiveRecursive extends JobStep implements FileVisitor { result.setItemTitle(filePath.getParent().toString()); result.setMediaHouseId(mediaHouseId); result.setMediaTitle(fileName); - //TODO kesobb kivenni - //result.setMediaDescription("/OMARCHIVE"); + result.setMediaDescription("/ARCHIVE-TEST"); result.setMediaType("Generic"); + result.setMediaFile(filePath.toString()); } - result.setMediaFile(filePath.toString()); return result; } @StepEntry - public Object[] execute(String sourcePath, int killDateDays, int limit) throws Exception { + public Object[] execute(String sourcePath, int killDateDays, int limit, boolean disableProxy) throws Exception { this.killDateDays = killDateDays; this.limit = limit; + this.disableProxy = disableProxy; + logger.info(getSessionMarker(), "Starting in {}", sourcePath); try { if (getJobRuntime().forkPrepare()) { Files.walkFileTree(Paths.get(sourcePath), this); } } catch (Exception e) { - logger.error(getSessionMarker(), "Az '{}' mappa elérése sikertelen. A rendszer hibaüzenete: {}", sourcePath, e.getMessage()); + logger.error(getSessionMarker(), "Az '{}' mappa elérése sikertelen. A rendszer hibaüzenete: {}", sourcePath, + e.getMessage()); } finally { if (submitted > 0) getJobRuntime().forkWaitComplete(); else getJobRuntime().cancelForkPrepare(); - } return null; } @@ -139,7 +167,12 @@ public class ArchiveRecursive extends JobStep implements FileVisitor { List data = null; IMetadataProviderFactory factory = getService(IMetadataProviderFactory.class); + if (factory == null) + logger.info(getSessionMarker(), "IMetadataProviderFactory is null"); + IMetadataProvider planairProvider = factory.getProvider(MetadataProviderType.PLANAIR); + if (planairProvider == null) + logger.info(getSessionMarker(), "IMetadataProvider is null"); ArchiveItem result = null; data = planairProvider.list(opt); @@ -175,36 +208,7 @@ public class ArchiveRecursive extends JobStep implements FileVisitor { if (getManager().isMediaFileExists(sourceFileName)) { EscortFiles.createMediaCatch(mediaPath); result = true; - - // try { - // Path parent = sourcePath.getParent(); - // Path conflictPath = Paths.get(parent.toString(), CONFLICT); - // File folder = conflictPath.toFile(); - // if (!folder.exists() || !folder.isDirectory()) { - // Set perms = PosixFilePermissions.fromString("rwxrwxrwx"); - // FileAttribute> attr = PosixFilePermissions.asFileAttribute(perms); - // try { - // Files.createDirectories(conflictPath, attr); - // } catch (Exception e) { - // try { - // Files.createDirectory(conflictPath); - // } catch (Exception e1) { - // logger.catching(e); - // throw e; - // } - // } - // } - // - // Path conflictFile = Paths.get(conflictPath.toString(), sourceFileName); - // if (conflictFile.toFile().exists()) - // conflictFile = Paths.get(conflictPath.toString(), sourceFileName + (new Date()).getTime()); - // Files.move(sourcePath, conflictFile); - // result = true; - // } catch (Exception e1) { - // logger.error(getSessionMarker(), "Hiba az '{}' állomány mappába mozgatásakor. A rendszer üzenete: {}", CONFLICT, e1.getMessage()); - // } } - return result; } @@ -214,46 +218,71 @@ public class ArchiveRecursive extends JobStep implements FileVisitor { } @Override - public FileVisitResult preVisitDirectory(Path dir, BasicFileAttributes paramBasicFileAttributes) throws IOException { + public FileVisitResult preVisitDirectory(Path dir, BasicFileAttributes paramBasicFileAttributes) + throws IOException { Path dirName = dir.getFileName(); - logger.info("PreVisit {}", dir); - if (skipPathNames.contains(dirName.toString())) + if (skipPathNames.contains(dirName.toString())) { + logger.info(getSessionMarker(), "PreVisit skip {}", dir); return FileVisitResult.SKIP_SUBTREE; - else - logger.info("PreVisit {}", dir); + } else + logger.info(getSessionMarker(), "PreVisit {}", dir); return FileVisitResult.CONTINUE; } private boolean processPathItem(Path mediaPath) throws Exception { - if (submitted == limit) + if (limit != 0 && submitted == limit) { + logger.info(getSessionMarker(), "Limit reached {}, canceling", limit); + return false; + } + + String fileName = mediaPath.getFileName().toString(); + if (fileName.startsWith(".") || fileName.endsWith(".nomd")) return false; - // if (!"Transfered".equals(mediaPath.getParent().getFileName().toString())) - // return false; + logger.info(getSessionMarker(), "Processing {}", mediaPath); - if (mediaPath.getFileName().toString().startsWith(".")) + File mediaFile = mediaPath.toFile(); + if (mediaFile.isDirectory()) { + //logger.info(getSessionMarker(), "Skipping directory {}", mediaPath); return false; + } - File mediaFile = mediaPath.toFile(); - if (mediaFile.isDirectory()) + if (EscortFiles.isMediaCatched(mediaPath)) { + //logger.info(getSessionMarker(), "Skipping already catched {}", mediaPath); return false; + } - if (EscortFiles.isMediaCatched(mediaPath)) + Path nomdFile = Paths.get(mediaPath.toString() + ".nomd"); + + if (nomdFile.toFile().exists()) { + //logger.info(getSessionMarker(), "Skipping nomd file exists {}", nomdFile); return false; + } - if (handleArchiveConflict(mediaPath)) + if (handleArchiveConflict(mediaPath)) { + logger.info(getSessionMarker(), "Skipping archive db already contains {}", mediaPath); return false; + } - if (!canReadMediaInfo(mediaPath)) + if (!canReadMediaInfo(mediaPath)) { + logger.info(getSessionMarker(), "Skipping cant read mediainfo {}", mediaPath); return false; + } ArchiveItem archiveItem = createArchiveItem(mediaPath); + if (archiveItem == null) { + Message msg = new ParameterizedMessage("Nincs metaadat!"); + logger.info(new MediaCubeMarker("vasary@elgekko.net,muszak@mediavivantis.hu", + "Értesítés problémás " + mediaPath.getFileName().toString() + " archiválásról"), msg); + Files.createFile(nomdFile); + return false; + } + try { - checkArchiveItem(archiveItem, mediaPath); - submitted++; + checkArchiveItem(archiveItem); Map parameters = ListUtils.asMap(ARCHIVE_ITEM, archiveItem, KILL_DATE_DAYS, killDateDays); IJobRuntime runtime = getEngine().submit(getJobRuntime(), e -> { @@ -264,9 +293,13 @@ public class ArchiveRecursive extends JobStep implements FileVisitor { throw new Exception("Submit returned null runtime"); runtime.setRelated(mediaPath.toString()); EscortFiles.createMediaCatch(mediaPath); + String metadata = archiveItemJSON(archiveItem); + EscortFiles.createMetadata(mediaPath.getParent().toString(), mediaPath.getFileName().toString(), metadata); + submitted++; } catch (Exception e) { - String fileName = new File(archiveItem.getMediaFile()).getName(); - logger.error(getSessionMarker(), "Az '{}' állomány archiválási kísérlete sikertelen. A rendszer üzenete: {}", fileName, e.getMessage()); + logger.error(getSessionMarker(), + "Az '{}' állomány archiválási kísérlete sikertelen. A rendszer üzenete: {}", mediaPath, + e.getMessage()); } return true; @@ -274,7 +307,7 @@ public class ArchiveRecursive extends JobStep implements FileVisitor { @Override public FileVisitResult visitFile(Path filePath, BasicFileAttributes paramBasicFileAttributes) throws IOException { - //logger.info("Will archived {}", filePath); + logger.info(getSessionMarker(), "Will checked {}", filePath); try { processPathItem(filePath); } catch (Exception e) { diff --git a/server/-product/production/MEDIAVIVANTIS/jobs/steps/CancelableStep.java b/server/-product/production/MEDIAVIVANTIS/jobs/steps/CancelableStep.java index ebbe2d36..29906887 100644 --- a/server/-product/production/MEDIAVIVANTIS/jobs/steps/CancelableStep.java +++ b/server/-product/production/MEDIAVIVANTIS/jobs/steps/CancelableStep.java @@ -1,43 +1,30 @@ package user.jobengine.server.steps; -import org.apache.commons.net.ftp.FTPClient; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; -import org.apache.logging.log4j.Marker; -import org.apache.logging.log4j.MarkerManager; +import org.apache.logging.log4j.message.Message; +import org.apache.logging.log4j.message.ParameterizedMessage; -import user.commons.StoreUri; -import user.commons.remotestore.FtpDirectoryLister; -import user.commons.remotestore.RemoteStoreProtocol; +import user.commons.MediaCubeMarker; public class CancelableStep extends JobStep { private static final Logger logger = LogManager.getLogger(); - int count = 10; @StepEntry public Object[] execute(int param) throws Exception { - try { + logger.info(getMarker(), "Executing"); + Message msg = new ParameterizedMessage(""); + logger.info(new MediaCubeMarker("vasary@elgekko.net,vasary@gmail.com", "MediaCube rendszerüzenet"), msg); - Marker marker = MarkerManager.getMarker("MEDIAPROFILE"); - getJobRuntime().setRelated("TESZT" + param); - - for (int i = 0; i < count; i++) { - if (getJobRuntime().isWaitingCancel()) - break; - Thread.sleep(500); - if (param==0) - throw new Exception(); - if (param==1) { - getJobRuntime().setDisableRetry(true); - throw new Exception(); - } - int progress = (i + 1) * 100 / count; - setProgress(progress); - } - } catch (Exception e) { - logger.info(e.getMessage()); - throw e; + for (int i = 0; i < 10; i++) { + logger.info(getMarker(), "Executing {}", i); + if (getJobRuntime().isWaitingCancel()) + break; + Thread.sleep(100); + int progress = (i + 1) * 100 / 10; + setProgress(progress); } + return null; } diff --git a/server/-product/production/MEDIAVIVANTIS/jobs/steps/CleanupMountedLocationStep.java b/server/-product/production/MEDIAVIVANTIS/jobs/steps/CleanupMountedLocationStep.java index 4e6f10d1..3ed0e4f4 100644 --- a/server/-product/production/MEDIAVIVANTIS/jobs/steps/CleanupMountedLocationStep.java +++ b/server/-product/production/MEDIAVIVANTIS/jobs/steps/CleanupMountedLocationStep.java @@ -23,12 +23,7 @@ import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.apache.logging.log4j.Marker; -import user.commons.RemoteFile; -import user.commons.StoreUri; -import user.commons.remotestore.RemoteStoreProtocol; -import user.jobengine.db.Store; -import user.jobengine.server.IJobEngine; -import user.jobengine.server.IJobRuntime; +import user.jobengine.server.steps.shared.ItemManagerExtensions; public class CleanupMountedLocationStep extends JobStep implements FileVisitor { @@ -68,7 +63,7 @@ public class CleanupMountedLocationStep extends JobStep implements FileVisitor

killDateFiles) { Date killDate = null; @@ -83,7 +78,8 @@ public class CleanupMountedLocationStep extends JobStep implements FileVisitor

directoryStream = null; @@ -97,14 +93,6 @@ public class CleanupMountedLocationStep extends JobStep implements FileVisitor

{ - tsmName[0] = rs.getString("relativepath"); - return false; - }, null); - - if (tsmName[0] != null) { - try { - RemoteFile remoteFile = tsmStoreUri.getRemoteFile(tsmName[0]); - result = remoteFile != null; - } catch (Exception e) { - logger.error(e.getMessage()); - } finally { - tsmStoreUri.cleanUp(); - } - } - return result; - } - @Override public FileVisitResult postVisitDirectory(Path dir, IOException exc) throws IOException { if (!dir.equals(sourcePath) && isEmpty(dir)) { @@ -246,9 +211,11 @@ public class CleanupMountedLocationStep extends JobStep implements FileVisitor

0) { + if (!ItemManagerExtensions.isArchived(getManager(), filePath)) { + logger.error(marker, "A(z) {} anyag törlésre van kijelölve, de nem található az archívumban.", filePath); + return; + } } if (removeFiles(filePath, killDateFiles)) diff --git a/server/-product/production/MEDIAVIVANTIS/jobs/steps/CreateMissingLowresStep.java b/server/-product/production/MEDIAVIVANTIS/jobs/steps/CreateMissingLowresStep.java index 19bad479..b549e59e 100644 --- a/server/-product/production/MEDIAVIVANTIS/jobs/steps/CreateMissingLowresStep.java +++ b/server/-product/production/MEDIAVIVANTIS/jobs/steps/CreateMissingLowresStep.java @@ -13,6 +13,7 @@ import user.commons.MediaCubeUndoMarker; import user.commons.nosql.NoSQLUtils; import user.jobengine.db.IItemManager; import user.jobengine.db.Media; +import user.jobengine.db.MediaFile; import user.jobengine.server.IJobEngine; import user.jobengine.server.IJobRuntime; @@ -22,6 +23,7 @@ public class CreateMissingLowresStep extends JobStep { @StepEntry public Object[] execute(String localHiresPath, IJobEngine jobEngine, IJobRuntime jobRuntime) throws Exception { Object[] result = new Object[] { null, null, "%s", null, 0, true }; + DB db = NoSQLUtils.getNoSQLDB(); DBCollection collection = db.getCollection("missing_lowres"); IItemManager manager = jobEngine.getItemManager(); @@ -55,21 +57,30 @@ public class CreateMissingLowresStep extends JobStep { private Media getFirstUntranscodedMedia(IItemManager manager, DBCollection collection) { Media[] result = new Media[] { null }; - //MV + // MV String query = "SELECT mediaid FROM VW_MISSING_PROXY_IDS WHERE HOUSEID like 'M%' or HOUSEID like 'P%' or HOUSEID like 'R%' ORDER BY modified DESC"; - //HTV - //String query = "SELECT mediaid FROM VW_MISSING_PROXY_IDS"; + // HTV + // String query = "SELECT mediaid FROM VW_MISSING_PROXY_IDS"; manager.executeQuery(query, rs -> { try { long mediaId = rs.getLong(1); Media media = manager.getMedia(mediaId); - //a nevgeneralas miatt az eredeti MediaFilesName nem jo, a pontos nev kell nekunk + // a nevgeneralas miatt az eredeti MediaFilesName nem jo, a pontos nev kell + // nekunk String name = media.getMediaFileRealName(); - logger.info(getSessionMarker(), "Checking {}", name); + //logger.info(getSessionMarker(), "Checking {}", name); long existing = collection.find(new BasicDBObject("name", name)).count(); if (existing > 0) { - logger.info(getSessionMarker(), "{} is on missing_lowres list", name); + // logger.info(getSessionMarker(), "{} is on missing_lowres list", name); + return true; + } + + // 210617 proxy keszites tiltasa + MediaFile mf = manager.getSystemMediaFile(media); + if (mf.isDisableProxy()) { + logger.info(getSessionMarker(), "Proxy disabled {}", name); + return true; } diff --git a/server/-product/production/MEDIAVIVANTIS/jobs/steps/ExternalCommandExecutor.java b/server/-product/production/MEDIAVIVANTIS/jobs/steps/ExternalCommandExecutor.java deleted file mode 100644 index a34ff7f0..00000000 --- a/server/-product/production/MEDIAVIVANTIS/jobs/steps/ExternalCommandExecutor.java +++ /dev/null @@ -1,32 +0,0 @@ -package user.jobengine.server.steps.shared; - -import user.commons.configuration.SystemConfiguration; - -public class ExternalCommandExecutor { - - public void execute(String profileName, String input, String output, IExternalCallback responseCallBack) throws Exception { - ExternalCommand externalCommand = getExternalCommand(profileName); - externalCommand.execute(input, output, false, responseCallBack); - } - - private ExternalCommand getExternalCommand(String profileName) throws Exception { - ExternalProfilesConfig config = SystemConfiguration.getInstance().load("settings/external-commands.yaml", ExternalProfilesConfig.class); - - if (config == null) - throw new Exception("Missing external-commands.yaml configuration"); - - ExternalProfile selectedProfile = null; - for (ExternalProfile profile : config.getProfiles()) { - if (profileName.equals(profile.getName())) { - selectedProfile = profile; - break; - } - } - - if (selectedProfile == null) - throw new Exception("Missing profile " + profileName + " in external-commands.yaml configuration"); - - return new ExternalCommand(selectedProfile); - } - -} diff --git a/server/-product/production/MEDIAVIVANTIS/jobs/steps/ExternalProfile.java b/server/-product/production/MEDIAVIVANTIS/jobs/steps/ExternalProfile.java deleted file mode 100644 index 68e22f4c..00000000 --- a/server/-product/production/MEDIAVIVANTIS/jobs/steps/ExternalProfile.java +++ /dev/null @@ -1,33 +0,0 @@ -package user.jobengine.server.steps.shared; - -import java.util.List; - -public class ExternalProfile { - private String executable; - private String name; - private List arguments; - - public List getArguments() { - return arguments; - } - - public String getExecutable() { - return executable; - } - - public String getName() { - return name; - } - - public void setArguments(List arguments) { - this.arguments = arguments; - } - - public void setExecutable(String executable) { - this.executable = executable; - } - - public void setName(String name) { - this.name = name; - } -} \ No newline at end of file diff --git a/server/-product/production/MEDIAVIVANTIS/jobs/steps/ExternalProfilesConfig.java b/server/-product/production/MEDIAVIVANTIS/jobs/steps/ExternalProfilesConfig.java deleted file mode 100644 index 7ef77291..00000000 --- a/server/-product/production/MEDIAVIVANTIS/jobs/steps/ExternalProfilesConfig.java +++ /dev/null @@ -1,15 +0,0 @@ -package user.jobengine.server.steps.shared; - -import java.util.List; - -public class ExternalProfilesConfig { - private List profiles; - - public List getProfiles() { - return profiles; - } - - public void setProfiles(List profiles) { - this.profiles = profiles; - } -} \ No newline at end of file diff --git a/server/-product/production/MEDIAVIVANTIS/jobs/steps/IExternalCallback.java b/server/-product/production/MEDIAVIVANTIS/jobs/steps/IExternalCallback.java deleted file mode 100644 index 1e071913..00000000 --- a/server/-product/production/MEDIAVIVANTIS/jobs/steps/IExternalCallback.java +++ /dev/null @@ -1,5 +0,0 @@ -package user.jobengine.server.steps.shared; - -public interface IExternalCallback { - void onResponse(String data); -} \ No newline at end of file diff --git a/server/-product/production/MEDIAVIVANTIS/jobs/steps/ItemManagerExtensions.java b/server/-product/production/MEDIAVIVANTIS/jobs/steps/ItemManagerExtensions.java deleted file mode 100644 index bf807791..00000000 --- a/server/-product/production/MEDIAVIVANTIS/jobs/steps/ItemManagerExtensions.java +++ /dev/null @@ -1,58 +0,0 @@ -package user.jobengine.server.steps; - -import com.ibm.nosql.json.api.BasicDBObject; - -import user.jobengine.db.IItemManager; -import user.jobengine.db.IResultSetConsumer; - -public class ItemManagerExtensions { - - public static BasicDBObject getArchiveInfo(IItemManager manager, long houseid) { - final BasicDBObject[] result = { null }; - StringBuilder query = new StringBuilder(); - query.append("select count(*) as count, sum(length) as duration FROM media"); - query.append(" "); - query.append(String.format("where houseid='%d' and itemtypeid = 82", houseid)); - query.append(" "); - query.append("group by houseid"); - IResultSetConsumer consumer = rs -> { - BasicDBObject o = new BasicDBObject(); - o.put("count", rs.getLong("count")); - o.put("duration", rs.getLong("duration")); - result[0] = o; - return false; - }; - manager.executeQuery(query.toString(), consumer, null); - return result[0]; - } - - public static long getExistingRundownMedia(IItemManager manager, String houseid) { - final long[] result = { 0 }; - final String[] idToCheck = { houseid }; - int pos = houseid.lastIndexOf("-"); - //a hivas a CopyForArchiveNEXIOMaterialsStep-bol is johet, ott meg nincs idobelyegezve a nev! - if (pos > 0 && houseid.length() - pos > 4) - idToCheck[0] = houseid.substring(0, pos); - MetadataType metadataType = MetadataTypeDetector.GuessMetadataType(idToCheck[0]); - if (metadataType == MetadataType.OctopusPlaceholder) { - StringBuilder query = new StringBuilder(); - query.append("select mediaid, mediafilehouseid, filename"); - query.append(" "); - query.append(String.format("from vw_rundown_items where mediafilehouseid like '%s%%'", idToCheck[0])); - query.append(" "); - query.append("order by filename, mediaid"); - IResultSetConsumer consumer = rs -> { - String fileName = rs.getString("filename"); - if (idToCheck[0].equals(fileName)) { - result[0] = rs.getLong("mediaid"); - return false; - } else - return true; - }; - manager.executeQuery(query.toString(), consumer, null); - } - - return result[0]; - } - -} diff --git a/server/-product/production/MEDIAVIVANTIS/jobs/steps/MediaCubeClient.java b/server/-product/production/MEDIAVIVANTIS/jobs/steps/MediaCubeClient.java deleted file mode 100644 index 840925e4..00000000 --- a/server/-product/production/MEDIAVIVANTIS/jobs/steps/MediaCubeClient.java +++ /dev/null @@ -1,77 +0,0 @@ -package user.jobengine.server.steps; - -import javax.ws.rs.client.Entity; -import javax.ws.rs.client.Invocation.Builder; -import javax.ws.rs.core.MediaType; -import javax.ws.rs.core.MultivaluedMap; -import javax.ws.rs.core.Response; -import javax.ws.rs.core.Response.Status; - -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; -import org.jboss.resteasy.client.jaxrs.ResteasyClient; -import org.jboss.resteasy.client.jaxrs.ResteasyClientBuilder; -import org.jboss.resteasy.client.jaxrs.ResteasyWebTarget; -import org.jboss.resteasy.specimpl.MultivaluedMapImpl; - -import com.ibm.nosql.json.JSONUtil; -import com.ibm.nosql.json.api.BasicDBObject; - -public class MediaCubeClient { - private static Logger logger = LogManager.getLogger(); - private ResteasyWebTarget webTarget; - - public MediaCubeClient(String address) { - ResteasyClient client = new ResteasyClientBuilder().build(); - webTarget = client.target(address); - } - - BasicDBObject getDbObject(String json) { - BasicDBObject result = (BasicDBObject) JSONUtil.jsonToDbObject(json); - - if (result == null) - throw new NullPointerException("API Result is null!"); - - if (result.containsKey("exception")) { - BasicDBObject e = (BasicDBObject) result.get("exception"); - throw new RuntimeException(e.getString("message")); - } - //{"exception":{"message":"Invalid credentials.","publicName":"AuthenticationFailedException"}} - return result; - } - - public BasicDBObject getStatus(long jobId) { - MultivaluedMap vars = new MultivaluedMapImpl<>(); - vars.add("jobId", jobId); - Response response = query("services/rest/jobengine/jobstatus", vars).get(); - if (response.getStatus() != Status.OK.getStatusCode()) { - logger.error(response.readEntity(String.class)); - System.out.println(response.readEntity(String.class)); - return null; - } - String result = response.readEntity(String.class); - return getDbObject(result); - } - - private Builder query(String path, MultivaluedMap vars) { - ResteasyWebTarget target = webTarget.path(path).queryParams(vars); - Builder result = target.request(); - return result; - } - - public long startjob(String template, String name, BasicDBObject jobParams) throws Exception { - MultivaluedMap vars = new MultivaluedMapImpl<>(); - vars.add("template", template); - vars.add("name", name); - Response response = query("services/rest/jobengine/startjob", vars).post(Entity.entity(jobParams.toString(), MediaType.APPLICATION_JSON)); - - if (response.getStatus() != Status.OK.getStatusCode()) { - logger.error(response.readEntity(String.class)); - return 0; - } - - String resultObject = response.readEntity(String.class); - return Long.parseLong(resultObject); - } - -} diff --git a/server/-product/production/MEDIAVIVANTIS/jobs/steps/MediaToolStep.java b/server/-product/production/MEDIAVIVANTIS/jobs/steps/MediaToolStep.java index c0c56f68..ecc0ba2d 100644 --- a/server/-product/production/MEDIAVIVANTIS/jobs/steps/MediaToolStep.java +++ b/server/-product/production/MEDIAVIVANTIS/jobs/steps/MediaToolStep.java @@ -5,35 +5,23 @@ import java.nio.file.Paths; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; -import org.apache.logging.log4j.Marker; -import user.commons.mediatool.MediaInfo; +import user.commons.mediaarea.MediaArea; import user.jobengine.db.Media; public class MediaToolStep extends JobStep { private static final Logger logger = LogManager.getLogger(); - private Marker marker; @StepEntry public Object[] execute(ArchiveItem archiveItem, Media mediaCubeMedia) throws Exception { - marker = getSessionMarker(); - - //TODO ez most minden esetben ujranezi - // if (mediaCubeMedia.getLength() > 0) - // return null; - Path filePath = Paths.get(archiveItem.getMediaFile()); - MediaInfo mi = new MediaInfo(filePath); - try { - mi.process(); - long frames = mi.getFrames(); - if (frames > 0) { - mediaCubeMedia.setLength(frames); - getManager().modify(mediaCubeMedia); - } - } catch (Exception e) { - logger.catching(e); - logger.error(marker, "A '{}' média analizálása sikertelen. A rendszer üzenete: {}", filePath, e.getMessage()); + MediaArea ma = new MediaArea(filePath); + ma.process(); + long frames = ma.getFrameCount(); + if (frames > 0) { + logger.info("Media {} length is {}", filePath, frames); + mediaCubeMedia.setLength(frames); + getManager().modify(mediaCubeMedia); } return null; } diff --git a/server/-product/production/MEDIAVIVANTIS/jobs/steps/MetadataType.java b/server/-product/production/MEDIAVIVANTIS/jobs/steps/MetadataType.java deleted file mode 100644 index 12e166cc..00000000 --- a/server/-product/production/MEDIAVIVANTIS/jobs/steps/MetadataType.java +++ /dev/null @@ -1,5 +0,0 @@ -package user.jobengine.server.steps; - -public enum MetadataType { - TrafficMaterial, TrafficPromo, TrafficAD, OctopusStory, OctopusPlaceholder, Generic -} diff --git a/server/-product/production/MEDIAVIVANTIS/jobs/steps/MetadataTypeDetector.java b/server/-product/production/MEDIAVIVANTIS/jobs/steps/MetadataTypeDetector.java deleted file mode 100644 index 0bb69923..00000000 --- a/server/-product/production/MEDIAVIVANTIS/jobs/steps/MetadataTypeDetector.java +++ /dev/null @@ -1,49 +0,0 @@ -package user.jobengine.server.steps; - -import org.apache.commons.lang.StringUtils; - -public class MetadataTypeDetector { - - private static final String HYPHEN = "-"; - private static final String DOT = "."; - - private static final String REGEXP_TRAFFICMATERIALID = "^M{1}[0-9]{6}[A-Z]{1}$"; - private static final String REGEXP_TRAFFICADID = "^R{1}[0-9]{6}[A-Z]{1}$"; - private static final String REGEXP_TRAFFICPROMOID = "^P{1}[0-9]{6}[A-Z]{1}$"; - private static final String REGEXP_OCTOPUSSTORYID = "^[0-9]+$"; - private static final String REGEXP_OCTOPUSPLACEHOLDERID = "^[0-9]+_[0-9]+$"; - private static final String REGEXP_OCTOPUSPLACEHOLDERVERSIONEDID = "^[0-9]+_[0-9]+-[0-9]{3}$"; - - public static MetadataType GuessMetadataType(String id) { - if (StringUtils.isBlank(id)) - return MetadataType.Generic; - if (id.matches(REGEXP_TRAFFICMATERIALID)) - return MetadataType.TrafficMaterial; - if (id.matches(REGEXP_TRAFFICPROMOID)) - return MetadataType.TrafficPromo; - if (id.matches(REGEXP_TRAFFICADID)) - return MetadataType.TrafficAD; - if (id.matches(REGEXP_OCTOPUSSTORYID)) - return MetadataType.OctopusStory; - if (id.matches(REGEXP_OCTOPUSPLACEHOLDERID)) - return MetadataType.OctopusPlaceholder; - if (id.matches(REGEXP_OCTOPUSPLACEHOLDERVERSIONEDID)) - return MetadataType.OctopusPlaceholder; - return MetadataType.Generic; - } - - public static String truncateExtension(String name) { - String result = name; - if (result != null && result.contains(DOT)) - result = result.substring(0, result.lastIndexOf(DOT)); - return result; - } - - public static String truncateVersion(String name) { - String result = name; - if (result != null && result.contains(HYPHEN)) - result = result.split(HYPHEN)[0]; - return result; - } - -} diff --git a/server/-product/production/MEDIAVIVANTIS/jobs/steps/OutputPathAndNameSelectorStep.java b/server/-product/production/MEDIAVIVANTIS/jobs/steps/OutputPathAndNameSelectorStep.java index 6950babe..f0203ca5 100644 --- a/server/-product/production/MEDIAVIVANTIS/jobs/steps/OutputPathAndNameSelectorStep.java +++ b/server/-product/production/MEDIAVIVANTIS/jobs/steps/OutputPathAndNameSelectorStep.java @@ -17,12 +17,9 @@ import org.apache.logging.log4j.Marker; import user.jobengine.db.ArchivedMedia; import user.jobengine.server.IJobEngine; import user.jobengine.server.IJobRuntime; - -/* -package user.jobengine.server.steps.MetadataType; -package user.jobengine.server.steps.MetadataTypeDetector; -package user.jobengine.server.steps.EscortFiles; -*/ +import user.jobengine.server.steps.shared.EscortFiles; +import user.jobengine.server.steps.shared.MetadataType; +import user.jobengine.server.steps.shared.MetadataTypeDetector; public class OutputPathAndNameSelectorStep extends JobStep { diff --git a/server/-product/production/MEDIAVIVANTIS/jobs/steps/PrepareMediaRestoreStep.java b/server/-product/production/MEDIAVIVANTIS/jobs/steps/PrepareMediaRestoreStep.java index fd7d25ec..3708aa3e 100644 --- a/server/-product/production/MEDIAVIVANTIS/jobs/steps/PrepareMediaRestoreStep.java +++ b/server/-product/production/MEDIAVIVANTIS/jobs/steps/PrepareMediaRestoreStep.java @@ -1,5 +1,7 @@ package user.jobengine.server.steps; +import java.util.List; + import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; @@ -7,16 +9,21 @@ import user.commons.StoreUri; import user.commons.remotestore.RemoteStoreProtocol; import user.jobengine.db.Media; import user.jobengine.db.MediaFile; +import user.jobengine.db.Store; public class PrepareMediaRestoreStep extends JobStep { private static final Logger logger = LogManager.getLogger(); @StepEntry - public Object[] execute(Media media, String mediaStoreProtocol, String tempStoreName, String tempStoreProtocol) throws Exception { + public Object[] execute(Media media, String tempStoreName, String tempStoreProtocol) throws Exception { StoreUri mediaStoreUri = null; - for (MediaFile mediaFile : media.getMediaFiles()) { - if (mediaFile.getStore().isSystem() && !mediaFile.getStore().isLowres()) { - mediaStoreUri = mediaFile.getStore().getStoreUri(Enum.valueOf(RemoteStoreProtocol.class, mediaStoreProtocol)); + List mediaFiles = media.getMediaFiles(); + if (mediaFiles.size() > 1) + throw new Exception("Media " + media.getId() + " already has proxy"); + for (MediaFile mediaFile : mediaFiles) { + Store store = mediaFile.getStore(); + if (store.isSystem() && !store.isLowres()) { + mediaStoreUri = store.getStoreUri(RemoteStoreProtocol.TSM); if (mediaStoreUri != null) break; } diff --git a/server/-product/production/MEDIAVIVANTIS/jobs/steps/PrepareRemoteTranscodeJobParametersStep.java b/server/-product/production/MEDIAVIVANTIS/jobs/steps/PrepareRemoteTranscodeJobParametersStep.java deleted file mode 100644 index 37306e60..00000000 --- a/server/-product/production/MEDIAVIVANTIS/jobs/steps/PrepareRemoteTranscodeJobParametersStep.java +++ /dev/null @@ -1,20 +0,0 @@ -package user.jobengine.server.steps; - -import java.nio.file.Paths; - -import com.ibm.nosql.json.api.BasicDBObject; - -public class PrepareRemoteTranscodeJobParametersStep extends JobStep { - @StepEntry - public Object[] execute(String profileName, String fileName) throws Exception { - String transcoderInputDir = "m:\"; - String transcoderOutputDir = "m:\"; - String outFileName = fileName.substring(0, fileName.lastIndexOf(".")) + ".mp4"; - BasicDBObject parameters = new BasicDBObject(); - parameters.put("profile", profileName); - parameters.put("input", Paths.get(transcoderInputDir, fileName)); - parameters.put("output", Paths.get(transcoderOutputDir, outFileName)); - return new Object[] { parameters }; - } - -} diff --git a/server/-product/production/MEDIAVIVANTIS/jobs/steps/QueryMissingProxyMediaStep.java b/server/-product/production/MEDIAVIVANTIS/jobs/steps/QueryMissingProxyMediaStep.java index 9e8d1949..a91f9e97 100644 --- a/server/-product/production/MEDIAVIVANTIS/jobs/steps/QueryMissingProxyMediaStep.java +++ b/server/-product/production/MEDIAVIVANTIS/jobs/steps/QueryMissingProxyMediaStep.java @@ -11,37 +11,28 @@ import user.commons.MediaCubeUndoMarker; import user.commons.nosql.NoSQLUtils; import user.jobengine.db.IItemManager; import user.jobengine.db.Media; +import user.jobengine.db.MediaFile; public class QueryMissingProxyMediaStep extends JobStep { private static final Logger logger = LogManager.getLogger(); @StepEntry public Object[] execute() throws Exception { - Object[] result = new Object[] { null }; + setDescription("Looking for missing proxy"); DB db = NoSQLUtils.getNoSQLDB(); DBCollection collection = db.getCollection("missing_lowres"); Media media = getFirstUntranscodedMedia(collection); - try { - if (media == null) { - logger.info(new MediaCubeUndoMarker(getSessionMarker().getSessionID()), "Nincs feldolgozandó hiány."); - // throw new Exception("Nincs feldolgozandó hiány."); - cancel(); - return null; - } - - String name = media.getMediaFileRealName(); - collection.save(new BasicDBObject("name", name)); - logger.info(getSessionMarker(), "Processing mediaID: {}", media.getId()); - result[0] = media; - } catch (Exception e) { - logger.catching(e); - logger.error(getSessionMarker(), e.getMessage()); - throw e; - } finally { - setProgress(100); + if (media == null) { + logger.info(new MediaCubeUndoMarker(getSessionMarker().getSessionID()), "Nincs feldolgozandó hiány."); + cancel(); + return null; } - return result; + + String name = media.getMediaFileRealName(); + collection.save(new BasicDBObject("name", name)); + logger.info(getSessionMarker(), "Processing mediaID: {}", media.getId()); + return new Object[] { media }; } private Media getFirstUntranscodedMedia(DBCollection collection) { @@ -58,10 +49,17 @@ public class QueryMissingProxyMediaStep extends JobStep { Media media = manager.getMedia(mediaId); //a nevgeneralas miatt az eredeti MediaFilesName nem jo, a pontos nev kell nekunk String name = media.getMediaFileRealName(); - logger.info(getSessionMarker(), "Checking {}", name); + //logger.info(getSessionMarker(), "Checking {}", name); long existing = collection.find(new BasicDBObject("name", name)).count(); if (existing > 0) { - logger.info(getSessionMarker(), "{} is on missing_lowres list", name); + //logger.info(getSessionMarker(), "{} is on missing_lowres list", name); + return true; + } + // 210617 proxy keszites tiltasa + MediaFile mf = manager.getSystemMediaFile(media); + if (mf.isDisableProxy()) { + //logger.info(getSessionMarker(), "Proxy disabled {}", name); + return true; } diff --git a/server/-product/production/MEDIAVIVANTIS/jobs/steps/RemoteJobStep.java b/server/-product/production/MEDIAVIVANTIS/jobs/steps/RemoteJobStep.java index 8e022f43..d4900d76 100644 --- a/server/-product/production/MEDIAVIVANTIS/jobs/steps/RemoteJobStep.java +++ b/server/-product/production/MEDIAVIVANTIS/jobs/steps/RemoteJobStep.java @@ -5,39 +5,32 @@ import org.apache.logging.log4j.Logger; import com.ibm.nosql.json.api.BasicDBObject; -/* -import user.jobengine.server.steps.MediaCubeClient; -*/ +import user.jobengine.server.steps.shared.MediaCubeClient; + public class RemoteJobStep extends JobStep { private static final Logger logger = LogManager.getLogger(); @StepEntry - public Object[] execute(String server, String template, String name, String profile) throws Exception { - try { - MediaCubeClient mc = new MediaCubeClient(server); - BasicDBObject params = new BasicDBObject(); - params.put("profile", profile); - params.put("input", "c:/_workspace/data/video/proba1.mxf"); - params.put("output", "c:/_video/proba1.mp4"); - - long jobId = mc.startjob(template, name, params); - logger.info(getMarker(), "Started {} on server {}", jobId, server); - while (true) { - BasicDBObject status = mc.getStatus(jobId); - if (status != null) - setProgress(status.getInt("progress")); - - String jobStatus = status.getString("status"); - if ("SUSPENDED".equals(jobStatus)) - throw new Exception(status.getString("description")); - - if ("FINISHED".equals(jobStatus)) - break; - } - - } catch (Exception e) { - throw e; + public Object[] execute(String remoteServer, String template, String name, BasicDBObject remoteJobParameters) throws Exception { + setDescription("Executing on {}, template {}, profile {}", remoteServer, template, remoteJobParameters.getString("profile")); + MediaCubeClient mc = new MediaCubeClient(remoteServer); + long jobId = mc.startjob(template, name, remoteJobParameters); + logger.info(getMarker(), "Started {} on server {}", jobId, remoteServer); + while (true) { + BasicDBObject status = mc.getStatus(jobId); + if (status != null) + setProgress(status.getInt("progress")); + + Thread.sleep(2000); + + String jobStatus = status.getString("status"); + if ("SUSPENDED".equals(jobStatus)) + throw new Exception(status.getString("description")); + + if ("FINISHED".equals(jobStatus)) + break; } - return new Object[] { null }; + + return null; } } diff --git a/server/-product/production/MEDIAVIVANTIS/jobs/steps/SafeDeleteRecursive.java b/server/-product/production/MEDIAVIVANTIS/jobs/steps/SafeDeleteRecursive.java new file mode 100644 index 00000000..42c1bf9b --- /dev/null +++ b/server/-product/production/MEDIAVIVANTIS/jobs/steps/SafeDeleteRecursive.java @@ -0,0 +1,416 @@ +package user.jobengine.server.steps; + +import java.io.File; +import java.io.IOException; +import java.nio.file.DirectoryStream; +import java.nio.file.FileVisitResult; +import java.nio.file.FileVisitor; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.nio.file.attribute.BasicFileAttributes; +import java.text.SimpleDateFormat; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.Date; +import java.util.LinkedHashSet; +import java.util.List; +import java.util.Set; + +import org.apache.commons.io.FileUtils; +import org.apache.commons.io.FilenameUtils; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + +import com.ibm.nosql.json.api.BasicDBObject; + +import user.commons.IEntityBase; +import user.commons.RemoteFile; +import user.commons.StoreUri; +import user.commons.mediatool.MediaInfo; +import user.commons.remotestore.RemoteStoreProtocol; +import user.jobengine.db.Item; +import user.jobengine.db.Media; +import user.jobengine.db.MediaFile; +import user.jobengine.db.MediaFileDAO; +import user.jobengine.db.Store; +import user.jobengine.server.steps.shared.EscortFiles; +import user.mediacube.metadata.interfaces.IMetadata; +import user.mediacube.metadata.interfaces.IMetadataListOptions; +import user.mediacube.metadata.interfaces.IMetadataProvider; +import user.mediacube.metadata.interfaces.IMetadataProviderFactory; +import user.mediacube.metadata.interfaces.MetadataProviderType; +import user.mediacube.metadata.interfaces.MetadataType; +import user.mediacube.metadata.interfaces.PlanAirMetadataListOptions; + +public class SafeDeleteRecursive extends JobStep implements FileVisitor { + private static final Logger logger = LogManager.getLogger(); + private static final String ITEM_TITLE = "itemTitle"; + private static final String ITEM_HOUSEID = "itemHouseId"; + private static final String MEDIA_HOUSEID = "mediaHouseId"; + private static final String MEDIA_TITLE = "mediaTitle"; + private static final String MEDIA_DESCRIPTION = "mediaDescription"; + private static final String MEDIA_TYPE = "mediaType"; + private static final String KILLDATEEXT = ".killdate"; + private SimpleDateFormat df = new SimpleDateFormat("yyyy.MM.dd HH:mm:ss"); + + private List skipPathNames = Arrays.asList("!ARCHIVALAS_ALATT", EscortFiles.STATUSFOLDER, + EscortFiles.CONFLICTFOLDER); + private Set includeList; + + private boolean canReadMediaInfo(Path mediaFilePath) { + boolean result = false; + try { + MediaInfo mi = new MediaInfo(mediaFilePath); + mi.process(); + result = true; + } catch (Exception e) { + logger.warn(getSessionMarker(), e.getMessage()); + } + return result; + } + + private boolean checkArchiveItem(ArchiveItem archiveItem, Item item, Media media) { + if (archiveItem == null) + return false; + + String itemHouseId = archiveItem.getItemHouseId() == null ? "" : archiveItem.getItemHouseId(); + if (!itemHouseId.equals(item.getHouseId())) { + logger.error(getSessionMarker(), "ItemHouseId"); + return false; + } + + String itemTitle = archiveItem.getItemTitle() == null ? "" : archiveItem.getItemTitle(); + if (!itemTitle.equals(item.getTitle())) { + logger.error(getSessionMarker(), "ItemTitle"); + return false; + } + String mediaHouseId = archiveItem.getMediaHouseId() == null ? "" : archiveItem.getMediaHouseId(); + if (!mediaHouseId.equals(media.getHouseId())) { + logger.error(getSessionMarker(), "MediaHouseId"); + return false; + } + String mediaTitle = archiveItem.getMediaTitle() == null ? "" : archiveItem.getMediaTitle(); + if (!mediaTitle.equals(media.getTitle())) { + logger.error(getSessionMarker(), "MediaTitle"); + return false; + } + + String mediaDescription = archiveItem.getMediaDescription() == null ? "" : archiveItem.getMediaDescription(); + String storedMediaDescription = media.getDescription() == null ? "" : media.getDescription(); + if (!mediaDescription.equals(storedMediaDescription)) { + logger.error(getSessionMarker(), "MediaDescription"); + return false; + } + + return true; + } + + private ArchiveItem createArchiveItem(Path filePath) throws Exception { + ArchiveItem result = null; + + String fileName = filePath.getFileName().toString(); + String mediaHouseId = FilenameUtils.removeExtension(fileName); + try { + result = getPlanAirMetadata(mediaHouseId); + + if (result != null) + result.setMediaFile(filePath.toString()); + + } catch (Exception e) { + logger.error(getSessionMarker(), "PlanAir metadata error", e); + } + + return result; + } + + @StepEntry + public Object[] execute(String sourcePath) throws Exception { + logger.info(getSessionMarker(), "Starting in {}", sourcePath); + logger.info(getSessionMarker(), "{};{};{};{};{};{};{};{};{};{};{};{};{};{};{};{};{};{};", "Napló időbélyeg", + "Neve", "Elérése", "Mérete", "Létrehozva", "Módosítva", "Archiválva (MC)", "Archiválva (TSM)", + "Méret (TSM)", ".catched", ".killdate", ".nomd", "Archivált (MC)", "Archivált (TSM)", + "Metaadat egyezés", "Méret egyezés", "Törölhető", "Kezelendő"); + + try { + String location = "/opt/test-mediacube/file_list_original.txt"; + includeList = loadIncludeList(location); + Files.walkFileTree(Paths.get(sourcePath), this); + } catch (Exception e) { + logger.error(getSessionMarker(), "Az '{}' mappa elérése sikertelen. A rendszer hibaüzenete: {}", sourcePath, + e.getMessage()); + } + return null; + } + + private ArchiveItem getPlanAirMetadata(String mediaHouseId) throws Exception { + PlanAirMetadataListOptions opt = new PlanAirMetadataListOptions(); + opt.setSearch(mediaHouseId); + opt.setType(MetadataType.Material); + + BasicDBObject json = null; + List data = null; + + IMetadataProviderFactory factory = getService(IMetadataProviderFactory.class); + if (factory == null) + logger.info(getSessionMarker(), "IMetadataProviderFactory is null"); + + IMetadataProvider planairProvider = factory.getProvider(MetadataProviderType.PLANAIR); + if (planairProvider == null) + logger.info(getSessionMarker(), "IMetadataProvider is null"); + + ArchiveItem result = null; + data = planairProvider.list(opt); + if (data.size() != 0) + json = data.get(0).asJSON(); + else { + opt.setType(MetadataType.Promo); + data = planairProvider.list(opt); + if (data.size() != 0) + json = data.get(0).asJSON(); + else { + opt.setType(MetadataType.AD); + data = planairProvider.list(opt); + if (data.size() != 0) + json = data.get(0).asJSON(); + } + } + if (json != null) { + result = new ArchiveItem(); + result.setItemHouseId(json.getString(ITEM_HOUSEID)); + result.setItemTitle(json.getString(ITEM_TITLE)); + result.setMediaHouseId(json.getString(MEDIA_HOUSEID)); + result.setMediaTitle(json.getString(MEDIA_TITLE)); + result.setMediaDescription(json.getString(MEDIA_DESCRIPTION)); + result.setMediaType(json.getString(MEDIA_TYPE)); + } + return result; + } + + @Override + public FileVisitResult postVisitDirectory(Path paramT, IOException paramIOException) throws IOException { + return FileVisitResult.CONTINUE; + } + + @Override + public FileVisitResult preVisitDirectory(Path dir, BasicFileAttributes paramBasicFileAttributes) + throws IOException { + Path dirName = dir.getFileName(); + + if (skipPathNames.contains(dirName.toString())) { + //logger.info(getSessionMarker(), "PreVisit skip {}", dir); + return FileVisitResult.SKIP_SUBTREE; + } //else + //logger.info(getSessionMarker(), "PreVisit {}", dir); + + return FileVisitResult.CONTINUE; + } + + private boolean processPathItem(Path mediaPath) throws Exception { + if (getJobRuntime().isWaitingCancel()) { + cancel(); + return false; + } + + BasicFileAttributes attr = Files.readAttributes(mediaPath, BasicFileAttributes.class); + File mediaFSFile = mediaPath.toFile(); + String fileName = mediaPath.getFileName().toString(); + + if (mediaFSFile.isDirectory()) + return true; + + if (fileName.startsWith(".") || fileName.endsWith(".nomd")) + return true; + + Path nomdFile = Paths.get(mediaPath.toString() + ".nomd"); + ArchiveItem archiveItem = createArchiveItem(mediaPath); + List killDateFiles = getKillDateFiles(mediaPath); + MediaFile mediaFile = getMediaFile(fileName); + + long size = mediaFSFile.length(); + Date lastModifiedDate = new Date(attr.lastModifiedTime().toMillis()); + //Date lastAccesDate = new Date(attr.lastAccessTime().toMillis()); + Date createDate = new Date(attr.creationTime().toMillis()); + boolean catchedExists = EscortFiles.isMediaCatched(mediaPath); + boolean noMDExists = nomdFile.toFile().exists(); + boolean killdateExists = killDateFiles.size() > 0; + //boolean mediaInfoAvailable = canReadMediaInfo(mediaPath); + boolean mcArchived = mediaFile != null; + boolean metadataEquals = false; + boolean tsmArchived = false; + boolean sizeEquals = false; + Date mcArchivedDate = null; + long tsmSize = 0; + Date tsmBackupDate = null; + boolean includeContains = includeList.contains(mediaPath.toString()); + + if (mcArchived) { + //metadata + Media media = getManager().getMedia(mediaFile.getMediaId()); + mcArchivedDate = media.getCreated(); + Item item = getManager().getItem(media.getItemId()); + metadataEquals = checkArchiveItem(archiveItem, item, media); + + //tsm + String tsmFileName = mediaFile.getRelativePath(); + RemoteFile tsmFile = getTSMFile(tsmFileName); + tsmArchived = tsmFile != null; + if (tsmArchived) { + tsmSize = tsmFile.getSize(); + tsmBackupDate = getTSMBackupDate(tsmFileName); + sizeEquals = tsmSize == size; + } + } + + boolean canDelete = !noMDExists && catchedExists && killdateExists && mcArchived && tsmArchived + && metadataEquals; + Date now = new Date(System.currentTimeMillis()); + logger.info(getSessionMarker(), "{};{};{};{};{};{};{};{};{};{};{};{};{};{};{};{};{};{};", D(now), fileName, + mediaPath.getParent(), size, D(createDate), D(lastModifiedDate), D(mcArchivedDate), D(tsmBackupDate), + tsmSize, YN(catchedExists), YN(killdateExists), YN(noMDExists), YN(mcArchived), YN(tsmArchived), + YN(metadataEquals), YN(sizeEquals), YN(canDelete), YN(includeContains)); + + return true; + } + + private Date getTSMBackupDate(String tsmFileName) throws Exception { + Date result = null; + IMetadataProviderFactory factory = getService(IMetadataProviderFactory.class); + if (factory == null) + return null; + IMetadataProvider provider = factory.getProvider(MetadataProviderType.TSM); + if (provider == null) + return null; + + IMetadataListOptions opt = provider.createOptions(new BasicDBObject("fileName", tsmFileName)); + + List tsmContents = provider.list(opt); + if (tsmContents != null && tsmContents.size() > 0) { + + for (IMetadata md : tsmContents) { + Date backupDate = md.asJSON().getDate("backupDate"); + if (result == null) + result = backupDate; + else { + if (backupDate.after(result)) + result = backupDate; + } + } + } + return result; + } + + private char YN(boolean value) { + return value ? 'Y' : 'N'; + } + + private String D(Date value) { + return value == null ? "" : df.format(value); + } + + @Override + public FileVisitResult visitFile(Path filePath, BasicFileAttributes paramBasicFileAttributes) throws IOException { + //logger.info(getSessionMarker(), "Will checked {}", filePath); + try { + if (!processPathItem(filePath)) + return FileVisitResult.TERMINATE; + } catch (Exception e) { + logger.catching(e); + } + return FileVisitResult.CONTINUE; + } + + @Override + public FileVisitResult visitFileFailed(Path filePath, IOException paramIOException) throws IOException { + logger.info("Error archive {}", filePath); + return FileVisitResult.CONTINUE; + } + + private List getKillDateFiles(Path filePath) { + String killDateFilePattern = String.format("%s.*%s", filePath.getFileName().toString(), KILLDATEEXT); + List result = new ArrayList<>(); + Path statusPath = null; + try { + statusPath = Paths.get(filePath.getParent().toString(), EscortFiles.STATUSFOLDER); + } catch (Exception e) { + logger.catching(e); + return null; + } + File statusPathFile = statusPath.toFile(); + if (statusPathFile.exists() && statusPathFile.isDirectory()) { + try (DirectoryStream stream = Files.newDirectoryStream(statusPath, killDateFilePattern)) { + stream.forEach(p -> result.add(p)); + } catch (Exception e) { + logger.catching(e); + } + } + Collections.sort(result); + return result; + } + + MediaFile getMediaFile(String fileName) { + MediaFile result = null; + MediaFileDAO mfDAO = (MediaFileDAO) getManager().getBaseDAO(MediaFile.class); + List mfList = mfDAO.getByHouseId(fileName); + if (mfList != null && mfList.size() == 1) + result = (MediaFile) mfList.get(0); + return result; + } + + RemoteFile getTSMFile(String mcFileName) { + RemoteFile result = null; + Store tsmStore = getManager().getSystemStore(false); + StoreUri tsmStoreUri = tsmStore.getSourceStoreUri(RemoteStoreProtocol.TSM); + try { + result = tsmStoreUri.getRemoteFile(mcFileName); + } catch (Exception e) { + logger.error(e); + } finally { + tsmStoreUri.cleanUp(); + } + return result; + } + + private Set loadIncludeList(String location) throws IOException { + Set result = new LinkedHashSet<>(); + + Path path = Paths.get(location); + List lines = FileUtils.readLines(path.toFile()); + + String lastDir = null; + for (String line : lines) { + line = line.trim(); + if (line.startsWith("Directory of")) { + lastDir = line; + lastDir = lastDir.replace("Directory of", ""); + lastDir = lastDir.replace("X:", ""); + lastDir = lastDir.replace("\\", "/"); + lastDir = lastDir.trim(); + // if (!lastDir.endsWith(".STATUS")) + // System.out.println(">> " + lastDir); + } + + if (lastDir != null && lastDir.endsWith(".STATUS")) + continue; + + if (line.startsWith("2") && line.length() > 39) { + String file = line.substring(39).trim(); + + if (file.equals(".") || file.equals("..")) + continue; + + String len = line.substring(21, 39).trim(); + if (len.length() != 0) { + String fullpath = "/mnt/POLC/FINISHED_SHOWS" + lastDir + "/" + file; + //System.out.println("/mnt/POLC/FINISHED_SHOWS" + lastDir + "/" + file + " : " + len); + result.add(fullpath); + } + } + + } + + return result; + } + +} diff --git a/server/-product/production/MEDIAVIVANTIS/jobs/steps/TSMBackupStep.java b/server/-product/production/MEDIAVIVANTIS/jobs/steps/TSMBackupStep.java index 60adc661..0220f751 100644 --- a/server/-product/production/MEDIAVIVANTIS/jobs/steps/TSMBackupStep.java +++ b/server/-product/production/MEDIAVIVANTIS/jobs/steps/TSMBackupStep.java @@ -2,8 +2,6 @@ package user.jobengine.server.steps; import java.io.File; import java.io.IOException; -import java.nio.file.Files; -import java.nio.file.attribute.BasicFileAttributes; import java.sql.Timestamp; import java.util.Date; import java.util.List; @@ -17,6 +15,7 @@ import org.apache.logging.log4j.message.ParameterizedMessage; import user.commons.RemoteFile; import user.commons.StoreUri; +import user.commons.configuration.SystemConfiguration; import user.commons.mediatool.Timecode; import user.commons.mediatool.Timecode.Type; import user.commons.remotestore.IProgressEventListener; @@ -32,10 +31,13 @@ import user.jobengine.db.Store; import user.jobengine.server.IJobEngine; import user.jobengine.server.IJobRuntime; import user.jobengine.server.JobEngineException; +import user.jobengine.server.steps.shared.EscortFiles; +import user.jobengine.server.steps.shared.ItemManagerExtensions; public class TSMBackupStep extends JobStep { private static final String MXFEXT = ".MXF"; private static final Logger logger = LogManager.getLogger(); + private static boolean RANDOMIZE_ARCHIVES = SystemConfiguration.getInstance().value("tsm.randomize-archives"); private IItemManager manager; private File sourceMediaFile; private Store tsmStore; @@ -49,37 +51,49 @@ public class TSMBackupStep extends JobStep { File sourceMediaFile = new File(archiveItem.getMediaFile()); String sourceFileName = sourceMediaFile.getName(); + long fileSize = sourceMediaFile.length(); + + try { + Timecode timecode = new Timecode(mediaCubeMedia.getLength(), Type.PAL); + String details = String.format("%s (%s, %d bytes)", sourceFileName, timecode.toString(), fileSize); + logger.info(marker, details); + getJobRuntime().setDescription(details); + } catch (Exception e) { + String details = String.format("%s (%d bytes)", sourceFileName, fileSize); + getJobRuntime().setDescription(details); + } Timecode timecode = new Timecode(mediaCubeMedia.getLength(), Type.PAL); - String details = String.format("%s (%s, %d bytes)", sourceFileName, timecode.toString(), sourceMediaFile.length()); + String details = String.format("%s (%s, %d bytes)", sourceFileName, timecode.toString(), fileSize); getJobRuntime().setDescription(details); try { setAndCheck(archiveItem, mediaCubeMedia, getEngine()); - //TODO mxf helyett az osszes kiterjesztest!!!!! - //A dupla ellenorzes a napon beluli ismetlesek miatt kell + // TODO mxf helyett az osszes kiterjesztest!!!!! + // A dupla ellenorzes a napon beluli ismetlesek miatt kell long existingMediaId = archiveItem.getExistingMediaId(); - if (sourceMediaFile.length() == 0 && existingMediaId == 0) { - existingMediaId = ItemManagerExtensions.getExistingRundownMedia(manager, sourceFileName.replace(MXFEXT, "")); + + if (fileSize == 0 && existingMediaId == 0) { + existingMediaId = ItemManagerExtensions.getExistingRundownMedia(manager, + sourceFileName.replace(MXFEXT, "")); if (existingMediaId == 0) existingMediaId = -1; } if (existingMediaId == 0) - existingMediaId = ItemManagerExtensions.getExistingRundownMedia(manager, sourceFileName.replace(MXFEXT, "")); - - boolean randomizeName = Boolean.parseBoolean(System.getProperty("jobengine.randomize.archives", "false")); - + existingMediaId = ItemManagerExtensions.getExistingRundownMedia(manager, + sourceFileName.replace(MXFEXT, "")); String targetFileName; - if (randomizeName) { + if (RANDOMIZE_ARCHIVES) { // a-z, A-Z, 0-9. For example: WRMcpIk7, s57JwCVA - //veletlenszeru neveket adunk! + // veletlenszeru neveket adunk! targetFileName = String.format("%s-%s", RandomStringUtils.randomAlphanumeric(8), sourceFileName); } else targetFileName = sourceFileName; if (existingMediaId == 0) { - StoreUri sourceUri = manager.createStoreUri(RemoteStoreProtocol.LOCAL, sourceMediaFile.getParent().toString()); + StoreUri sourceUri = manager.createStoreUri(RemoteStoreProtocol.LOCAL, + sourceMediaFile.getParent().toString()); final IJobRuntime runtime = getJobRuntime(); sourceUri.addProgressListener(new IProgressEventListener() { @@ -99,18 +113,22 @@ public class TSMBackupStep extends JobStep { } if (existingMediaId > 0) - logger.info(marker, "Az '{}' TSM mentése nem szükséges, mert már megtalálható az archívumban.", sourceFileName); + logger.info(marker, "Az '{}' TSM mentése nem szükséges, mert már megtalálható az archívumban.", + sourceFileName); - //Fel kell szabadítani, hogy a kovetkezo archivalaskor is nekifusson + // Fel kell szabadítani, hogy a kovetkezo archivalaskor is nekifusson if (existingMediaId == -1) { - logger.info(marker, "Az '{}' mentése jelenleg nem lehetséges, mert a szükséges metaadat még nem található meg az archívumban.", sourceFileName); + logger.info(marker, + "Az '{}' mentése jelenleg nem lehetséges, mert a szükséges metaadat még nem található meg az archívumban.", + sourceFileName); if (!archiveItem.removeCatchedFile()) logger.error(marker, "Az '{}' állomány .catched jelző állománya nem törölhető. Az újabb archiválási kísérlethez annak kézi eltávolítása szükséges!", sourceMediaFile.getName()); mediaCubeMedia.remove(); } else { - saveMetadata(mediaCubeMedia, sourceMediaFile, targetFileName, existingMediaId); + saveMetadata(mediaCubeMedia, sourceMediaFile, targetFileName, existingMediaId, fileSize, + archiveItem.isDisableProxy()); logger.info(marker, "Az '{}' archiválása sikeres.", sourceFileName); if (killDateDays != 0) EscortFiles.createUNCKillDate(sourceMediaFile.getParent(), sourceFileName, killDateDays, marker); @@ -118,7 +136,8 @@ public class TSMBackupStep extends JobStep { } catch (Exception e) { logger.catching(e); - Message m = new ParameterizedMessage("Az '{}' állomány archiválása sikertelen. A rendszer hibaüzenete: {}", details, e.getMessage()); + Message m = new ParameterizedMessage("Az '{}' állomány archiválása sikertelen. A rendszer hibaüzenete: {}", + details, e.getMessage()); logger.error(marker, m); if (!archiveItem.removeCatchedFile()) logger.error(marker, @@ -129,12 +148,16 @@ public class TSMBackupStep extends JobStep { return null; } - private void saveMetadata(Media mediaCubeMedia, File sourceFile, String targetFileName, long existingMediaId) { + private void saveMetadata(Media mediaCubeMedia, File sourceFile, String targetFileName, long existingMediaId, + long fileSize, boolean disableProxy) { if (existingMediaId == 0) { - MediaFile mediaFile = manager.createMediaFile(targetFileName, fileType, tsmStore, mediaCubeMedia); - mediaFile.setHouseId(sourceFile.getName()); - mediaFile.add(); + MediaFile mf = manager.createMediaFile(targetFileName, fileType, tsmStore, mediaCubeMedia); + mf.setHouseId(sourceFile.getName()); + mf.setFileSize(fileSize); + // 210617 proxy keszites tiltasa + mf.setDisableProxy(disableProxy); + mf.add(); } else { Media existingMedia = manager.getMedia(existingMediaId); List mediaFiles = existingMedia.getMediaFiles(); @@ -143,24 +166,31 @@ public class TSMBackupStep extends JobStep { mf.setPersister(manager); mf.setId(0); mf.setMedia(mediaCubeMedia); + // mivel itt masolat keszul, nem allitunk at semmit +// mf.setFileSize(fileSize); +// mf.setDisableProxy(disableProxy); mf.add(); } } } mediaCubeMedia.setPersister(manager); + // 210614 megis maradjon az aktualis idopont + mediaCubeMedia.setArchived(new Timestamp(new Date().getTime())); + + /* try { BasicFileAttributes attr = Files.readAttributes(sourceFile.toPath(), BasicFileAttributes.class); mediaCubeMedia.setArchived(new Timestamp(attr.creationTime().toMillis())); } catch (IOException e) { logger.catching(e); - mediaCubeMedia.setArchived(new Timestamp(new Date().getTime())); } - + */ mediaCubeMedia.modify(); } - private void setAndCheck(ArchiveItem archiveItem, Media mediaCubeMedia, IJobEngine jobEngine) throws JobEngineException, IOException { + private void setAndCheck(ArchiveItem archiveItem, Media mediaCubeMedia, IJobEngine jobEngine) + throws JobEngineException, IOException { if (jobEngine == null) { logger.error(marker, "Az folyamatkezelő réteg nem elérhető."); throw new NullPointerException("Internal error, missing JobEngine reference."); @@ -177,7 +207,8 @@ public class TSMBackupStep extends JobStep { sourceMediaFile = new File(archiveItem.getMediaFile()); if (sourceMediaFile == null) { logger.error(marker, "A folyamat 'archiveItem' bemeneti paraméter 'mediaFile' értéke üres."); - throw new NullPointerException("System is not configured properly, missing 'mediaFile' value in 'archiveItem' input parameter."); + throw new NullPointerException( + "System is not configured properly, missing 'mediaFile' value in 'archiveItem' input parameter."); } if (!sourceMediaFile.exists()) { logger.error(marker, "Az {} állomány nem létezik vagy nem érhető el.", sourceMediaFile.getName()); @@ -200,7 +231,8 @@ public class TSMBackupStep extends JobStep { } if (mediaCubeMedia == null) { logger.error(marker, "A folyamat 'mediaCubeMedia' bemeneti paramétere üres."); - throw new NullPointerException("System is not configured properly, 'mediaCubeMedia' input parameter missing."); + throw new NullPointerException( + "System is not configured properly, 'mediaCubeMedia' input parameter missing."); } } diff --git a/server/-product/production/MEDIAVIVANTIS/jobs/steps/TSMExtendedRetrieveStep.java b/server/-product/production/MEDIAVIVANTIS/jobs/steps/TSMExtendedRetrieveStep.java index 79a601ca..fd222344 100644 --- a/server/-product/production/MEDIAVIVANTIS/jobs/steps/TSMExtendedRetrieveStep.java +++ b/server/-product/production/MEDIAVIVANTIS/jobs/steps/TSMExtendedRetrieveStep.java @@ -10,6 +10,10 @@ import user.jobengine.db.IItemManager; import user.jobengine.server.IJobEngine; import user.jobengine.server.IJobRuntime; +/* +import user.jobengine.server.steps.EscortFiles; +*/ + public class TSMExtendedRetrieveStep extends TSMRestoreStep { private static String NEXIO_HOST = System.getProperty("nexio.host"); // private static String NEXIO_HOST = SystemConfiguration.getInstance().value("services.nexio.host"); diff --git a/server/-product/production/MEDIAVIVANTIS/jobs/steps/TSMRestoreStep.java b/server/-product/production/MEDIAVIVANTIS/jobs/steps/TSMRestoreStep.java index 6b40e3b7..26d8607b 100644 --- a/server/-product/production/MEDIAVIVANTIS/jobs/steps/TSMRestoreStep.java +++ b/server/-product/production/MEDIAVIVANTIS/jobs/steps/TSMRestoreStep.java @@ -28,6 +28,7 @@ import user.jobengine.db.MediaFile; import user.jobengine.db.Store; import user.jobengine.server.IJobEngine; import user.jobengine.server.IJobRuntime; +import user.jobengine.server.steps.shared.EscortFiles; public class TSMRestoreStep extends JobStep { private static final String DOT = "."; @@ -40,7 +41,7 @@ public class TSMRestoreStep extends JobStep { private Marker marker; protected void afterRestore(StoreUri targetUri, String targetPath, int killDateDays, String targetFileName) throws IOException, Exception { - if (killDateDays > 0) + if (killDateDays != 0) EscortFiles.createUNCKillDate(targetPath, targetFileName, killDateDays, marker); } @@ -87,8 +88,10 @@ public class TSMRestoreStep extends JobStep { }); RemoteFile result = sourceUri.transferFrom(targetUri, sourceFileName, targetFileName); - String globalTargetPath = Paths.get(targetPath, targetFileName).getParent().toString().replace(Paths.get(localRetrievePath).toString(), - globalRetrievePath); + String globalTargetPath = Paths.get(targetPath, targetFileName) + .getParent() + .toString() + .replace(Paths.get(localRetrievePath).toString(), globalRetrievePath); logger.info(marker, "Az '{}' állomány visszatöltése sikeres volt '{}' néven. A célmappa a ide kattintva nyitható meg.", diff --git a/server/-product/production/MEDIAVIVANTIS/jobs/steps/UpdateGhostMediaDataStep.java b/server/-product/production/MEDIAVIVANTIS/jobs/steps/UpdateGhostMediaDataStep.java index e59008d1..39cb7b30 100644 --- a/server/-product/production/MEDIAVIVANTIS/jobs/steps/UpdateGhostMediaDataStep.java +++ b/server/-product/production/MEDIAVIVANTIS/jobs/steps/UpdateGhostMediaDataStep.java @@ -13,7 +13,8 @@ import user.jobengine.db.MediaFile; import user.jobengine.db.Store; import user.jobengine.server.IJobEngine; import user.jobengine.server.IJobRuntime; -import user.jobengine.server.steps.MetadataTypeDetector.MetadataType; +import user.jobengine.server.steps.shared.MetadataType; +import user.jobengine.server.steps.shared.MetadataTypeDetector; public class UpdateGhostMediaDataStep extends JobStep { private static final Logger logger = LogManager.getLogger(); diff --git a/server/-product/production/MEDIAVIVANTIS/jobs/templates/archive-recursive.xml b/server/-product/production/MEDIAVIVANTIS/jobs/templates/archive-recursive.xml index 02b97884..5102e92c 100644 --- a/server/-product/production/MEDIAVIVANTIS/jobs/templates/archive-recursive.xml +++ b/server/-product/production/MEDIAVIVANTIS/jobs/templates/archive-recursive.xml @@ -1,10 +1,11 @@ - + + @@ -19,6 +20,9 @@ + + + diff --git a/server/-product/production/MEDIAVIVANTIS/jobs/templates/cancelable.xml b/server/-product/production/MEDIAVIVANTIS/jobs/templates/cancelable.xml index 2be289b3..fa29cc3f 100644 --- a/server/-product/production/MEDIAVIVANTIS/jobs/templates/cancelable.xml +++ b/server/-product/production/MEDIAVIVANTIS/jobs/templates/cancelable.xml @@ -1,5 +1,5 @@ - + diff --git a/server/-product/production/MEDIAVIVANTIS/jobs/templates/create-lowres-ondemand - Copy.xml b/server/-product/production/MEDIAVIVANTIS/jobs/templates/create-lowres-ondemand - Copy.xml new file mode 100644 index 00000000..f7922734 --- /dev/null +++ b/server/-product/production/MEDIAVIVANTIS/jobs/templates/create-lowres-ondemand - Copy.xml @@ -0,0 +1,106 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/server/-product/production/MEDIAVIVANTIS/jobs/templates/create-lowres-ondemand.xml b/server/-product/production/MEDIAVIVANTIS/jobs/templates/create-lowres-ondemand.xml index f7922734..51dcb626 100644 --- a/server/-product/production/MEDIAVIVANTIS/jobs/templates/create-lowres-ondemand.xml +++ b/server/-product/production/MEDIAVIVANTIS/jobs/templates/create-lowres-ondemand.xml @@ -1,105 +1,128 @@ - + + - - - - - - - - - - - + + + + + + + + + + + + - + + + + + + + + + + - + - + + + + - + + + + + + + - - + + - - - - - - - + - + - + - - - - + - - + + - + - + - - + + + + + + + + + + - + - + - + - - - - + + + + + - + - + - + + + + diff --git a/server/-product/production/MEDIAVIVANTIS/jobs/templates/delete-materials.xml b/server/-product/production/MEDIAVIVANTIS/jobs/templates/delete-materials.xml index 365fe19c..f3b653c9 100644 --- a/server/-product/production/MEDIAVIVANTIS/jobs/templates/delete-materials.xml +++ b/server/-product/production/MEDIAVIVANTIS/jobs/templates/delete-materials.xml @@ -3,6 +3,7 @@ + @@ -11,6 +12,9 @@ + + + diff --git a/server/-product/production/MEDIAVIVANTIS/jobs/templates/integration-test.xml b/server/-product/production/MEDIAVIVANTIS/jobs/templates/integration-test.xml new file mode 100644 index 00000000..6593e5d7 --- /dev/null +++ b/server/-product/production/MEDIAVIVANTIS/jobs/templates/integration-test.xml @@ -0,0 +1,6 @@ + + + + + + \ No newline at end of file diff --git a/server/-product/production/MEDIAVIVANTIS/jobs/templates/remote-transcode.xml b/server/-product/production/MEDIAVIVANTIS/jobs/templates/remote-transcode.xml index ce4c77b7..b5ea4ae1 100644 --- a/server/-product/production/MEDIAVIVANTIS/jobs/templates/remote-transcode.xml +++ b/server/-product/production/MEDIAVIVANTIS/jobs/templates/remote-transcode.xml @@ -1,5 +1,5 @@ - + @@ -26,6 +26,8 @@ + + @@ -76,7 +78,7 @@ - + @@ -89,6 +91,9 @@ + + + @@ -107,5 +112,25 @@ + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/server/-product/production/MEDIAVIVANTIS/jobs/templates/safe-delete-recursive.xml b/server/-product/production/MEDIAVIVANTIS/jobs/templates/safe-delete-recursive.xml new file mode 100644 index 00000000..bfa8e371 --- /dev/null +++ b/server/-product/production/MEDIAVIVANTIS/jobs/templates/safe-delete-recursive.xml @@ -0,0 +1,17 @@ + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/server/-product/production/MEDIAVIVANTIS/restart-mediacube.sh b/server/-product/production/MEDIAVIVANTIS/restart-mediacube.sh new file mode 100644 index 00000000..b4080f75 --- /dev/null +++ b/server/-product/production/MEDIAVIVANTIS/restart-mediacube.sh @@ -0,0 +1,4 @@ +./stop-mediacube.sh +./start-mediacube.sh + + diff --git a/server/-product/production/MEDIAVIVANTIS/settings/application.yaml b/server/-product/production/MEDIAVIVANTIS/settings/application.yaml index a50dcc2c..220883b9 100644 --- a/server/-product/production/MEDIAVIVANTIS/settings/application.yaml +++ b/server/-product/production/MEDIAVIVANTIS/settings/application.yaml @@ -17,6 +17,10 @@ datasource: url: jdbc:db2://10.11.1.89:51500/tsmdb1 user: tsminst1 password: tsminst1 + tsm: + url: jdbc:db2://10.11.1.90:52500/tsmdb1 + user: tsminst1 + password: passwordpassword planair: url: jdbc:sqlserver://10.11.254.86;databaseName=PA_Vivantis; user: MAM diff --git a/server/-product/production/MEDIAVIVANTIS/start-mediacube.sh b/server/-product/production/MEDIAVIVANTIS/start-mediacube.sh new file mode 100644 index 00000000..34db87ec --- /dev/null +++ b/server/-product/production/MEDIAVIVANTIS/start-mediacube.sh @@ -0,0 +1,28 @@ +cd /opt/mediacube +rm -fr tmp/* +rm -fr configuration/org.eclipse.osgi +export DSMI_CONFIG=/opt/mediacube/settings/dsm.opt +export DSMI_LOG=/opt/mediacube/log +export DSMI_DIR=/opt/tivoli/tsm/server/bin/dbbkapi +export DSM_DIR=/opt/tivoli/tsm/server/bin/dbbkapi +export LD_LIBRARY_PATH=/opt/tivoli/tsm/server/bin/dbbkapi:/opt/mediacube +export LIBPATH=/opt/tivoli/tsm/server/bin/dbbkapi +export SHLIB_PATH=/opt/tivoli/tsm/server/bin/dbbkapi +#export TZ=NFT-1DFT,M3.5.0,M10.5.0 +export PATH=/usr/lib/jvm/java-1.8.0-openjdk-1.8.0.102-4.b14.el7.x86_64/bin:$PATH +java \ +-DPRODMEDIACUBE \ +-Dclean \ +-Dorg.eclipse.epp.logging.aeri.skipReports=true \ +-Declipse.ignoreApp=true \ +-Dosgi.noShutdown=true \ +-Dlog4j.configurationFile=settings/log4j2.xml \ +-Djetty.home=settings \ +-Djetty.etc.config.urls=jetty.xml \ +-Dgosh.home=configuration \ +-Djava.io.tmpdir=tmp \ +-Dfile.encoding=UTF-8 \ +-jar plugins/org.eclipse.equinox.launcher_1.3.201.v20161025-1711.jar \ +-agentlib:jdwp=transport=dt_socket,server=y,suspend=n,address=8888 \ +-Xms512m \ +-Xmx4096m & diff --git a/server/-product/production/MEDIAVIVANTIS/stop-mediacube.sh b/server/-product/production/MEDIAVIVANTIS/stop-mediacube.sh new file mode 100644 index 00000000..f9e54a4b --- /dev/null +++ b/server/-product/production/MEDIAVIVANTIS/stop-mediacube.sh @@ -0,0 +1,3 @@ +pkill -f PRODMEDIACUBE +#wget -qO- http://localhost:9080/services/rest/jobengine/halt +