From: Omar Sweidan Date: Tue, 10 May 2022 10:51:47 +0000 (+0200) Subject: OMAR mappa hozzáadása git-hez X-Git-Url: http://git.useribm.hu/?a=commitdiff_plain;h=39126a305164cfb98ff4cef1ccc74efa258ccc31;p=mediacube.git OMAR mappa hozzáadása git-hez --- diff --git a/server/-configuration/run-mediacube-server-omar.launch b/server/-configuration/run-mediacube-server-omar.launch index 3d55a3c3..7e9aae74 100644 --- a/server/-configuration/run-mediacube-server-omar.launch +++ b/server/-configuration/run-mediacube-server-omar.launch @@ -6,17 +6,17 @@ - + - + - + @@ -31,7 +31,7 @@ - + diff --git a/server/-product/production/OMAR/configuration/etc/gosh_profile b/server/-product/production/OMAR/configuration/etc/gosh_profile new file mode 100644 index 00000000..1c659f49 --- /dev/null +++ b/server/-product/production/OMAR/configuration/etc/gosh_profile @@ -0,0 +1 @@ +prompt=mc> \ No newline at end of file diff --git a/server/-product/production/OMAR/jobs/executors.xml b/server/-product/production/OMAR/jobs/executors.xml new file mode 100644 index 00000000..0fd7a2f2 --- /dev/null +++ b/server/-product/production/OMAR/jobs/executors.xml @@ -0,0 +1,43 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/server/-product/production/OMAR/jobs/schedules.json b/server/-product/production/OMAR/jobs/schedules.json new file mode 100644 index 00000000..0134aa6b --- /dev/null +++ b/server/-product/production/OMAR/jobs/schedules.json @@ -0,0 +1,181 @@ +{ +"joblist":[ + + { + "name":"n1", + "template":"cancelable.xml", + "active":true, + "executeimmediate":true, + "parameters":[{"name":"param","value":1000,"type":"java.lang.Integer"}] + }, + + { + "name":"n2", + "template":"cancelable.xml", + "active":true, + "executeimmediate":true, + "parameters":[{"name":"param","value":1000,"type":"java.lang.Integer"}] + }, + + { + "name":"Limitált archiválás", + "template":"archive-limited.xml", + "cronexpression":"0 */10 * * * ?", + "parameters":[ + + { + "name":"sourcePath", + "value":"/opt/mediacube/ARCHIVE", + "type":"java.lang.String" + }, + + { + "name":"globalSourcePath", + "value":"\\\\10.11.1.90\\data\\ARCHIVE", + "type":"java.lang.String" + }, + + { + "name":"transcoderTargetPath", + "value":"/mnt/PROMISE/TRANSCODER/FFASTRANSCODER/Out", + "type":"java.lang.String" + }, + + { + "name":"killDateDays", + "value":-1, + "type":"java.lang.Integer" + }, + + { + "name":"limit", + "value":1000, + "type":"java.lang.Integer" + } + ] + }, + + { + "name":"SYS: batch-retrieve-ondemand", + "template":"batch-retrieve-ondemand.xml" + }, + + { + "template":"calculatemd5.xml", + "name":"MD5 kiszámítása", + "parameters":[ + + { + "name":"fileName", + "value":"C:\\Users\\machine\\Downloads\\sample.mxf", + "type":"java.lang.String" + } + ] + }, + + { + "template":"dummy-test-job.xml", + "name":"dummy test job", + "parameters":[{"name":"param1","value":"Jozsi","type":"java.lang.String"}], + "cronexpression":"0/3 * * * * ?" + }, + + { + "template":"move-jpeg-to-isilon.xml", + "name":"JPEG másolása ISILON-ba", + "parameters":[ + + { + "name":"sourceUri", + "value":"localhost", + "type":"java.lang.String" + }, + + { + "name":"sourceProtocol", + "value":"LOCAL", + "type":"java.lang.String" + }, + + { + "name":"sourceFolder", + "value":"c:\\data\\video", + "type":"java.lang.String" + }, + + { + "name":"targetUri", + "value":"localhost", + "type":"java.lang.String" + }, + + { + "name":"targetProtocol", + "value":"LOCAL", + "type":"java.lang.String" + }, + + { + "name":"targetFolder", + "value":"c:\\data\\video2", + "type":"java.lang.String" + }, + + { + "name":"userName", + "value":"dani", + "type":"java.lang.String" + }, + + { + "name":"password", + "value":"dani", + "type":"java.lang.String" + }, + + { + "name":"port", + "value":"21", + "type":"java.lang.Integer" + } + ] + }, + + { + "template":"prores-archive.xml", + "name":"ProRes archiválás", + "parameters":[ + + { + "name":"sourceStoreFolder", + "value":"c:\\data", + "type":"java.lang.String" + }, + + { + "name":"sourceStoreName", + "value":"AMC_LOCAL", + "type":"java.lang.String" + }, + + { + "name":"filter", + "value":{"fileName":".*\\.(mov)$"}, + "type":"java.lang.String" + }, + + { + "name":"targetStoreName", + "value":"AMC_LOCAL", + "type":"java.lang.String" + }, + + { + "name":"escortStoreFolder", + "value":"AMC_LOCAL", + "type":"java.lang.String" + } + ] + } + ] +} \ No newline at end of file diff --git a/server/-product/production/OMAR/jobs/steps/AnalyzeMediaFilesStep.java b/server/-product/production/OMAR/jobs/steps/AnalyzeMediaFilesStep.java new file mode 100644 index 00000000..98cf7fa9 --- /dev/null +++ b/server/-product/production/OMAR/jobs/steps/AnalyzeMediaFilesStep.java @@ -0,0 +1,28 @@ +package user.jobengine.server.steps; + +import java.nio.file.Paths; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + +import user.commons.mediaarea.MediaArea; + +public class AnalyzeMediaFilesStep extends JobStep { + private static final Logger logger = LogManager.getLogger(); + + @StepEntry + public Object[] execute(List foundFiles) { + Map proResFiles = new HashMap(foundFiles.size()); + + for (int i = 0; i < foundFiles.size(); i++) { + MediaArea mediaArea = new MediaArea(Paths.get(foundFiles.get(i))); + mediaArea.process(); + proResFiles.put(Paths.get(foundFiles.get(i)).toAbsolutePath().toString(), mediaArea); + } + + return new Object[] { proResFiles }; + } +} diff --git a/server/-product/production/OMAR/jobs/steps/ArchiveListBuilderStep.java b/server/-product/production/OMAR/jobs/steps/ArchiveListBuilderStep.java new file mode 100644 index 00000000..b2321829 --- /dev/null +++ b/server/-product/production/OMAR/jobs/steps/ArchiveListBuilderStep.java @@ -0,0 +1,167 @@ +package user.jobengine.server.steps; + +import java.io.File; +import java.io.IOException; +import java.nio.file.DirectoryStream; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.LinkedList; +import java.util.List; + +import org.apache.commons.lang.StringUtils; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.apache.logging.log4j.Marker; + +import user.jobengine.server.IJobEngine; +import user.jobengine.server.IJobRuntime; + +/** + * Az archivalhato mediak listazasa MediaFileWrapper objektumokban. A listazott media allomanyokat megjeloli .catched signal allomannyal, hogy a legkozelebbi + * listazas figyelmen kivul hagyja. + * + * @author robi + */ +public class ArchiveListBuilderStep extends JobStep { + private static final Logger logger = LogManager.getLogger(); + // private static final String UTF8 = "utf-8"; + private static final String STATUSFOLDER = ".STATUS"; + private static final String JSONEXT = ".json"; + private static final String CATCHEDEXT = ".catched"; + + public static final String ITEM_TITLE = "itemTitle"; + public static final String ITEM_HOUSEID = "itemHouseId"; + public static final String ITEM_DESCRIPTION = "itemDescription"; + public static final String MEDIA_HOUSEID = "mediaHouseId"; + public static final String MEDIA_TITLE = "mediaTitle"; + public static final String MEDIA_DESCRIPTION = "mediaDescription"; + public static final String MEDIA_TYPE = "mediaType"; + private static final String DURATION = "duration"; + private static final String EXISTING_MEDIAID = "existingMediaId"; + private static final String TAGS = "tags"; + + private Marker marker; + + private ArchiveItem createArchiveItem(Path jsonFilePath, Path mediaFilePath, Path catchedFilePath) { + ArchiveItem result = null; + try { + result = ArchiveItem.fromFile(jsonFilePath); + result.setMediaFile(mediaFilePath.toString()); + result.setCatchedFile(catchedFilePath.toString()); + } catch (Exception e) { + logger.catching(e); + } + + return result; + } + + private void createCatchedFile(Path catchedFilePath) { + try { + Files.createFile(catchedFilePath); + //Files.write(catchedFilePath, CATCHED.getBytes(UTF8), StandardOpenOption.CREATE); + } catch (Exception e) { + logger.catching(e); + } + } + + @StepEntry + public Object[] execute(String sourcePath, int limit, IJobEngine jobEngine, IJobRuntime jobRuntime) { + marker = jobRuntime.getSessionMarker(); + List archiveList = new LinkedList(); + DirectoryStream directoryStream = null; + try { + DirectoryStream stream = Files.newDirectoryStream(Paths.get(sourcePath)); + for (Path p : stream) { + processPathItem(p, archiveList); + } + } catch (Exception e) { + logger.catching(e); + logger.error(marker, "Az '{}' mappa elérése sikertelen. A rendszer hibaüzenete: {}", e.getMessage()); + } finally { + if (directoryStream != null) { + try { + directoryStream.close(); + } catch (IOException e) { + } + } + } + + if (limit > 0 && archiveList.size() > limit) { + archiveList = archiveList.subList(0, limit); + logger.info(marker, "A folyamat alkalmazza a beállított {} limitet.", limit); + } + + if (archiveList.size() == 0) + logger.info(marker, "Nincs archiválandó anyag."); + else + logger.info(marker, "Az archiváló folyamat {} új anyagot érzékelt.", archiveList == null ? 0 : archiveList.size()); + + for (ArchiveItem archiveItem : archiveList) { + createCatchedFile(Paths.get(archiveItem.getCatchedFile())); + } + + return new Object[] { archiveList }; + } + + private boolean processPathItem(Path mediaFilePath, final List archiveList) { + File mediaFile = mediaFilePath.toFile(); + + // if (mediaFile.length() > 0) + // return false; + + if (mediaFile.isDirectory()) { + return false; + } + + Path dotStorePath = Paths.get(mediaFilePath.getParent().toString(), STATUSFOLDER); + Path catchedFilePath = Paths.get(dotStorePath.toString(), mediaFile.getName() + CATCHEDEXT); + File catchedFile = catchedFilePath.toFile(); + if (catchedFile.exists()) { + logger.warn("{} file is already catched.", mediaFile.getName()); + return false; + } + + Path jsonFilePath = Paths.get(dotStorePath.toString(), mediaFile.getName() + JSONEXT); + File jsonFile = jsonFilePath.toFile(); + if (!jsonFile.exists()) { + logger.warn("{} has no json metadata.", mediaFile.getName()); + return false; + } + + ArchiveItem archiveItem = createArchiveItem(jsonFilePath, mediaFilePath, catchedFilePath); + + if (archiveItem == null) { + logger.warn("{} has no metadata specified.", mediaFile.getName()); + return false; + } + + if (StringUtils.isBlank(archiveItem.getItemHouseId())) { + logger.warn("{} has no Item HouseID specified in metadata.", mediaFile.getName()); + return false; + } + + if (StringUtils.isBlank(archiveItem.getItemTitle())) { + logger.warn("{} has no Item Title specified in metadata.", mediaFile.getName()); + return false; + } + + if (StringUtils.isBlank(archiveItem.getMediaHouseId())) { + logger.warn("{} has no Media HouseID specified in metadata.", mediaFile.getName()); + return false; + } + + if (StringUtils.isBlank(archiveItem.getMediaTitle())) { + logger.warn("{} has no Media Title specified in metadata.", mediaFile.getName()); + return false; + } + + //A tenyleges archivalast vesszuk elore + if (mediaFile.length() == 0) + archiveList.add(archiveItem); + else + archiveList.add(0, archiveItem); + //createCatchedFile(catchedFilePath); + return true; + } +} diff --git a/server/-product/production/OMAR/jobs/steps/ArchiveMaterialSubmitStep.java b/server/-product/production/OMAR/jobs/steps/ArchiveMaterialSubmitStep.java new file mode 100644 index 00000000..0aedbbe8 --- /dev/null +++ b/server/-product/production/OMAR/jobs/steps/ArchiveMaterialSubmitStep.java @@ -0,0 +1,52 @@ +package user.jobengine.server.steps; + +import java.io.File; +import java.util.List; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.apache.logging.log4j.Marker; + +import user.commons.ListUtils; +import user.jobengine.server.IJobEngine; +import user.jobengine.server.IJobRuntime; + +public class ArchiveMaterialSubmitStep extends JobStep { + private static final Logger logger = LogManager.getLogger(); + private static final String JOBTEMPLATE = "archive-material.xml"; + private static final String KILL_DATE_DAYS = "killDateDays"; + private static final String ARCHIVE = "Archiválás"; + private static final String ARCHIVE_ITEM = "archiveItem"; + private Marker marker; + + @StepEntry + public Object[] execute(List archiveList, int killDateDays, IJobEngine jobEngine, IJobRuntime jobRuntime) throws Exception { + marker = jobRuntime.getSessionMarker(); + if (archiveList == null || archiveList.size() == 0) + return null; + + if (jobRuntime.forkPrepare()) { + for (int i = 0; i < archiveList.size(); i++) { + ArchiveItem archiveItem = archiveList.get(i); + try { + IJobRuntime runtime = jobEngine.submit(jobRuntime, null, JOBTEMPLATE, ARCHIVE, + ListUtils.asMap(ARCHIVE_ITEM, archiveItem, KILL_DATE_DAYS, killDateDays)); + int progress = (i + 1) * 100 / archiveList.size(); + setProgress(progress); + //TODO kivezetni a submit hibaüzenetet + if (runtime == null) + throw new Exception("Submit returned null runtime"); + } catch (Exception e) { + logger.catching(e); + String fileName = new File(archiveItem.getMediaFile()).getName(); + logger.error(marker, "A(z) '{}' állomány archiválási kísérlete sikertelen. A rendszer üzenete: {}", fileName, e.getMessage()); + if (!archiveItem.removeCatchedFile()) + logger.error(marker, "A(z) '{}' állomány .catched jelző állománya nem törölhető.", fileName); + throw e; + } + } + } + jobRuntime.forkWaitComplete(); + return null; + } +} diff --git a/server/-product/production/OMAR/jobs/steps/ArchiveRecursive.java b/server/-product/production/OMAR/jobs/steps/ArchiveRecursive.java new file mode 100644 index 00000000..3f01f32b --- /dev/null +++ b/server/-product/production/OMAR/jobs/steps/ArchiveRecursive.java @@ -0,0 +1,324 @@ +package user.jobengine.server.steps; + +import java.io.File; +import java.io.IOException; +import java.nio.file.FileVisitResult; +import java.nio.file.FileVisitor; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.nio.file.attribute.BasicFileAttributes; +import java.text.SimpleDateFormat; +import java.util.Arrays; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; + +import org.apache.commons.io.FilenameUtils; +import org.apache.commons.lang.StringUtils; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.apache.logging.log4j.message.Message; +import org.apache.logging.log4j.message.ParameterizedMessage; + +import com.ibm.nosql.json.api.BasicDBObject; + +import user.commons.JobStatus; +import user.commons.ListUtils; +import user.commons.log4j2.marker.MediaCubeMarker; +import user.commons.mediatool.MediaInfo; +import user.jobengine.server.IJobEngine; +import user.jobengine.server.IJobRuntime; +import user.jobengine.server.steps.shared.EscortFiles; +import user.mediacube.metadata.interfaces.IMetadata; +import user.mediacube.metadata.interfaces.IMetadataProvider; +import user.mediacube.metadata.interfaces.IMetadataProviderFactory; +import user.mediacube.metadata.interfaces.MetadataProviderType; +import user.mediacube.metadata.interfaces.MetadataType; +import user.mediacube.metadata.interfaces.PlanAirMetadataListOptions; + +public class ArchiveRecursive extends JobStep implements FileVisitor { + private static final Logger logger = LogManager.getLogger(); + private static final String JOBTEMPLATE = "archive-material.xml"; + private static final String ITEM_TITLE = "itemTitle"; + private static final String ITEM_HOUSEID = "itemHouseId"; + private static final String MEDIA_HOUSEID = "mediaHouseId"; + private static final String MEDIA_TITLE = "mediaTitle"; + private static final String MEDIA_DESCRIPTION = "mediaDescription"; + private static final String MEDIA_TYPE = "mediaType"; + private static final String ARCHIVE = "Archiválás"; + private static final String ARCHIVE_ITEM = "archiveItem"; + private static final String KILL_DATE_DAYS = "killDateDays"; + private SimpleDateFormat df = new SimpleDateFormat("yyyy.MM.dd"); + + private List skipPathNames = Arrays.asList("!ARCHIVALAS_ALATT", EscortFiles.STATUSFOLDER, + EscortFiles.CONFLICTFOLDER); + private int limit; + private int submitted; + private int killDateDays; + private boolean disableProxy; + + private boolean canReadMediaInfo(Path mediaFilePath) { + boolean result = false; + try { + MediaInfo mi = new MediaInfo(mediaFilePath); + mi.process(); + result = true; + } catch (Exception e) { + logger.warn(getSessionMarker(), e.getMessage()); + } + return result; + } + + private void checkArchiveItem(ArchiveItem archiveItem) throws Exception { + if (archiveItem == null) + throw new Exception("No metadata specified."); + + if (StringUtils.isBlank(archiveItem.getItemHouseId())) + throw new Exception("No Item HouseID specified in metadata."); + + if (StringUtils.isBlank(archiveItem.getItemTitle())) + throw new Exception("No Item Title specified in metadata."); + + if (StringUtils.isBlank(archiveItem.getMediaHouseId())) + throw new Exception("No Media HouseID specified in metadata."); + + if (StringUtils.isBlank(archiveItem.getMediaTitle())) + throw new Exception("No Media Title specified in metadata."); + } + + private String archiveItemJSON(ArchiveItem result) { + BasicDBObject obj = new BasicDBObject(); + obj.put("itemHouseId", result.getItemHouseId()); + obj.put("itemTitle", result.getItemTitle()); + obj.put("mediaHouseId", result.getMediaHouseId()); + obj.put("mediaTitle", result.getMediaTitle()); + obj.put("mediaDescription", result.getMediaDescription()); + obj.put("mediaType", result.getMediaType()); + return obj.toPrettyString(""); + } + + private ArchiveItem createArchiveItem(Path filePath) throws Exception { + ArchiveItem result = null; + + String fileName = filePath.getFileName().toString(); + String mediaHouseId = FilenameUtils.removeExtension(fileName); + try { + result = getPlanAirMetadata(mediaHouseId); + + if (result != null) { + result.setMediaFile(filePath.toString()); + // 210617 proxy keszites tiltasa + result.setDisableProxy(disableProxy); + } + + logger.info(getSessionMarker(), "PlanAir query done for {}", filePath); + } catch (Exception e) { + logger.error(getSessionMarker(), "PlanAir metadata error", e); + // nem latja a drivert pl. + throw e; + } + +//210616 Ha nincs metaadat, nem archivalunk + if (result == null) { + result = new ArchiveItem(); + BasicFileAttributes attr = Files.readAttributes(filePath, BasicFileAttributes.class); + result.setItemHouseId(df.format(attr.lastModifiedTime().toMillis())); + result.setItemTitle(filePath.getParent().toString()); + result.setMediaHouseId(mediaHouseId); + result.setMediaTitle(fileName); + result.setMediaDescription("/ARCHIVE-TEST"); + result.setMediaType("Generic"); + result.setMediaFile(filePath.toString()); + } + + return result; + } + + @StepEntry + public Object[] execute(String sourcePath, int killDateDays, int limit, boolean disableProxy) throws Exception { + this.killDateDays = killDateDays; + this.limit = limit; + this.disableProxy = disableProxy; + logger.info(getSessionMarker(), "Starting in {}", sourcePath); + try { + if (getJobRuntime().forkPrepare()) { + Files.walkFileTree(Paths.get(sourcePath), this); + } + } catch (Exception e) { + logger.error(getSessionMarker(), "Az '{}' mappa elérése sikertelen. A rendszer hibaüzenete: {}", sourcePath, + e.getMessage()); + } finally { + if (submitted > 0) + getJobRuntime().forkWaitComplete(); + else + getJobRuntime().cancelForkPrepare(); + } + return null; + } + + private ArchiveItem getPlanAirMetadata(String mediaHouseId) throws Exception { + PlanAirMetadataListOptions opt = new PlanAirMetadataListOptions(); + opt.setSearch(mediaHouseId); + opt.setType(MetadataType.Material); + + BasicDBObject json = null; + List data = null; + + IMetadataProviderFactory factory = getService(IMetadataProviderFactory.class); + if (factory == null) + logger.info(getSessionMarker(), "IMetadataProviderFactory is null"); + + IMetadataProvider planairProvider = factory.getProvider(MetadataProviderType.PLANAIR); + if (planairProvider == null) + logger.info(getSessionMarker(), "IMetadataProvider is null"); + + ArchiveItem result = null; + data = planairProvider.list(opt); + if (data.size() != 0) + json = data.get(0).asJSON(); + else { + opt.setType(MetadataType.Promo); + data = planairProvider.list(opt); + if (data.size() != 0) + json = data.get(0).asJSON(); + else { + opt.setType(MetadataType.AD); + data = planairProvider.list(opt); + if (data.size() != 0) + json = data.get(0).asJSON(); + } + } + if (json != null) { + result = new ArchiveItem(); + result.setItemHouseId(json.getString(ITEM_HOUSEID)); + result.setItemTitle(json.getString(ITEM_TITLE)); + result.setMediaHouseId(json.getString(MEDIA_HOUSEID)); + result.setMediaTitle(json.getString(MEDIA_TITLE)); + result.setMediaDescription(json.getString(MEDIA_DESCRIPTION)); + result.setMediaType(json.getString(MEDIA_TYPE)); + } + return result; + } + + private boolean handleArchiveConflict(Path mediaPath) throws Exception { + boolean result = false; + String sourceFileName = mediaPath.getFileName().toString(); + if (getManager().isMediaFileExists(sourceFileName)) { + EscortFiles.createMediaCatch(mediaPath); + result = true; + } + return result; + } + + @Override + public FileVisitResult postVisitDirectory(Path paramT, IOException paramIOException) throws IOException { + return FileVisitResult.CONTINUE; + } + + @Override + public FileVisitResult preVisitDirectory(Path dir, BasicFileAttributes paramBasicFileAttributes) + throws IOException { + Path dirName = dir.getFileName(); + + if (skipPathNames.contains(dirName.toString())) { + logger.info(getSessionMarker(), "PreVisit skip {}", dir); + return FileVisitResult.SKIP_SUBTREE; + } else + logger.info(getSessionMarker(), "PreVisit {}", dir); + + return FileVisitResult.CONTINUE; + } + + private boolean processPathItem(Path mediaPath) throws Exception { + if (limit != 0 && submitted == limit) { + logger.info(getSessionMarker(), "Limit reached {}, canceling", limit); + return false; + } + + String fileName = mediaPath.getFileName().toString(); + if (fileName.startsWith(".") || fileName.endsWith(".nomd")) + return false; + + logger.info(getSessionMarker(), "Processing {}", mediaPath); + + File mediaFile = mediaPath.toFile(); + if (mediaFile.isDirectory()) { + //logger.info(getSessionMarker(), "Skipping directory {}", mediaPath); + return false; + } + + if (EscortFiles.isMediaCatched(mediaPath)) { + //logger.info(getSessionMarker(), "Skipping already catched {}", mediaPath); + return false; + } + + Path nomdFile = Paths.get(mediaPath.toString() + ".nomd"); + + if (Files.exists(nomdFile)) { + //logger.info(getSessionMarker(), "Skipping nomd file exists {}", nomdFile); + return false; + } + + if (handleArchiveConflict(mediaPath)) { + logger.info(getSessionMarker(), "Skipping archive db already contains {}", mediaPath); + return false; + } + + if (!canReadMediaInfo(mediaPath)) { + logger.info(getSessionMarker(), "Skipping cant read mediainfo {}", mediaPath); + return false; + } + + ArchiveItem archiveItem = createArchiveItem(mediaPath); + + if (archiveItem == null) { + Message msg = new ParameterizedMessage("Nincs metaadat!"); + logger.info(new MediaCubeMarker("vasary@elgekko.net,muszak@mediavivantis.hu", + "Értesítés problémás " + mediaPath.getFileName().toString() + " archiválásról"), msg); + Files.createFile(nomdFile); + return false; + } + + try { + checkArchiveItem(archiveItem); + + Map parameters = ListUtils.asMap(ARCHIVE_ITEM, archiveItem, KILL_DATE_DAYS, killDateDays); + IJobRuntime runtime = getEngine().submit(getJobRuntime(), e -> { + if (e.getStatus().equals(JobStatus.CANCELED) || e.getStatus().equals(JobStatus.SUSPENDED)) + EscortFiles.removeMediaCatch(mediaPath); + }, JOBTEMPLATE, ARCHIVE, 1, IJobEngine.DEFAULT_OWNER, parameters); + if (runtime == null) + throw new Exception("Submit returned null runtime"); + runtime.setRelated(mediaPath.toString()); + EscortFiles.createMediaCatch(mediaPath); + String metadata = archiveItemJSON(archiveItem); + EscortFiles.createMetadata(mediaPath.getParent().toString(), mediaPath.getFileName().toString(), metadata); + submitted++; + } catch (Exception e) { + logger.error(getSessionMarker(), + "A(z) '{}' állomány archiválási kísérlete sikertelen. A rendszer üzenete: {}", mediaPath, + e.getMessage()); + } + + return true; + } + + @Override + public FileVisitResult visitFile(Path filePath, BasicFileAttributes paramBasicFileAttributes) throws IOException { + logger.info(getSessionMarker(), "Will checked {}", filePath); + try { + processPathItem(filePath); + } catch (Exception e) { + logger.catching(e); + } + return FileVisitResult.CONTINUE; + } + + @Override + public FileVisitResult visitFileFailed(Path filePath, IOException paramIOException) throws IOException { + logger.info("Error archive {}", filePath); + return FileVisitResult.CONTINUE; + } +} diff --git a/server/-product/production/OMAR/jobs/steps/BatchRetrieveForkStep.java b/server/-product/production/OMAR/jobs/steps/BatchRetrieveForkStep.java new file mode 100644 index 00000000..4f6b130b --- /dev/null +++ b/server/-product/production/OMAR/jobs/steps/BatchRetrieveForkStep.java @@ -0,0 +1,76 @@ +package user.jobengine.server.steps; + +import java.util.List; +import java.util.Map; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + +import user.commons.log4j2.marker.MediaCubeFinishMarker; +import user.commons.log4j2.marker.MediaCubeMarker; +import user.jobengine.db.ArchivedMedia; +import user.jobengine.server.IJobEngine; +import user.jobengine.server.IJobRuntime; +import user.jobengine.server.scheduler.ScheduledJob; + +public class BatchRetrieveForkStep extends JobStep { + private static final String TARGET_PATH_TYPE = "targetPathType"; + private static final Logger logger = LogManager.getLogger(); + private static final String CHILD_TEMPLATE = "retrieve-ondemand.xml"; + private static final String ARCHIVEDMEDIA = "archivedMedia"; + private static final String RECIPIENT = "successRecipient"; + private static final String HOUSEID = "houseId"; + private MediaCubeMarker marker; + + @StepEntry + public Object[] execute(List basket, String houseId, String recipient, String targetPathType, IJobEngine jobEngine, IJobRuntime jobRuntime) + throws Exception { + marker = (MediaCubeMarker) jobRuntime.getSessionMarker(); + + //session szinten csak a finishMarker cimzettje az erdekes, es ezt a cimet pluszban hasznalja a konfigban megadott cimmel + //a finishMarker orokli a cim bellitast a sessionMarkertol + marker.setTo(recipient); + + ((MediaCubeMarker) jobRuntime.getFinishMarker()).setTo(recipient); + + if (basket == null || basket.size() == 0) + return null; + setProgress(10); + + MediaCubeMarker mailMarker = new MediaCubeMarker(recipient); + mailMarker.setSessionName("Archívum viszatöltés"); + mailMarker.setSessionID(houseId); + logger.info(mailMarker, "A visszatöltések elindultak az alábbi állományokra:"); + + if (jobRuntime.forkPrepare()) { + for (ArchivedMedia archivedMedia : basket) { + logger.info(mailMarker, archivedMedia.getMedia().getMediaFilesName()); + submit(archivedMedia, recipient, houseId, targetPathType, jobEngine, jobRuntime); + } + } + setProgress(50); + logger.info(new MediaCubeFinishMarker(mailMarker), "A visszatöltések végeztével megerősítő üzenetet küldünk."); + jobRuntime.forkWaitComplete(); + setProgress(100); + return null; + } + + public void submit(ArchivedMedia archivedMedia, String recipient, String houseId, String targetPathType, IJobEngine jobEngine, IJobRuntime jobRuntime) + throws Exception { + try { + ScheduledJob scheduledJob = jobEngine.getScheduledJob(CHILD_TEMPLATE); + Map parameters = scheduledJob.getJobParameters(); + parameters.put(ARCHIVEDMEDIA, archivedMedia); + parameters.put(HOUSEID, houseId); + parameters.put(RECIPIENT, recipient); + parameters.put(TARGET_PATH_TYPE, targetPathType); + IJobRuntime child = jobEngine.submit(jobRuntime, null, CHILD_TEMPLATE, String.format("Visszatöltés %s részére", recipient), parameters); + ((MediaCubeMarker) child.getSessionMarker()).setTo(recipient); + } catch (Exception e) { + logger.catching(e); + logger.error(marker, "Hiba a kötegelt visszatöltésben. A rendszer üzenete: {}", e.getMessage()); + } + + } + +} diff --git a/server/-product/production/OMAR/jobs/steps/CalculateMD5Step.java b/server/-product/production/OMAR/jobs/steps/CalculateMD5Step.java new file mode 100644 index 00000000..9db442c7 --- /dev/null +++ b/server/-product/production/OMAR/jobs/steps/CalculateMD5Step.java @@ -0,0 +1,34 @@ +package user.jobengine.server.steps; + +import java.io.InputStream; +import java.nio.file.Files; +import java.nio.file.Paths; +import java.security.DigestInputStream; +import java.security.MessageDigest; + +import javax.xml.bind.DatatypeConverter; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + +public class CalculateMD5Step extends JobStep { + private static final Logger logger = LogManager.getLogger(CalculateMD5Step.class); + + @StepEntry + public Object[] execute(String fileName) throws Exception { + logger.info(getMarker(), "Executing"); + + MessageDigest messageDigest = MessageDigest.getInstance("MD5"); + InputStream is = Files.newInputStream(Paths.get(fileName)); + DigestInputStream dis = new DigestInputStream(is, messageDigest); + byte[] digest = new byte[32768]; //32k buffer + + while ((dis.read(digest)) != -1) { + messageDigest.update(digest); + } + + String md5String = DatatypeConverter.printHexBinary(digest).toUpperCase(); + logger.info("calculated MD5 hash= {}", md5String); + return new Object[] { md5String }; + } +} \ No newline at end of file diff --git a/server/-product/production/OMAR/jobs/steps/CancelableStep.java b/server/-product/production/OMAR/jobs/steps/CancelableStep.java new file mode 100644 index 00000000..9293427a --- /dev/null +++ b/server/-product/production/OMAR/jobs/steps/CancelableStep.java @@ -0,0 +1,42 @@ +package user.jobengine.server.steps; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + +public class CancelableStep extends JobStep { + private static final Logger logger = LogManager.getLogger(); + int count = 10; + + @StepEntry + public Object[] execute(int param) throws Exception { + + getJobRuntime().setRelated("TESZT" + param); + for (int i = 0; i < count; i++) { + if (getJobRuntime().isWaitingCancel()) + break; +// if (i == 1) +// throw new Exception("AAAAAAAAAA"); + + Thread.sleep(1000); + int progress = (i + 1) * 100 / count; + setProgress(progress); + } + +// try { +// +// logger.warn("Ez a fo logba megy"); +// +// //marker="MEDIACUBE" +// logger.warn(getMarker(), "Ez a markered logba megy"); +// +// //marker="MEDIACUBE | folyamat_nev" +// logger.warn(getJobRuntime().getSessionMarker(), "Ez a markered logba megy es a nevesitett logba"); +// +// } catch (Exception e) { +// e.printStackTrace(); +// throw e; +// } + return new Object[] { param }; + } + +} diff --git a/server/-product/production/OMAR/jobs/steps/CleanupMountedLocationStep.java b/server/-product/production/OMAR/jobs/steps/CleanupMountedLocationStep.java new file mode 100644 index 00000000..95fff73f --- /dev/null +++ b/server/-product/production/OMAR/jobs/steps/CleanupMountedLocationStep.java @@ -0,0 +1,283 @@ +package user.jobengine.server.steps; + +import java.io.File; +import java.io.IOException; +import java.nio.file.DirectoryStream; +import java.nio.file.FileVisitResult; +import java.nio.file.FileVisitor; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.nio.file.SimpleFileVisitor; +import java.nio.file.attribute.BasicFileAttributes; +import java.text.ParseException; +import java.text.SimpleDateFormat; +import java.util.ArrayList; +import java.util.Collections; +import java.util.Date; +import java.util.List; + +import org.apache.commons.io.FileUtils; +import org.apache.commons.lang.StringUtils; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.apache.logging.log4j.Marker; + +import user.jobengine.server.steps.shared.ItemManagerExtensions; + +public class CleanupMountedLocationStep extends JobStep implements FileVisitor { + + private static final Logger logger = LogManager.getLogger(); + private static final String PROJECTFOLDER = "PROJECT"; + private static final String DATEFORMAT = "yyyyMMdd"; + private static final String DOT = "."; + private static final String STATUSFOLDER = ".STATUS"; + private static final String EWC2EXT = ".ewc2"; + private static final String XMPEXT = ".xmp"; + private static final String CATCHEDEXT = ".catched"; + private static final String KILLDATEEXT = ".killdate"; + private static final String JSONEXT = ".json"; + + private static boolean isEmpty(final Path directory) throws IOException { + try (DirectoryStream dirStream = Files.newDirectoryStream(directory)) { + final int[] count = new int[] { 0 }; + final int[] specialCount = new int[] { 0 }; + dirStream.forEach(p -> { + count[0]++; + // if (p.getFileName().toString().toLowerCase().equals(PROJECTFOLDER.toLowerCase())) + // specialCount[0]++; + if (p.getFileName().toString().toLowerCase().equals(STATUSFOLDER.toLowerCase())) + specialCount[0]++; + + }); + if (specialCount[0] == count[0]) + return true; + } + return false; + } + + private Marker marker; + + final int[] allCount = new int[] { 0 }; + final int[] currentCount = new int[] { 0 }; + + private Path sourcePath; + private SimpleDateFormat dateFormat; + private boolean skipArchiveCheck; + + private Date checkExpiration(List killDateFiles) { + Date killDate = null; + for (Path killDateFile : killDateFiles) { + Date currentKillDate = getKillDate(killDateFile); + if (currentKillDate == null) + continue; + if ((killDate != null && currentKillDate.after(killDate)) || killDate == null) + killDate = currentKillDate; + } + return new Date().after(killDate) ? killDate : null; + } + + @StepEntry + public Object[] execute(String sourceFolder, boolean skipArchiveCheck) throws Exception { + this.skipArchiveCheck = skipArchiveCheck; + marker = getSessionMarker(); + sourcePath = Paths.get(sourceFolder); + DirectoryStream directoryStream = null; + if (StringUtils.isBlank(sourcePath.toString())) { + logger.error(marker, "A folyamat 'sourcePath' bemeneti paramétere üres."); + throw new NullPointerException("System is not configured properly, 'sourceFolder' input parameter missing."); + } + + if (!Files.exists(sourcePath) || !Files.isDirectory(sourcePath)) { + logger.error(marker, "A {} mappa nem létezik.", sourceFolder); + throw new NullPointerException(String.format("Directory %s does not exist.", sourceFolder)); + } + + try { + setProgress(1); + dateFormat = new SimpleDateFormat(DATEFORMAT); + + Files.walkFileTree(sourcePath, new SimpleFileVisitor() { + @Override + public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) throws IOException { + allCount[0]++; + return super.visitFile(file, attrs); + } + }); + Files.walkFileTree(sourcePath, this); + } catch (Exception e) { + logger.catching(e); + logger.error(marker, "Hiba a '{}' mappa feldolgozásában. A rendszer hibaüzenete: {}", sourcePath, e.getMessage()); + throw e; + } finally { + if (directoryStream != null) { + try { + directoryStream.close(); + } catch (IOException e) { + } + } + } + return null; + } + + private Date getKillDate(Path killDateFile) { + String fileName = killDateFile.getFileName().toString(); + int end = fileName.lastIndexOf(DOT); + if (end < 1) + return null; + int start = fileName.lastIndexOf(DOT, end - 1); + if (start < 0) + return null; + String strKillDate = fileName.substring(start + 1, end); + Date result = null; + if (StringUtils.isNumeric(strKillDate)) { + try { + result = dateFormat.parse(strKillDate); + } catch (ParseException e) { + logger.error(marker, "A {} fájl 'killdate' állománya hibás formátumú, a {} karaktersorozat nem konvertálható dátummá.", fileName, strKillDate); + return null; + } + } else + logger.error(marker, "A {} fájl 'killdate' állománya hibás formátumú, az dátum helyett ez áll: '{}'.", fileName, strKillDate); + return result; + } + + private List getKillDateFiles(Path filePath) { + String killDateFilePattern = String.format("%s.*%s", filePath.getFileName().toString(), KILLDATEEXT); + List result = new ArrayList<>(); + Path statusPath = null; + try { + statusPath = Paths.get(filePath.getParent().toString(), STATUSFOLDER); + } catch (Exception e) { + logger.catching(e); + return null; + } + File statusPathFile = statusPath.toFile(); + if (statusPathFile.exists() && statusPathFile.isDirectory()) { + try (DirectoryStream stream = Files.newDirectoryStream(statusPath, killDateFilePattern)) { + stream.forEach(p -> result.add(p)); + } catch (Exception e) { + logger.catching(e); + } + } + Collections.sort(result); + return result; + } + + @Override + public FileVisitResult postVisitDirectory(Path dir, IOException exc) throws IOException { + if (!dir.equals(sourcePath) && isEmpty(dir)) { + if (!removeExistingSpecialDirectory(dir, PROJECTFOLDER)) + return FileVisitResult.CONTINUE; + if (!removeExistingSpecialDirectory(dir, STATUSFOLDER)) + return FileVisitResult.CONTINUE; + if (removeFile(dir)) + logger.info(marker, "A {} üres mappa törlése sikeres.", dir); + + } + return FileVisitResult.CONTINUE; + } + + @Override + public FileVisitResult preVisitDirectory(Path dir, BasicFileAttributes attrs) throws IOException { + + //A .-al kezdodo mappakat kihagyjuk + if (dir.getFileName().toString().startsWith(".")) + return FileVisitResult.SKIP_SUBTREE; + + return FileVisitResult.CONTINUE; + } + + private void processPathItem(Path filePath) { + currentCount[0]++; + + if (filePath.getFileName().toString().startsWith(".")) + return; + + int progress = currentCount[0] * 100 / allCount[0]; + setProgress(progress); + + logger.info("Checking {}", filePath); + List killDateFiles = getKillDateFiles(filePath); + if (killDateFiles == null || killDateFiles.size() == 0) { + logger.warn(marker, "A {} fájlhoz nem található 'killdate' állomány.", filePath); + return; + } + + if (killDateFiles.size() != 1) + logger.warn(marker, "A {} fájlhoz több 'killdate' állomány található, a legújabb dátum határozza meg a törlés időpontját.", filePath); + + Date killDate = checkExpiration(killDateFiles); + if (killDate == null) + return; + + if (!skipArchiveCheck && filePath.toFile().length() > 0) { + if (!ItemManagerExtensions.isArchived(getManager(), filePath)) { + logger.error(marker, "A(z) {} anyag törlésre van kijelölve, de nem található az archívumban.", filePath); + return; + } + } + + if (removeFiles(filePath, killDateFiles)) + logger.info(marker, "A {} fájl és kapcsolódó állományai a {} killdate bejegyzés alapján sikeresen törlődtek.", filePath.getFileName(), + dateFormat.format(killDate)); + else + logger.warn(marker, "A {} fájl és kapcsolódó állományai a {} killdate bejegyzés alapján csak részlegesen vagy egyáltalán nem törlődtek.", + filePath.getFileName(), dateFormat.format(killDate)); + } + + private boolean removeExistingSpecialDirectory(Path dir, String folderName) throws IOException { + File projectPath = Paths.get(dir.toString(), folderName).toFile(); + if (projectPath.exists() && projectPath.isDirectory()) { + FileUtils.deleteDirectory(projectPath); + if (projectPath.exists()) { + logger.warn(marker, "A {} alatti {} mappa törlése nem sikerült.", dir, folderName); + return false; + } + } + return true; + } + + private boolean removeFile(Path filePath) { + boolean result = false; + try { + //logger.error("REMOVE {}", filePath); + File file = filePath.toFile(); + if (file.exists()) + result = file.delete(); + } catch (Exception e) { + logger.error(marker, "A {} fájl nem törölhető. A rendszer hibaüzenete: {}", filePath, e.getMessage()); + } + return result; + } + + private boolean removeFiles(Path filePath, List killDateFiles) { + if (!removeFile(filePath)) + return false; + + removeFile(Paths.get(filePath.toString() + EWC2EXT)); + removeFile(Paths.get(filePath.toString() + XMPEXT)); + removeFile(Paths.get(filePath.getParent().toString(), STATUSFOLDER, filePath.getFileName().toString() + CATCHEDEXT)); + removeFile(Paths.get(filePath.getParent().toString(), STATUSFOLDER, filePath.getFileName().toString() + JSONEXT)); + + boolean result = true; + for (Path killDateFile : killDateFiles) { + if (!removeFile(killDateFile)) + result = false; + } + + return result; + } + + @Override + public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) throws IOException { + processPathItem(file); + return FileVisitResult.CONTINUE; + } + + @Override + public FileVisitResult visitFileFailed(Path file, IOException e) throws IOException { + logger.error(marker, "A {} fájl nem érhető el. A rendszer hibaüzenete: {}", file.toString(), e.getMessage()); + return FileVisitResult.CONTINUE; + } +} diff --git a/server/-product/production/OMAR/jobs/steps/CreateArchiveItemStep.java b/server/-product/production/OMAR/jobs/steps/CreateArchiveItemStep.java new file mode 100644 index 00000000..ec165e87 --- /dev/null +++ b/server/-product/production/OMAR/jobs/steps/CreateArchiveItemStep.java @@ -0,0 +1,47 @@ +package user.jobengine.server.steps; + +import java.nio.file.Paths; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + +import com.ibm.nosql.json.api.BasicDBObject; +import com.ibm.nosql.json.api.DB; +import com.ibm.nosql.json.api.DBCollection; + +import user.commons.nosql.NoSQLUtils; +import user.jobengine.db.Media; +import user.jobengine.db.Store; + +public class CreateArchiveItemStep extends JobStep { + private static final Logger logger = LogManager.getLogger(); + + @StepEntry + public Object[] execute(Media mediaCubeMedia, String localHiresPath) throws Exception { + DB db = NoSQLUtils.getNoSQLDB(); + DBCollection collection = db.getCollection("missing_lowres"); + Store highResStore = getManager().getSystemStore(false); + + ArchiveItem archiveItem = null; + try { + if (mediaCubeMedia.getMediaFilesCount() != 1) { + throw new Exception("Expected media count is 1, found " + mediaCubeMedia.getMediaFilesCount()); + } + if (mediaCubeMedia.getMediaFiles().get(0).getStoreId() != highResStore.getId()) { + throw new Exception("Expected media store is a high-res store"); + } + + String name = mediaCubeMedia.getMediaFileRealName(); + archiveItem = new ArchiveItem(); + archiveItem.setMediaFile(Paths.get(localHiresPath, name).toString()); + collection.save(new BasicDBObject("name", name)); + } catch (Exception e) { + logger.catching(e); + logger.info(getMarker(), e.getMessage()); + throw e; + } finally { + setProgress(100); + } + return new Object[] { archiveItem }; + } +} diff --git a/server/-product/production/OMAR/jobs/steps/CreateMissingLowresStep.java b/server/-product/production/OMAR/jobs/steps/CreateMissingLowresStep.java new file mode 100644 index 00000000..72b4cfb1 --- /dev/null +++ b/server/-product/production/OMAR/jobs/steps/CreateMissingLowresStep.java @@ -0,0 +1,95 @@ +package user.jobengine.server.steps; + +import java.nio.file.Paths; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + +import com.ibm.nosql.json.api.BasicDBObject; +import com.ibm.nosql.json.api.DB; +import com.ibm.nosql.json.api.DBCollection; + +import user.commons.log4j2.marker.MediaCubeUndoMarker; +import user.commons.nosql.NoSQLUtils; +import user.jobengine.db.IItemManager; +import user.jobengine.db.Media; +import user.jobengine.db.MediaFile; +import user.jobengine.server.IJobEngine; +import user.jobengine.server.IJobRuntime; + +public class CreateMissingLowresStep extends JobStep { + private static final Logger logger = LogManager.getLogger(); + + @StepEntry + public Object[] execute(String localHiresPath, IJobEngine jobEngine, IJobRuntime jobRuntime) throws Exception { + Object[] result = new Object[] { null, null, "%s", null, 0, true }; + + DB db = NoSQLUtils.getNoSQLDB(); + DBCollection collection = db.getCollection("missing_lowres"); + IItemManager manager = jobEngine.getItemManager(); + Media media = getFirstUntranscodedMedia(manager, collection); + + try { + if (media == null) { + logger.info(new MediaCubeUndoMarker(getSessionMarker().getSessionID()), "Nincs feldolgozandó hiány."); + // throw new Exception("Nincs feldolgozandó hiány."); + cancel(); + return null; + } + + String name = media.getMediaFileRealName(); + result[0] = media; + ArchiveItem archiveItem = new ArchiveItem(); + archiveItem.setMediaFile(Paths.get(localHiresPath, name).toString()); + result[1] = archiveItem; + collection.save(new BasicDBObject("name", name)); + logger.info(getSessionMarker(), "Processing mediaID: {}", media.getId()); + + } catch (Exception e) { + logger.catching(e); + logger.error(getSessionMarker(), e.getMessage()); + throw e; + } finally { + setProgress(100); + } + return result; + } + + private Media getFirstUntranscodedMedia(IItemManager manager, DBCollection collection) { + Media[] result = new Media[] { null }; + // MV + String query = "SELECT mediaid FROM VW_MISSING_PROXY_IDS WHERE HOUSEID like 'M%' or HOUSEID like 'P%' or HOUSEID like 'R%' ORDER BY modified DESC"; + + // HTV + // String query = "SELECT mediaid FROM VW_MISSING_PROXY_IDS"; + manager.executeQuery(query, rs -> { + try { + long mediaId = rs.getLong(1); + Media media = manager.getMedia(mediaId); + // a nevgeneralas miatt az eredeti MediaFilesName nem jo, a pontos nev kell + // nekunk + String name = media.getMediaFileRealName(); + //logger.info(getSessionMarker(), "Checking {}", name); + long existing = collection.find(new BasicDBObject("name", name)).count(); + if (existing > 0) { + // logger.info(getSessionMarker(), "{} is on missing_lowres list", name); + return true; + } + + // 210617 proxy keszites tiltasa + MediaFile mf = manager.getSystemMediaFile(media); + if (mf.isDisableProxy()) { + logger.info(getSessionMarker(), "Proxy disabled {}", name); + + return true; + } + + result[0] = media; + } catch (Exception e) { + logger.error(e); + } + return false; + }, null); + return result[0]; + } +} diff --git a/server/-product/production/OMAR/jobs/steps/DeleteFile.java b/server/-product/production/OMAR/jobs/steps/DeleteFile.java new file mode 100644 index 00000000..539faf74 --- /dev/null +++ b/server/-product/production/OMAR/jobs/steps/DeleteFile.java @@ -0,0 +1,22 @@ +package user.jobengine.server.steps; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + +import user.commons.RemoteFile; +import user.commons.StoreUri; + +public class DeleteFile extends JobStep { + private static final Logger logger = LogManager.getLogger(); + + @StepEntry + public Object[] execute(String fileName, StoreUri tempStoreUri) throws Exception { + try { + RemoteFile remoteFile = tempStoreUri.getRemoteFile(fileName); + tempStoreUri.delete(remoteFile); + } catch (Exception e) { + logger.warn(getJobRuntime().getSessionMarker(), e.getMessage()); + } + return null; + } +} diff --git a/server/-product/production/OMAR/jobs/steps/DummyTestStep1.java b/server/-product/production/OMAR/jobs/steps/DummyTestStep1.java new file mode 100644 index 00000000..22b3ab97 --- /dev/null +++ b/server/-product/production/OMAR/jobs/steps/DummyTestStep1.java @@ -0,0 +1,23 @@ +package user.jobengine.server.steps; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + +public class DummyTestStep1 extends JobStep { + private static final Logger logger = LogManager.getLogger(DummyTestStep1.class); + + @StepEntry + public Object[] execute(String param1) { + logger.info("Executing DummyTestStep1"); + int var1 = 0; + if (param1.equals("Jozsi")) { + var1 = 1; + Thread.sleep(1000); + setProgress(50); + Thread.sleep(1000); + setProgress(50); + } + + return new Object[] {var1}; + } +} diff --git a/server/-product/production/OMAR/jobs/steps/DummyTestStep2.java b/server/-product/production/OMAR/jobs/steps/DummyTestStep2.java new file mode 100644 index 00000000..e7579f12 --- /dev/null +++ b/server/-product/production/OMAR/jobs/steps/DummyTestStep2.java @@ -0,0 +1,20 @@ +package user.jobengine.server.steps; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + +public class DummyTestStep2 extends JobStep { + private static final Logger logger = LogManager.getLogger(DummyTestStep2.class); + + @StepEntry + public Object[] execute(int var2) { + logger.info("Executing DummyTestStep2"); + long var3 = 0L; + Thread.sleep(1000); + setProgress(50); + Thread.sleep(1000); + setProgress(50); + + return new Object[] { var3 }; + } +} diff --git a/server/-product/production/OMAR/jobs/steps/DummyTestStep3.java b/server/-product/production/OMAR/jobs/steps/DummyTestStep3.java new file mode 100644 index 00000000..93d76a75 --- /dev/null +++ b/server/-product/production/OMAR/jobs/steps/DummyTestStep3.java @@ -0,0 +1,17 @@ +package user.jobengine.server.steps; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + +public class DummyTestStep3 extends JobStep { + private static final Logger logger = LogManager.getLogger(DummyTestStep3.class); + + @StepEntry + public void execute(long var3) { + Thread.sleep(1000); + setProgress(50); + Thread.sleep(1000); + setProgress(50); + logger.info("Executing DummyTestStep3"); + } +} diff --git a/server/-product/production/OMAR/jobs/steps/FileCopyStep.java b/server/-product/production/OMAR/jobs/steps/FileCopyStep.java new file mode 100644 index 00000000..c4172550 --- /dev/null +++ b/server/-product/production/OMAR/jobs/steps/FileCopyStep.java @@ -0,0 +1,95 @@ +package user.jobengine.server.steps; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.apache.logging.log4j.Marker; + +import user.commons.StoreUri; +import user.commons.remotestore.IProgressEventListener; +import user.commons.remotestore.ProgressEvent; +import user.commons.remotestore.RemoteStoreProtocol; +import user.jobengine.db.IItemManager; +import user.jobengine.server.IJobEngine; +import user.jobengine.server.IJobRuntime; + +public class FileCopyStep extends JobStep { + private static final Logger logger = LogManager.getLogger(); + private IItemManager manager; + private Marker marker; + + private void check(String sourceProtocol, String sourcePath, String sourceFileName, String targetProtocol, String targetPath, String targetFileName, + IJobEngine jobEngine, IJobRuntime jobRuntime) { + if (jobEngine == null) { + logger.error(marker, "A folyamatkezelő réteg nem elérhető."); + throw new NullPointerException("Internal error, missing JobEngine reference."); + } + manager = jobEngine.getItemManager(); + if (manager == null) { + logger.error(marker, "Az adatbáziskezelő réteg nem elérhető."); + throw new NullPointerException("Internal error, missing ItemManager reference."); + } + if (sourceProtocol == null) { + logger.error(marker, "A forrás protokol bemeneti paramétere üres."); + throw new NullPointerException("System is not configured properly, 'sourceProtocol' input parameter missing."); + } + if (sourcePath == null) { + logger.error(marker, "A forrás fájl elérés bemeneti paramétere üres."); + throw new NullPointerException("System is not configured properly, 'sourcePath' input parameter missing."); + } + if (sourceFileName == null) { + logger.error(marker, "A forrás fájlnév bemeneti paramétere üres."); + throw new NullPointerException("System is not configured properly, 'sourceFileName' input parameter missing."); + } + if (targetProtocol == null) { + logger.error(marker, "A cél protokol bemeneti paramétere üres."); + throw new NullPointerException("System is not configured properly, 'targetProtocol' input parameter missing."); + } + if (targetPath == null) { + logger.error(marker, "A cél fájl elérés bemeneti paramétere üres."); + throw new NullPointerException("System is not configured properly, 'targetPath' input parameter missing."); + } + if (targetFileName == null) { + logger.error(marker, "A cél fájlnév bemeneti paramétere üres."); + throw new NullPointerException("System is not configured properly, 'targetFileName' input parameter missing."); + } + } + + @StepEntry + public Object[] execute(String sourceProtocol, String sourcePath, String sourceFileName, String targetProtocol, String targetPath, String targetFileName, + int killDateDays, IJobEngine jobEngine, IJobRuntime jobRuntime) throws Exception { + marker = jobRuntime.getSessionMarker(); + + check(sourceProtocol, sourcePath, sourceFileName, targetProtocol, targetPath, targetFileName, jobEngine, jobRuntime); + + StoreUri source = null; + StoreUri target = null; + + try { + source = manager.createStoreUri(Enum.valueOf(RemoteStoreProtocol.class, sourceProtocol), sourcePath); + target = manager.createStoreUri(Enum.valueOf(RemoteStoreProtocol.class, targetProtocol), targetPath); + source.addProgressListener(new IProgressEventListener() { + @Override + public void progressChanged(ProgressEvent evt) { + jobRuntime.incrementProgress(evt.getProgress()); + } + }); + + source.transferFrom(target, sourceFileName, targetFileName); + if (killDateDays > -1) + EscortFiles.createUNCKillDate(targetPath, targetFileName, killDateDays, marker); + + } catch (Exception e) { + logger.catching(e); + throw e; + } finally { + if (source != null) { + source.cleanUp(); + } + if (target != null) { + target.cleanUp(); + } + } + + return null; + } +} diff --git a/server/-product/production/OMAR/jobs/steps/GenerateJSONMetadataStep.java b/server/-product/production/OMAR/jobs/steps/GenerateJSONMetadataStep.java new file mode 100644 index 00000000..210d7354 --- /dev/null +++ b/server/-product/production/OMAR/jobs/steps/GenerateJSONMetadataStep.java @@ -0,0 +1,65 @@ +package user.jobengine.server.steps; + +import java.io.IOException; +import java.nio.file.Paths; +import java.sql.Timestamp; +import java.util.Map; + +import org.apache.commons.io.FilenameUtils; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + +import user.commons.DownloadableMedia; +import user.commons.StoreUri; +import user.commons.mediaarea.MediaArea; +import user.commons.remotestore.RemoteStoreProtocol; +import user.jobengine.server.steps.shared.EscortFiles; + +public class GenerateJSONMetadataStep extends JobStep { + private static final Logger logger = LogManager.getLogger(); + + @StepEntry + public Object[] execute(Map files, String sourceStoreName, String targetStoreName, + String escortStoreFolder) { + if (!files.isEmpty()) { + for (String fullPath : files.keySet()) { + MediaArea mediaArea = files.get(fullPath); + String title = FilenameUtils.getBaseName(fullPath).replace(FilenameUtils.getExtension(fullPath), ""); + String pathOnly = fullPath.substring(0, fullPath.lastIndexOf("\\") + 1); + String filenameOnly = FilenameUtils.getBaseName(fullPath); + Timestamp created = null; + Timestamp modified = null; + long frameCount = mediaArea.getFrameCount(); + long mediaId = 0; + StoreUri sourceStoreUri = getManager().getStoreUri(sourceStoreName, RemoteStoreProtocol.LOCAL); + StoreUri targetStoreUri = getManager().getStoreUri(targetStoreName, RemoteStoreProtocol.LOCAL); + StoreUri escortStoreUri = getManager().getStoreUri(escortStoreFolder, RemoteStoreProtocol.LOCAL); + + try { + String outputPath = Paths.get(escortStoreUri.toString(true)).toString(); + } catch (Exception e1) { + e1.printStackTrace(); + } + + DownloadableMedia downloadable = DownloadableMedia.create(title, filenameOnly, modified, created, + frameCount, 0, sourceStoreUri.getId(), targetStoreUri.getId(), mediaId); + String escortFileName = targetStoreName + "." + downloadable.getString("fileName"); // needed without + // extension + try { + if (EscortFiles.createMetadataIfNotExists(pathOnly, escortFileName, + downloadable.toPrettyString(""))) { + logger.info(getMarker(), "Archive status file created for {}", fullPath); + } else { + logger.info(getMarker(), "Archive status file already exists for {}", fullPath); + } + } catch (IOException e) { + logger.error("{}", e.getCause()); + e.printStackTrace(); + } + } + } else { + logger.info("files is empty!"); + } + return null; + } +} diff --git a/server/-product/production/OMAR/jobs/steps/HSMMigrateStep.java b/server/-product/production/OMAR/jobs/steps/HSMMigrateStep.java new file mode 100644 index 00000000..895b1411 --- /dev/null +++ b/server/-product/production/OMAR/jobs/steps/HSMMigrateStep.java @@ -0,0 +1,401 @@ +package user.jobengine.server.steps; + +import java.io.BufferedInputStream; +import java.io.BufferedOutputStream; +import java.io.File; +import java.io.FileInputStream; +import java.io.FileOutputStream; +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.nio.file.attribute.BasicFileAttributes; +import java.text.SimpleDateFormat; +import java.util.Arrays; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; + +import org.apache.commons.io.FilenameUtils; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.apache.logging.log4j.Marker; + +import com.ibm.nosql.json.api.BasicDBObject; +import com.ibm.nosql.json.api.DB; +import com.ibm.nosql.json.api.DBCollection; +import com.ibm.nosql.json.api.QueryBuilder; + +import user.commons.IEntityBase; +import user.commons.log4j2.marker.MediaCubeMarker; +import user.commons.nosql.NoSQLUtils; +import user.jobengine.db.IItemManager; +import user.jobengine.db.MediaFile; +import user.jobengine.db.MediaFileDAO; +import user.mediacube.metadata.interfaces.IMetadata; +import user.mediacube.metadata.interfaces.IMetadataProvider; +import user.mediacube.metadata.interfaces.IMetadataProviderFactory; +import user.mediacube.metadata.interfaces.MetadataProviderType; +import user.mediacube.metadata.interfaces.MetadataType; +import user.mediacube.metadata.interfaces.PlanAirMetadataListOptions; + +public class HSMMigrateStep extends JobStep { + private static final Logger logger = LogManager.getLogger(); + private static final String MXFEXT = ".mxf"; + private static final String MOVEXT = ".mov"; + private Marker marker = null; + private IMetadataProvider hsmProvider; + private Map tapeContents = new LinkedHashMap<>(); + private DBCollection excludes; + private DBCollection fileHistory; + private DBCollection volumeHistory; + private DB db; + private IMetadataProvider planairProvider; + private SimpleDateFormat df = new SimpleDateFormat("yyyy.MM.dd"); + + private void cleanupHistory() { + if (db == null) + db = NoSQLUtils.getNoSQLDB(); + if (fileHistory == null) + fileHistory = db.getCollection("hsm_migrate_file_history"); + if (volumeHistory == null) + volumeHistory = db.getCollection("hsm_migrate_volume_history"); + fileHistory.drop(); + volumeHistory.drop(); + } + + private BasicDBObject createMetadata(String volumeName, String fileName) throws Exception { + + Path filePath = Paths.get(fileName); + String mediaHouseId = FilenameUtils.removeExtension(filePath.getFileName().toString()); + BasicDBObject result = null; + try { + result = getPlanAirMetadata(mediaHouseId); + } catch (Exception e) { + + logger.error("PlanAir metadata error", e); + //nem latja a drivert pl. + //throw e; + } + + if (result == null) { + result = new BasicDBObject(); + BasicFileAttributes attr = Files.readAttributes(filePath, BasicFileAttributes.class); + result.put("itemHouseId", df.format(attr.lastModifiedTime().toMillis())); + result.put("itemTitle", filePath.getParent().toString()); + result.put("mediaHouseId", mediaHouseId); + result.put("mediaTitle", fileName); + result.put("mediaDescription", volumeName); + result.put("mediaType", "Generic"); + } + result.put("userName", "mediacube"); + return result; + } + + @SuppressWarnings("serial") + @StepEntry + public Object[] execute(String sourceLocation, String targetLocation) throws Exception { + marker = getJobRuntime().getSessionMarker(); + //remove from prod + //cleanupHistory(); + hsmProvider = getMetadataProvider(MetadataProviderType.HSM); + if (hsmProvider == null) + throw new NullPointerException("No HSM metadata provider available"); + planairProvider = getMetadataProvider(MetadataProviderType.PLANAIR); + if (planairProvider == null) + throw new NullPointerException("No PLANAIR metadata provider available"); + Path targetPath = Paths.get(targetLocation); + try { + List volumes = hsmProvider.list(new BasicDBObject()); + for (IMetadata volume : volumes) { + String volumeName = volume.getTitle(); + + BasicDBObject historyResult = queryVolumeHistory(volumeName); + if (historyResult != null) { + logger.info(marker, "A kazetta már feldolgozásra került: {}", volumeName); + continue; + } + logger.info(marker, "A kazetta feldolgozása elindul: {}", volumeName); + List contents = getContents(volumeName); + int p = 0; + boolean oneSuccess = false; + boolean hasError = false; + for (IMetadata content : contents) { + BasicDBObject c = content.asJSON(); + String hsmFileName = c.getString("fileName"); + long contentFileSize = NoSQLUtils.asLong(c, "fileSize"); + if (!tapeContents.containsKey(hsmFileName)) { + tapeContents.put(hsmFileName, c); + Path sourceFilePath = Paths.get(sourceLocation, hsmFileName); + if (contentFileSize < Files.getFileStore(targetPath).getUsableSpace()) { + Path targetFilePath = Paths.get(targetLocation, sourceFilePath.getFileName().toString()); + + try { + if (processHSMFile(volumeName, hsmFileName, sourceFilePath, targetFilePath)) { + oneSuccess = true; + } + } catch (Exception e) { + hasError = true; + } + } + } + setProgress(p++ * 100 / contents.size()); + + if (getJobRuntime().isWaitingCancel()) { + logger.info("Job canceled by user"); + //ne mentsuk a szalagot meg + return null; + } + + } + + if (oneSuccess && !hasError) { + saveVolumeHistory(volumeName); + String subject = "A kazetta eltávolítható a HSM rendszerből: " + volumeName; + logger.info(new MediaCubeMarker() { + { + setSubject(subject); + } + }, subject); + } + + //ha mar sikerult valamit archivalni kilepunk + if (oneSuccess) + break; + } + } catch (Exception e) { + logger.error(marker, "Hiba a migráció során. A rendszer hibaüzenete: {}", e.getMessage()); + throw e; + } + + return null; + } + + private List getContents(String volumeName) throws Exception { + List contents = null; + contents = hsmProvider.list(new BasicDBObject("volumeName", volumeName)); + return contents; + } + + protected IMetadataProvider getMetadataProvider(MetadataProviderType type) { + IMetadataProviderFactory factory = getService(IMetadataProviderFactory.class); + if (factory == null) + return null; + return factory.getProvider(type); + } + + private BasicDBObject getPlanAirMetadata(String mediaHouseId) throws Exception { + PlanAirMetadataListOptions opt = new PlanAirMetadataListOptions(); + opt.setSearch(mediaHouseId); + + List result = null; + opt.setType(MetadataType.Material); + result = planairProvider.list(opt); + if (result.size() != 0) + return result.get(0).asJSON(); + + opt.setType(MetadataType.Promo); + result = planairProvider.list(opt); + if (result.size() != 0) + return result.get(0).asJSON(); + + opt.setType(MetadataType.AD); + result = planairProvider.list(opt); + if (result.size() != 0) + return result.get(0).asJSON(); + + return null; + } + + //true if need copy + public boolean prepareCopy(String hsmFileName, Path source, Path target) throws IOException { + boolean result = true; + BasicDBObject excludeResult = queryExclude(hsmFileName); + if (excludeResult != null) { + logger.warn(marker, "Kivételként megjelölt: {}", hsmFileName); + return false; + } + + File sourceFile = source.toFile(); + File targetFile = target.toFile(); + + if (!sourceFile.exists()) { + logger.warn(marker, "A forrás nem elérhető: {}", source); + return false; + } + + // BasicDBObject historyResult = queryFileHistory(contentFileName); + // if (historyResult != null) + // return false; + + // if (!mediaFile.getName().toLowerCase().endsWith(MOVEXT.toLowerCase()) && !mediaFile.getName().toLowerCase().endsWith(MXFEXT.toLowerCase())) + // return; + // logger.info("Start copy from {} to {}", sourceFilePath, targetFilePath); + + boolean targetExists = targetFile.exists(); + + long targetLength = targetFile.length(); + long sourceLength = sourceFile.length(); + + if (targetLength == sourceLength) { + logger.warn(marker, "A fájl már fel van dolgozva: {}, {} -> {}", source, sourceLength, targetLength); + return false; + } + + if (targetLength > sourceLength) { + logger.warn(marker, "A célfájl nagyobb, törlöm: {}", target); + Files.delete(target); + targetLength = 0; + targetExists = false; + } + + if (targetExists) { + logger.warn(marker, "A fájl már létezik, a másolás folytatódik: {}, {} -> {}", target, sourceLength, targetLength); + } else + logger.warn(marker, "Migrálás: {}, {} -> {}", source, sourceLength, targetLength); + + return result; + } + + private boolean processHSMFile(String volumeName, String hsmFileName, Path sourceFilePath, Path targetFilePath) throws Exception { + int repeat = 4; + boolean successCopy = false; + + IItemManager manager = getManager(); + MediaFileDAO mfDAO = (MediaFileDAO) manager.getBaseDAO(MediaFile.class); + List mediaFiles = mfDAO.getByHouseId(sourceFilePath.getFileName().toString()); + if (mediaFiles != null && mediaFiles.size() > 0) { + logger.warn(marker, "Már archivált: {}", hsmFileName); + return false; + } + + if (prepareCopy(hsmFileName, sourceFilePath, targetFilePath)) { + while (repeat > 0) { + try { + resumeableCopy(sourceFilePath, targetFilePath); + repeat = 0; + successCopy = true; + } catch (Exception e) { + if (Files.exists(targetFilePath) && targetFilePath.toFile().length() == 0) + Files.delete(targetFilePath); + //logger.warn(marker, "Hiba a másolás során: {} ({})", sourceFilePath, e.getMessage()); + repeat--; + } + } + } + String metadataFileName = sourceFilePath.getFileName() + EscortFiles.DOT_JSON; + Path metadataPath = Paths.get(targetFilePath.getParent().toString(), EscortFiles.STATUSFOLDER, metadataFileName); + boolean createMetadata = Files.exists(targetFilePath) && !Files.exists(metadataPath); + + if (successCopy || createMetadata) { + String metadata = null; + try { + metadata = createMetadata(volumeName, hsmFileName).toPrettyString(""); + EscortFiles.createMetadata(targetFilePath.getParent().toString(), targetFilePath.getFileName().toString(), metadata); + //saveFileHistory(contentFileName); + + } catch (Exception e) { + logger.error(marker, "Metadata error", e); + return false; + } + return true; + } + + logger.error(marker, "A fájl másolása nem lehetséges: {}", sourceFilePath); + return false; + } + + public BasicDBObject queryExclude(String fileName) { + if (db == null) + db = NoSQLUtils.getNoSQLDB(); + if (excludes == null) + excludes = db.getCollection("hsm_migrate_exclude"); + Path filePath = Paths.get(fileName); + String pureFileName = FilenameUtils.removeExtension(filePath.getFileName().toString()); + QueryBuilder qb = QueryBuilder.start("name").in(Arrays.asList(fileName, pureFileName)); + BasicDBObject exceptionResult = NoSQLUtils.asSingle(excludes.find(qb.get())); + return exceptionResult; + } + + public BasicDBObject queryFileHistory(String fileName) { + if (db == null) + db = NoSQLUtils.getNoSQLDB(); + if (fileHistory == null) + fileHistory = db.getCollection("hsm_migrate_file_history"); + QueryBuilder qb = QueryBuilder.start("name").in(Arrays.asList(fileName)); + BasicDBObject historyResult = NoSQLUtils.asSingle(fileHistory.find(qb.get())); + return historyResult; + } + + public BasicDBObject queryVolumeHistory(String volumeName) { + if (db == null) + db = NoSQLUtils.getNoSQLDB(); + if (volumeHistory == null) + volumeHistory = db.getCollection("hsm_migrate_volume_history"); + QueryBuilder qb = QueryBuilder.start("name").in(Arrays.asList(volumeName)); + BasicDBObject historyResult = NoSQLUtils.asSingle(volumeHistory.find(qb.get())); + return historyResult; + } + + public void resumeableCopy(Path source, Path target) throws Exception { + File sourceFile = source.toFile(); + File targetFile = target.toFile(); + boolean targetExists = targetFile.exists(); + + long targetLength = targetFile.length(); + long sourceLength = sourceFile.length(); + + try (InputStream in = new BufferedInputStream(new FileInputStream(sourceFile)); + OutputStream out = new BufferedOutputStream(new FileOutputStream(targetFile, targetExists))) { + + byte[] buffer = new byte[128 * 1024]; + int lengthRead; + + if (targetExists) + in.skip(targetLength); + + while ((lengthRead = in.read(buffer)) > 0) { + out.write(buffer, 0, lengthRead); + out.flush(); + targetLength = targetFile.length(); + if (targetLength > sourceLength) { + throw new Exception("Hiba! A fájl túl nagy lett."); + } + + if (getJobRuntime().isWaitingCancel()) { + break; + } + } + + targetLength = targetFile.length(); + sourceLength = sourceFile.length(); + if (targetLength != sourceLength) { + throw new Exception("Hiba! A fájl mérete nem egyezik."); + } + } + } + + private void saveFileHistory(String fileName) { + if (db == null) + db = NoSQLUtils.getNoSQLDB(); + if (fileHistory == null) + fileHistory = db.getCollection("hsm_migrate_file_history"); + BasicDBObject item = new BasicDBObject(); + item.put("name", fileName); + fileHistory.save(item); + + } + + private void saveVolumeHistory(String volumeName) { + if (db == null) + db = NoSQLUtils.getNoSQLDB(); + if (volumeHistory == null) + volumeHistory = db.getCollection("hsm_migrate_volume_history"); + BasicDBObject item = new BasicDBObject(); + item.put("name", volumeName); + volumeHistory.save(item); + + } +} diff --git a/server/-product/production/OMAR/jobs/steps/IntegrationTestStep.java b/server/-product/production/OMAR/jobs/steps/IntegrationTestStep.java new file mode 100644 index 00000000..081d02d8 --- /dev/null +++ b/server/-product/production/OMAR/jobs/steps/IntegrationTestStep.java @@ -0,0 +1,128 @@ +package user.jobengine.server.steps; + +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.concurrent.CountDownLatch; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + +import user.commons.ListUtils; +import user.jobengine.db.ArchivedMedia; +import user.jobengine.db.Item; +import user.jobengine.db.Media; +import user.jobengine.server.steps.shared.ItemManagerExtensions; + +public class IntegrationTestStep extends JobStep { + private static final Logger logger = LogManager.getLogger(); + private ArchivedMedia archivedMedia; + + private void _00_test_cancelable() throws Exception { + CountDownLatch finishLatch = new CountDownLatch(1); + getEngine().submit(null, e -> { + if (e.isRuntimeTerminated()) + finishLatch.countDown(); + }, "cancelable.xml", "Test cancelable", ListUtils.asMap("param", 1)); + finishLatch.await(); + logger.info("_00_test_cancelable SUCCESS"); + } + + private void _01_test_retrieve_ondemand() throws Exception { + archivedMedia = new ArchivedMedia(); + Media media = getManager().getMedia(15285); + Item item = getManager().getItem(media.getItemId()); + archivedMedia.setItem(item); + archivedMedia.setMedia(media); + + CountDownLatch finishLatch = new CountDownLatch(1); + getEngine().submit(null, e -> { + if (e.isRuntimeTerminated()) + finishLatch.countDown(); + + }, "retrieve-ondemand.xml", "Test retrieve-ondemand", + ListUtils.asMap("globalRetrievePath", "file://10.11.1.100", "localRetrievePath", "/mediacube/data", "materialOutputFolder", "/", + "promoOutputFolder", "/", "advertisementOutputFolder", "/", "octopusOutputFolder", "/", "genericOutputFolder", "/", + "onlineOutputFolder", "/", "killDateDays", -1, "nexioAgency", "ARCHIVE_RESTORE", "nexioPort", 2098, "nexioUserName", "administrator", + "nexioPassword", "system", "archivedMedia", archivedMedia, "successRecipient", "vasary@elgekko.net", "houseId", + archivedMedia.getMedia().getHouseId(), "targetPathType", "0")); + + finishLatch.await(); + Path output = Paths.get("/mediacube/data", archivedMedia.getMedia().getHouseId(), + archivedMedia.getMedia().getHouseId() + "-ARCH-" + archivedMedia.getMedia().getMediaFileRealName()); + if (!Files.exists(output)) + throw new Exception("File does not exist: " + output); + + logger.info("_01_test_retrieve_ondemand SUCCESS"); + } + + private void _02_test_archive_material() throws Exception { + Path input = Paths.get("/mediacube/data", archivedMedia.getMedia().getHouseId(), + archivedMedia.getMedia().getHouseId() + "-ARCH-" + archivedMedia.getMedia().getMediaFileRealName()); + + String outputName = "IntegrationTest_" + System.currentTimeMillis() + ".mxf"; + Path output = Paths.get(input.getParent().toString(), outputName); + Files.copy(input, output); + if (!Files.exists(output)) + throw new Exception("File does not exist: " + output); + + ArchiveItem archiveItem = new ArchiveItem(); + archiveItem.setMediaFile(output.toString()); + archiveItem.setItemHouseId(outputName); + archiveItem.setItemTitle(outputName); + archiveItem.setMediaHouseId(outputName); + archiveItem.setMediaTitle(outputName); + archiveItem.setMediaType("Generic"); + + CountDownLatch finishLatch = new CountDownLatch(1); + getEngine().submit(null, e -> { + if (e.isRuntimeTerminated()) + finishLatch.countDown(); + }, "archive-material.xml", "Test archive-material", ListUtils.asMap("archiveItem", archiveItem, "killDateDays", 0)); + finishLatch.await(); + + if (!ItemManagerExtensions.isArchived(getManager(), output)) + throw new Exception("File not archived: " + output); + + Files.delete(output); + + logger.info("_02_test_archive_material SUCCESS"); + } + + private void _03_test_delete_materials() throws Exception { + Path output = Paths.get("/mediacube/data", archivedMedia.getMedia().getHouseId(), + archivedMedia.getMedia().getHouseId() + "-ARCH-" + archivedMedia.getMedia().getMediaFileRealName()); + String source = output.getParent().toString(); + + CountDownLatch finishLatch = new CountDownLatch(1); + getEngine().submit(null, e -> { + if (e.isRuntimeTerminated()) + finishLatch.countDown(); + }, "delete-materials.xml", "Test delete-materials", ListUtils.asMap("sourcePath", source, "skipArchiveCheck", true)); + finishLatch.await(); + + if (Files.exists(output)) + throw new Exception("File exists: " + output); + } + + @StepEntry + public Object[] execute() throws Exception { + getJobRuntime().setDescription("_00_test_cancelable"); + _00_test_cancelable(); + setProgress(25); + + getJobRuntime().setDescription("_01_test_retrieve_ondemand"); + _01_test_retrieve_ondemand(); + setProgress(50); + + getJobRuntime().setDescription("_02_test_archive_material"); + _02_test_archive_material(); + setProgress(75); + + getJobRuntime().setDescription("_03_test_delete_materials"); + _03_test_delete_materials(); + setProgress(100); + return null; + } + +} diff --git a/server/-product/production/OMAR/jobs/steps/MXFCutterStep.java b/server/-product/production/OMAR/jobs/steps/MXFCutterStep.java new file mode 100644 index 00000000..58a5944d --- /dev/null +++ b/server/-product/production/OMAR/jobs/steps/MXFCutterStep.java @@ -0,0 +1,129 @@ +package user.jobengine.server.steps; + +import org.apache.commons.lang.StringUtils; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.apache.logging.log4j.Marker; + +import user.commons.RemoteFile; +import user.commons.StoreUri; +import user.commons.remotestore.IProgressEventListener; +import user.commons.remotestore.ProgressEvent; +import user.commons.remotestore.RemoteStoreProtocol; +//import user.jobengine.db.Media; +import user.jobengine.db.ArchivedMedia; +import user.jobengine.db.IItemManager; +import user.jobengine.db.Store; +import user.jobengine.server.IJobEngine; +import user.jobengine.server.IJobRuntime; + +public class MXFCutterStep extends JobStep { + private static final String TARGETNAMEPATTERN = "-ARCH-%s"; + private static final Logger logger = LogManager.getLogger(); + private IItemManager manager; + private StoreUri tempTargetUri; + private StoreUri tempSourceUri; + private String sourceFileName; + private Marker marker; + private int nexioPort; + private String nexioUserName, nexioPassword; + private String nexioHost; + + protected void checkTargetPath(String targetPath) { + if (StringUtils.isBlank(targetPath)) { + logger.error(marker, "A folyamat 'targetPath' bemeneti paramétere üres."); + throw new NullPointerException("System is not configured properly, 'targetPath' input parameter missing."); + } + } + + protected StoreUri createTargetUri(IItemManager manager, String targetPath) throws NullPointerException { + StoreUri result = null; + result = manager.createStoreUri(RemoteStoreProtocol.FTP, nexioHost); + result.setPortNumber(nexioPort); + result.setUserName(nexioUserName); + result.setPassword(nexioPassword); + + return result; + } + + @StepEntry + public Object[] execute(ArchivedMedia archivedMedia, String targetPath, String houseId, String successRecipient, int killDateDays, boolean useNexioTarget, + String nexioAgency, int nexioPort, String nexioUserName, String nexioPassword, IJobEngine jobEngine, IJobRuntime jobRuntime) throws Exception { + this.nexioPort = nexioPort; + this.nexioUserName = nexioUserName; + this.nexioPassword = nexioPassword; + nexioHost = System.getProperty("nexio.host"); + marker = jobRuntime.getSessionMarker(); + + if (useNexioTarget && archivedMedia.getTcIn() != null && archivedMedia.getTcOut() != null) { + setAndCheck(archivedMedia, houseId, targetPath, useNexioTarget, jobEngine); + + final IJobRuntime runtime = jobRuntime; + sourceFileName = houseId + TARGETNAMEPATTERN; + tempSourceUri.addProgressListener(new IProgressEventListener() { + @Override + public void progressChanged(ProgressEvent evt) { + runtime.incrementProgress(evt.getProgress()); + } + }); + + RemoteFile result = tempSourceUri.transferFrom(tempTargetUri, sourceFileName, sourceFileName); + + EscortFiles.setNEXIOKillDate(killDateDays, houseId, nexioAgency, tempTargetUri); + + logger.info("A {} videó kivágva {}s - {}s", sourceFileName, archivedMedia.getTcIn(), archivedMedia.getTcOut()); + } + + return null; + } + + // private String getSourceFileName(ArchivedMedia archivedMedia, Store + // store) { + // List mediaFiles = archivedMedia.getMedia().getMediaFiles(); + // if (mediaFiles == null) + // return null; + // for (MediaFile mediaFile : mediaFiles) { + // if (mediaFile.getStore().getId() == store.getId()) + // return mediaFile.getRelativePath(); + // } + // return null; + // } + + private void setAndCheck(ArchivedMedia archivedMedia, String houseId, String targetPath, boolean useNexioTarget, IJobEngine jobEngine) { + if (jobEngine == null) { + logger.error(marker, "A folyamatkezelő réteg nem elérhető."); + throw new NullPointerException("Internal error, missing JobEngine reference."); + } + manager = jobEngine.getItemManager(); + if (manager == null) { + logger.error(marker, "Az adatbáziskezelő réteg nem elérhető."); + throw new NullPointerException("Internal error, missing ItemManager reference."); + } + if (archivedMedia == null) { + logger.error(marker, "A folyamat 'mediaCubeMedia' bemeneti paramétere üres."); + throw new NullPointerException("System is not configured properly, 'mediaCubeMedia' input parameter missing."); + } + checkTargetPath(targetPath); + if (StringUtils.isBlank(houseId)) { + logger.error(marker, "A folyamat 'houseId' bemeneti paramétere üres."); + throw new NullPointerException("System is not configured properly, 'houseId' input parameter missing."); + } + Store tsmStore = manager.getSystemStore(false); + if (tsmStore == null) { + logger.error(marker, "A TSM rendszer beállítás nem elérhető."); + throw new NullPointerException("System is not configured properly, missing TSM Store."); + } + + tempSourceUri = manager.createStoreUri(RemoteStoreProtocol.LOCAL, targetPath); + if (tempSourceUri == null) { + logger.error(marker, "A TSM rendszer beállítás paraméterei nem elérhetőek."); + throw new NullPointerException("System is not configured properly, missing TSM StoreUri."); + } + tempTargetUri = createTargetUri(manager, targetPath); + // sourceFileName = getSourceFileName(archivedMedia, tsmStore); + if (sourceFileName == null) { + logger.error(marker, "Adatbázis bejegyzés hiba, a visszatöltendő fájl neve nem található."); + throw new NullPointerException("Database error, missing MediaFile 'relativePath'."); + } + } +} diff --git a/server/-product/production/OMAR/jobs/steps/MediaToolStep.java b/server/-product/production/OMAR/jobs/steps/MediaToolStep.java new file mode 100644 index 00000000..ecc0ba2d --- /dev/null +++ b/server/-product/production/OMAR/jobs/steps/MediaToolStep.java @@ -0,0 +1,29 @@ +package user.jobengine.server.steps; + +import java.nio.file.Path; +import java.nio.file.Paths; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + +import user.commons.mediaarea.MediaArea; +import user.jobengine.db.Media; + +public class MediaToolStep extends JobStep { + private static final Logger logger = LogManager.getLogger(); + + @StepEntry + public Object[] execute(ArchiveItem archiveItem, Media mediaCubeMedia) throws Exception { + Path filePath = Paths.get(archiveItem.getMediaFile()); + MediaArea ma = new MediaArea(filePath); + ma.process(); + long frames = ma.getFrameCount(); + if (frames > 0) { + logger.info("Media {} length is {}", filePath, frames); + mediaCubeMedia.setLength(frames); + getManager().modify(mediaCubeMedia); + } + return null; + } + +} diff --git a/server/-product/production/OMAR/jobs/steps/MetadataTransformStep.java b/server/-product/production/OMAR/jobs/steps/MetadataTransformStep.java new file mode 100644 index 00000000..c7bfb3ca --- /dev/null +++ b/server/-product/production/OMAR/jobs/steps/MetadataTransformStep.java @@ -0,0 +1,168 @@ +package user.jobengine.server.steps; + +import java.io.File; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.nio.file.attribute.FileAttribute; +import java.nio.file.attribute.PosixFilePermission; +import java.nio.file.attribute.PosixFilePermissions; +import java.util.Date; +import java.util.Set; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.apache.logging.log4j.Marker; + +import com.ibm.nosql.json.api.BasicDBList; + +import user.jobengine.db.Item; +import user.jobengine.db.ItemManager; +import user.jobengine.db.ItemType; +import user.jobengine.db.Media; +import user.jobengine.server.IJobEngine; +import user.jobengine.server.IJobRuntime; + +/** + * Itemek es mediak krealasa az ArchiveItem objektum alapjan. + * + * @author robi + */ +public class MetadataTransformStep extends JobStep { + private static final String CONFLICT = ".CONFLICT"; + private static final Logger logger = LogManager.getLogger(); + private static final String ITEM_MANAGER_IS_NULL = "ItemManager is null"; + public static final String DEFAULT_MEDIATYPE = "Generic"; + private ItemManager itemManager; + + private Marker marker;; + + private void addTags(ArchiveItem archiveItem, Media mediaCubeMedia) { + BasicDBList tags = archiveItem.getTags(); + if (tags != null) { + for (Object tag : tags) { + + try { + String tagText = String.valueOf(tag); + itemManager.addMediaTag(tagText, mediaCubeMedia.getId()); + System.out.println(); + + } catch (Exception e) { + logger.catching(e); + } + } + } + } + + private void checkDuplicates(ArchiveItem archiveItem, String sourceFileName) throws Exception { + if (itemManager.isMediaFileExists(sourceFileName)) { + try { + Path sourcePath = Paths.get(archiveItem.getMediaFile()); + Path parent = sourcePath.getParent(); + Path conflictPath = Paths.get(parent.toString(), CONFLICT); + File folder = conflictPath.toFile(); + if (!folder.exists() || !folder.isDirectory()) { + Set perms = PosixFilePermissions.fromString("rwxrwxrwx"); + FileAttribute> attr = PosixFilePermissions.asFileAttribute(perms); + try { + Files.createDirectories(conflictPath, attr); + } catch (Exception e) { + try { + Files.createDirectory(conflictPath); + } catch (Exception e1) { + logger.catching(e); + throw e; + } + } + } + + Files.move(sourcePath, Paths.get(conflictPath.toString(), sourceFileName + (new Date()).getTime())); + } catch (Exception e1) { + logger.catching(e1); + logger.error(marker, "Hiba az '{}' állomány mappába másolásakor. A rendszer üzenete: {}", CONFLICT, e1.getMessage()); + } + throw new Exception("Az '" + sourceFileName + "' állomány már megtalálható az archívumban, archiválása nem lehetséges."); + } + } + + private Item createItem(ArchiveItem archiveItem) { + Item mediaCubeItem = getExistingItem(archiveItem.getItemHouseId(), archiveItem.getItemTitle()); + if (mediaCubeItem == null) + mediaCubeItem = itemManager.createItem(DEFAULT_MEDIATYPE, archiveItem.getItemTitle(), archiveItem.getItemDescription(), + archiveItem.getItemHouseId()); + return mediaCubeItem; + } + + private Media createMedia(ArchiveItem archiveItem, Item mediaCubeItem, String mediaType) { + Media mediaCubeMedia; + mediaCubeMedia = itemManager.createMedia(mediaType, archiveItem.getMediaTitle(), archiveItem.getMediaDescription(), archiveItem.getMediaHouseId()); + mediaCubeMedia.setLength(archiveItem.getDuration()); + mediaCubeItem.appendMedia(mediaCubeMedia); + + return mediaCubeMedia; + } + + @StepEntry + public Object[] execute(ArchiveItem archiveItem, IJobEngine jobEngine, IJobRuntime jobRuntime) throws Exception { + marker = jobRuntime.getSessionMarker(); + Media mediaCubeMedia = null; + itemManager = (ItemManager) jobEngine.getItemManager(); + if (itemManager == null) + throw new NullPointerException(ITEM_MANAGER_IS_NULL); + try { + File sourceMediaFile = new File(archiveItem.getMediaFile()); + String sourceFileName = sourceMediaFile.getName(); + checkDuplicates(archiveItem, sourceFileName); + Item mediaCubeItem = createItem(archiveItem); + jobRuntime.incrementProgress(50); + String mediaType = getCreateType(archiveItem); + mediaCubeMedia = createMedia(archiveItem, mediaCubeItem, mediaType); + //ha itemid 0 akkor merge, egyebkent media insert + + if (mediaCubeItem.getId() == 0) + itemManager.mergeItemStructure(mediaCubeItem); + else { + mediaCubeMedia.setItemId(mediaCubeItem.getId()); + mediaCubeMedia.add(); + } + + addTags(archiveItem, mediaCubeMedia); + + } catch (Exception e) { + logger.catching(e); + String fileName = new File(archiveItem.getMediaFile()).getName(); + logger.error(marker, "Az '{}' állomány nem archiválható, mert a metaadat transzformáció sikertelen. A rendszer üzenete: {}", fileName, + e.getMessage()); + if (!archiveItem.removeCatchedFile()) + logger.error(marker, "Az '{}' állomány .catched jelző állománya nem törölhető.", fileName); + throw e; + } finally { + jobRuntime.incrementProgress(100); + } + return new Object[] { mediaCubeMedia }; + } + + private String getCreateType(ArchiveItem archiveItem) { + String mediaType = archiveItem.getMediaType(); + if (mediaType == null || mediaType.length() == 0) + mediaType = DEFAULT_MEDIATYPE; + else { + ItemType mediaItemType = itemManager.getItemType(mediaType); + if (mediaItemType == null) + itemManager.createItemType(mediaType, mediaType).add(); + } + return mediaType; + } + + private Item getExistingItem(String itemHouseId, String itemTitle) { + Item[] result = new Item[] { null }; + String sql = String.format("select id from item where houseid='%s' and title='%s'", itemHouseId, itemTitle); + itemManager.executeQuery(sql, rs -> { + long id = rs.getLong("id"); + result[0] = itemManager.getItem(id); + return true; + }, null); + return result[0]; + } + +} diff --git a/server/-product/production/OMAR/jobs/steps/MetadataUpdater.java b/server/-product/production/OMAR/jobs/steps/MetadataUpdater.java new file mode 100644 index 00000000..a28abfa2 --- /dev/null +++ b/server/-product/production/OMAR/jobs/steps/MetadataUpdater.java @@ -0,0 +1,297 @@ +package user.jobengine.server.steps; + +import java.io.IOException; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.sql.ResultSet; +import java.text.SimpleDateFormat; +import java.util.Date; +import java.util.LinkedHashSet; +import java.util.List; +import java.util.Set; + +import org.apache.commons.io.FileUtils; +import org.apache.ibatis.jdbc.SQL; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.apache.logging.log4j.Marker; +import org.apache.logging.log4j.MarkerManager; + +import com.ibm.nosql.json.api.BasicDBObject; + +import user.jobengine.db.IItemManager; +import user.jobengine.server.steps.shared.MetadataTypeDetector; +import user.mediacube.metadata.interfaces.IMetadata; +import user.mediacube.metadata.interfaces.IMetadataProvider; +import user.mediacube.metadata.interfaces.IMetadataProviderFactory; +import user.mediacube.metadata.interfaces.MetadataProviderType; +import user.mediacube.metadata.interfaces.MetadataType; +import user.mediacube.metadata.interfaces.PlanAirMetadataListOptions; + +public class MetadataUpdater extends JobStep { + private static final String EMPTY = ""; + private static final String DOT = "."; + private static final Logger logger = LogManager.getLogger(MetadataUpdater.class); + private Marker csvMarker = MarkerManager.getMarker("METADATA-UPDATER-CSV"); + + private static final String ITEM_TITLE = "itemTitle"; + private static final String ITEM_HOUSEID = "itemHouseId"; + private static final String MEDIA_HOUSEID = "mediaHouseId"; + private static final String MEDIA_TITLE = "mediaTitle"; + private static final String MEDIA_DESCRIPTION = "mediaDescription"; + private static final String MEDIA_TYPE = "mediaType"; + private static final String MEDIAFILE_HOUSEID = "mediaFileHouseId"; + + private SimpleDateFormat df = new SimpleDateFormat("yyyy.MM.dd HH:mm:ss"); + private Set includeList; + + public String createQuery() { + SQL isql = new SQL(); + isql.SELECT("id"); + isql.FROM("STORE"); + isql.WHERE("name = 'TSM'"); + + SQL sql = new SQL(); + sql.SELECT("i.id AS itemId"); + sql.SELECT("i.houseid AS itemHouseId"); + sql.SELECT("i.title AS itemTitle"); + //sql.SELECT("i.description AS itemdescription"); + sql.SELECT("m.id AS mediaId"); + sql.SELECT("m.houseid AS mediaHouseId"); + sql.SELECT("m.title AS mediaTitle"); + sql.SELECT("m.description AS mediaDescription"); + sql.SELECT("mf.houseid AS mediaFileHouseId"); + sql.FROM("MEDIAFILE mf"); + sql.LEFT_OUTER_JOIN("MEDIA m ON (m.id = mf.mediaid)"); + sql.LEFT_OUTER_JOIN("ITEM i ON (i.id = m.itemid)"); + sql.WHERE(String.format("mf.storeid = (%s)", isql.toString())); + return sql.toString(); + } + + public String createCountQuery() { + SQL sql = new SQL(); + sql.SELECT("COUNT(*) as count"); + sql.FROM("MEDIA"); + return sql.toString(); + } + + @StepEntry + public Object[] execute() throws Exception { + try { + + String location = "/opt/test-mediacube/file_list_original.txt"; + includeList = loadIncludeList(location); + + IItemManager manager = getManager(); + String sql = createCountQuery(); + long[] count = new long[1]; + count[0] = 0; + manager.executeQuery(sql, rs -> { + try { + count[0] = rs.getLong("count"); + logger.info(getSessionMarker(), "Processing rs"); + } catch (Exception e) { + logger.error(getMarker(), e.getMessage()); + } + return true; + }, null); + + logger.info(getSessionMarker(), "Count {}", count[0]); + + long[] current = new long[1]; + current[0] = 0; + + logger.info(csvMarker, + "Date;Name;isProgramById;includeContains;isMetadataEquals;itemHouseId;P itemHouseId;itemHouseIdEquals;itemTitle;P itemTitle;itemTitleEquals;mediaHouseId;" + + "P mediaHouseId;mediaHouseIdEquals;mediaTitle;P mediaTitle;mediaTitleEquals;" + + "mediaDescription;P mediaDescription;mediaDescriptionEquals;"); + + sql = createQuery(); + manager.executeQuery(sql, rs -> { + + if (getJobRuntime().isWaitingCancel()) + return false; + + current[0]++; + processRecord(rs); + int p = ((int) current[0] * 100) / ((int) count[0]); + setProgress(p); + return true; + }, null); + + } catch (Exception e) { + logger.error(getSessionMarker(), e.getMessage()); + } + return null; + } + + private String set(String value) { + return value == null ? EMPTY : value; + } + + private void processRecord(ResultSet rs) { + try { + String itemHouseId = set(rs.getString(ITEM_HOUSEID)); + String itemTitle = set(rs.getString(ITEM_TITLE)); + String mediaHouseId = set(rs.getString(MEDIA_HOUSEID)); + String mediaTitle = set(rs.getString(MEDIA_TITLE)); + String mediaDescription = set(rs.getString(MEDIA_DESCRIPTION)); + String pitemHouseId = EMPTY; + String pitemTitle = EMPTY; + String pmediaHouseId = EMPTY; + String pmediaTitle = EMPTY; + String pmediaDescription = EMPTY; + + String fileName = rs.getString(MEDIAFILE_HOUSEID); + String mediaFileHouseId = fileName; + if (mediaFileHouseId.contains(DOT)) + mediaFileHouseId = mediaFileHouseId.substring(0, mediaFileHouseId.lastIndexOf(DOT)); + + user.jobengine.server.steps.shared.MetadataType metadataType = MetadataTypeDetector + .GuessMetadataType(mediaFileHouseId); + + boolean isProgramById = false; + boolean includeContains = includeList.contains(fileName); + boolean itemHouseIdEquals = false; + boolean itemTitleEquals = false; + boolean mediaHouseIdEquals = false; + boolean mediaTitleEquals = false; + boolean mediaDescriptionEquals = false; + + logger.info(getMarker(), "{} {}", mediaFileHouseId, metadataType); + ArchiveItem archiveItem = getPlanAirMetadata(mediaFileHouseId); + if (archiveItem != null) { + isProgramById = true; + + if (isProgramById) { + + pitemHouseId = set(archiveItem.getItemHouseId()); + itemHouseIdEquals = pitemHouseId.equals(itemHouseId); + + pitemTitle = set(archiveItem.getItemTitle()); + itemTitleEquals = pitemTitle.equals(itemTitle); + + pmediaHouseId = set(archiveItem.getMediaHouseId()); + mediaHouseIdEquals = pmediaHouseId.equals(mediaHouseId); + + pmediaTitle = set(archiveItem.getMediaTitle()); + mediaTitleEquals = pmediaTitle.equals(mediaTitle); + + pmediaDescription = set(archiveItem.getMediaDescription()); + mediaDescriptionEquals = pmediaDescription.equals(mediaDescription); + + boolean isMetadataEquals = itemHouseIdEquals && itemTitleEquals && mediaHouseIdEquals + && mediaTitleEquals && mediaDescriptionEquals; + + Date now = new Date(System.currentTimeMillis()); + logger.info(csvMarker, "{};{};{};{};{};{};{};{};{};{};{};{};{};{};{};{};{};{};{};{};", D(now), + fileName, YN(isProgramById), YN(includeContains), YN(isMetadataEquals), itemHouseId, + pitemHouseId, itemHouseIdEquals, itemTitle, pitemTitle, itemTitleEquals, mediaHouseId, + pmediaHouseId, mediaHouseIdEquals, mediaTitle, pmediaTitle, mediaTitleEquals, + mediaDescription, pmediaDescription, mediaDescriptionEquals); + + logger.info(getSessionMarker(), "Processed {} {}{}{}", fileName, YN(isProgramById), + YN(includeContains), YN(isMetadataEquals)); + } + } + + } catch ( + + Exception e) { + logger.error(getSessionMarker(), e.getMessage()); + } + + } + + private ArchiveItem getPlanAirMetadata(String mediaHouseId) throws Exception { + PlanAirMetadataListOptions opt = new PlanAirMetadataListOptions(); + opt.setSearch(mediaHouseId); + opt.setType(MetadataType.Material); + + BasicDBObject json = null; + List data = null; + + IMetadataProviderFactory factory = getService(IMetadataProviderFactory.class); + if (factory == null) + logger.info(getSessionMarker(), "IMetadataProviderFactory is null"); + + IMetadataProvider planairProvider = factory.getProvider(MetadataProviderType.PLANAIR); + if (planairProvider == null) + logger.info(getSessionMarker(), "IMetadataProvider is null"); + + ArchiveItem result = null; + data = planairProvider.list(opt); + if (data.size() != 0) + json = data.get(0).asJSON(); + else { + opt.setType(MetadataType.Promo); + data = planairProvider.list(opt); + if (data.size() != 0) + json = data.get(0).asJSON(); + else { + opt.setType(MetadataType.AD); + data = planairProvider.list(opt); + if (data.size() != 0) + json = data.get(0).asJSON(); + } + } + if (json != null) { + result = new ArchiveItem(); + result.setItemHouseId(json.getString(ITEM_HOUSEID)); + result.setItemTitle(json.getString(ITEM_TITLE)); + result.setMediaHouseId(json.getString(MEDIA_HOUSEID)); + result.setMediaTitle(json.getString(MEDIA_TITLE)); + result.setMediaDescription(json.getString(MEDIA_DESCRIPTION)); + result.setMediaType(json.getString(MEDIA_TYPE)); + } + return result; + } + + private char YN(boolean value) { + return value ? 'Y' : 'N'; + } + + private String D(Date value) { + return value == null ? EMPTY : df.format(value); + } + + private Set loadIncludeList(String location) throws IOException { + logger.info(getSessionMarker(), "Loading include list {}", location); + Set result = new LinkedHashSet<>(); + + Path path = Paths.get(location); + List lines = FileUtils.readLines(path.toFile()); + + String lastDir = null; + for (String line : lines) { + line = line.trim(); + if (line.startsWith("Directory of")) { + lastDir = line; + lastDir = lastDir.replace("Directory of", EMPTY); + lastDir = lastDir.replace("X:", EMPTY); + lastDir = lastDir.replace("\\", "/"); + lastDir = lastDir.trim(); + } + + if (lastDir != null && lastDir.endsWith(".STATUS")) + continue; + + if (line.startsWith("2") && line.length() > 39) { + String file = line.substring(39).trim(); + + if (file.equals(DOT) || file.equals("..")) + continue; + + String len = line.substring(21, 39).trim(); + if (len.length() != 0) { + String fullpath = "/mnt/POLC/FINISHED_SHOWS" + lastDir + "/" + file; + result.add(Paths.get(fullpath).getFileName().toString()); + } + } + + } + + return result; + } + +} diff --git a/server/-product/production/OMAR/jobs/steps/MoveJpegToIsilonStep.java b/server/-product/production/OMAR/jobs/steps/MoveJpegToIsilonStep.java new file mode 100644 index 00000000..3dc75b55 --- /dev/null +++ b/server/-product/production/OMAR/jobs/steps/MoveJpegToIsilonStep.java @@ -0,0 +1,165 @@ +package user.jobengine.server.steps; + +import java.io.File; +import java.io.IOException; +import java.nio.file.FileVisitResult; +import java.nio.file.FileVisitor; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.nio.file.SimpleFileVisitor; +import java.nio.file.StandardCopyOption; +import java.nio.file.attribute.BasicFileAttributes; +import java.text.ParseException; +import java.text.SimpleDateFormat; +import java.util.ArrayList; +import java.util.Date; +import java.util.List; + +import org.apache.commons.io.FilenameUtils; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + +import user.commons.StoreUri; +import user.commons.remotestore.RemoteStoreProtocol; + +public class MoveJpegToIsilonStep extends JobStep { + private static final Logger logger = LogManager.getLogger(); + + @StepEntry + public Object[] execute(String sourceUri, String sourceProtocol, String sourceFolder, String targetUri, + String targetProtocol, String targetFolder, String userName, String password, int port) throws Exception { + StoreUri sourceStoreUri = new StoreUri(sourceUri); + if (sourceFolder.endsWith("/")) { + sourceFolder = sourceFolder.substring(0, sourceFolder.length() - 1); + } + + List foundItems = new ArrayList<>(); + sourceStoreUri.setRootPath(sourceFolder); + sourceStoreUri.setPortNumber(port); + sourceStoreUri.setPassword(password); + sourceStoreUri.setUserName(userName); + sourceStoreUri.setProtocol(RemoteStoreProtocol.valueOf(sourceProtocol)); + + FileVisitor visitor = new SimpleFileVisitor() { + String fileNamePattern = "yyyymmdd"; + SimpleDateFormat dateFormatter = new SimpleDateFormat(fileNamePattern); + Date currentDate = new Date(); // initializes with the current date + Date dateFromFileName; + boolean isRootFolder = true; + + @Override + public FileVisitResult preVisitDirectory(Path folder, BasicFileAttributes attrs) throws IOException { + FileVisitResult result = null; + boolean isDateParseable = true; + + if (folder != null) { + String splitter = File.separator.replace("\\", "\\\\"); + String[] parentFolderSegments = folder.toString().split(splitter); + + try { + dateFromFileName = dateFormatter.parse(parentFolderSegments[parentFolderSegments.length - 1]); + + if (!(dateFromFileName.compareTo(currentDate) > 0)) { + if (isRootFolder) { + result = FileVisitResult.CONTINUE; + isRootFolder = false; + } else { + result = FileVisitResult.SKIP_SUBTREE; + } + } else { + result = FileVisitResult.CONTINUE; + } + } catch (ParseException e) { + isDateParseable = false; + logger.info("Illegal argument to parse as date: {}", + parentFolderSegments[parentFolderSegments.length - 1]); + } finally { + isRootFolder = false; + if (!isDateParseable) { + result = FileVisitResult.CONTINUE; + } + } + } + return result; + } + + @Override + public FileVisitResult visitFile(Path path, BasicFileAttributes attrs) throws IOException { + logger.info("path: {}", path.toString()); + try { + String fileExtension = FilenameUtils.getExtension(path.toString()); + + if (fileExtension.equals("jpg") || fileExtension.equals("jpeg")) { + foundItems.add(path.toString()); + StoreUri targetStoreUri = getManager() + .createStoreUri(RemoteStoreProtocol.valueOf(targetProtocol), targetUri); + targetStoreUri.setProtocol(RemoteStoreProtocol.valueOf(targetProtocol)); + targetStoreUri.setRootPath(targetFolder); + + copyFile(path, Paths.get(targetStoreUri.getRootPath())); + } + } catch (Exception e) { + logger.info("Exception: {}", e.getStackTrace()); + } + return FileVisitResult.CONTINUE; + } + + @Override + public FileVisitResult visitFileFailed(Path path, IOException exc) throws IOException { + return FileVisitResult.CONTINUE; + } + }; + + try { + Files.walkFileTree(Paths.get(sourceStoreUri.getRootPath()), visitor); + } catch (IOException e) { + logger.info("Error processing Paths.get(sourceStoreUri.getRootPath()) '{}'. System message: {}", + Paths.get(sourceStoreUri.getRootPath()), e.getStackTrace()); + logger.catching(e); + throw e; + } catch (SecurityException se) { + logger.info("SecurityException: {}", se.getStackTrace()); + } finally { + } + + return new Object[] { foundItems }; + } + + private void copyFile(Path sourceFullPath, Path targetRootPath) { + Path targetPath = Paths.get(targetRootPath.toString(), sourceFullPath.getFileName().toString()); + logger.info(getMarker(), "Root {} exists {}", targetRootPath, Files.exists(targetRootPath)); + + if (Files.exists(targetPath)) { + logger.info(getMarker(), "Skipping {}, target exists", targetPath); + return; + } + + logger.info(getMarker(), "Target {} synchronization required", targetPath); + + try { + Files.copy(sourceFullPath, targetPath, StandardCopyOption.REPLACE_EXISTING); + } catch (Exception e) { + logger.error(getMarker(), "Error synchronize {} to {}. System message: {}", sourceFullPath, targetPath, + e.getMessage()); + } + } + + protected String getPathUntilCurrentFile(String rootFolder, Path fullPath) { + String tempRootfolder = ""; + if (rootFolder.contains("/") && fullPath.toString().contains("\\")) { + tempRootfolder = rootFolder.replace('/', '\\'); + if (!tempRootfolder.endsWith("\\")) { + tempRootfolder = tempRootfolder.concat("\\"); + } + } + if (rootFolder.contains("\\") && fullPath.toString().contains("/")) { + tempRootfolder = rootFolder.replace('\\', '/'); + if (!tempRootfolder.endsWith("/")) { + tempRootfolder = tempRootfolder.concat("/"); + } + } + + return fullPath.toString().replace(rootFolder, "").substring(1); + } +} diff --git a/server/-product/production/OMAR/jobs/steps/OutputPathAndNameSelectorStep.java b/server/-product/production/OMAR/jobs/steps/OutputPathAndNameSelectorStep.java new file mode 100644 index 00000000..f0203ca5 --- /dev/null +++ b/server/-product/production/OMAR/jobs/steps/OutputPathAndNameSelectorStep.java @@ -0,0 +1,154 @@ +package user.jobengine.server.steps; + +import java.io.IOException; +import java.nio.file.FileVisitResult; +import java.nio.file.FileVisitor; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.nio.file.SimpleFileVisitor; +import java.nio.file.attribute.BasicFileAttributes; + +import org.apache.commons.lang.StringUtils; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.apache.logging.log4j.Marker; + +import user.jobengine.db.ArchivedMedia; +import user.jobengine.server.IJobEngine; +import user.jobengine.server.IJobRuntime; +import user.jobengine.server.steps.shared.EscortFiles; +import user.jobengine.server.steps.shared.MetadataType; +import user.jobengine.server.steps.shared.MetadataTypeDetector; + +public class OutputPathAndNameSelectorStep extends JobStep { + + private static final String TARGETNAMEPATTERN = "-ARCH-%s"; + + private static final Logger logger = LogManager.getLogger(); + + private Marker marker; + + private void check(String localRetrievePath, String materialOutputFolder, String promoOutputFolder, String advertisementOutputFolder, + String octopusOutputFolder, String genericOutputFolder, String houseId, String targetPathType) { + if (StringUtils.isBlank(localRetrievePath)) { + logger.error(marker, "A folyamat 'localRetrievePath' bemeneti paramétere üres."); + throw new NullPointerException("System is not configured properly, 'localRetrievePath' input parameter missing."); + } + if (StringUtils.isBlank(materialOutputFolder)) { + logger.error(marker, "A folyamat 'materialOutputFolder' bemeneti paramétere üres."); + throw new NullPointerException("System is not configured properly, 'materialOutputFolder' input parameter missing."); + } + if (StringUtils.isBlank(promoOutputFolder)) { + logger.error(marker, "A folyamat 'promoOutputFolder' bemeneti paramétere üres."); + throw new NullPointerException("System is not configured properly, 'promoOutputFolder' input parameter missing."); + } + if (StringUtils.isBlank(advertisementOutputFolder)) { + logger.error(marker, "A folyamat 'advertisementOutputFolder' bemeneti paramétere üres."); + throw new NullPointerException("System is not configured properly, 'advertisementOutputFolder' input parameter missing."); + } + if (StringUtils.isBlank(octopusOutputFolder)) { + logger.error(marker, "A folyamat 'octopusOutputFolder' bemeneti paramétere üres."); + throw new NullPointerException("System is not configured properly, 'octopusOutputFolder' input parameter missing."); + } + if (StringUtils.isBlank(genericOutputFolder)) { + logger.error(marker, "A folyamat 'genericOutputFolder' bemeneti paramétere üres."); + throw new NullPointerException("System is not configured properly, 'genericOutputFolder' input parameter missing."); + } + if (StringUtils.isBlank(houseId)) { + logger.error(marker, "A folyamat 'houseId' bemeneti paramétere üres."); + throw new NullPointerException("System is not configured properly, 'houseId' input parameter missing."); + } + if (StringUtils.isBlank(targetPathType)) { + logger.error(marker, "A folyamat 'targetPathType' bemeneti paramétere üres."); + throw new NullPointerException("System is not configured properly, 'targetPathType' input parameter missing."); + } + } + + @StepEntry + public Object[] execute(String localRetrievePath, String materialOutputFolder, String promoOutputFolder, String advertisementOutputFolder, + String octopusOutputFolder, String genericOutputFolder, String onlineOutputFolder, String houseId, String targetPathType, + ArchivedMedia archivedMedia, IJobEngine jobEngine, IJobRuntime jobRuntime) throws Exception { + marker = jobRuntime.getSessionMarker(); + check(localRetrievePath, materialOutputFolder, promoOutputFolder, advertisementOutputFolder, octopusOutputFolder, genericOutputFolder, houseId, + targetPathType); + Object[] result = null; + switch (Integer.parseInt(targetPathType)) { + case 0: + String outputFolder = getFolderById(materialOutputFolder, promoOutputFolder, advertisementOutputFolder, octopusOutputFolder, genericOutputFolder, + houseId, archivedMedia); + result = localTargetInit(localRetrievePath, outputFolder, houseId, jobRuntime); + break; + case 1: + result = localTargetInit(localRetrievePath, onlineOutputFolder, houseId, jobRuntime); + break; + case 2: + + if (archivedMedia.getTcIn() != null && archivedMedia.getTcOut() != null) + result = new Object[] { genericOutputFolder, houseId, true }; + else + result = new Object[] { null, houseId, true }; + break; + } + return result; + } + + private String getFolderById(String materialOutputFolder, String promoOutputFolder, String advertisementOutputFolder, String octopusOutputFolder, + String genericOutputFolder, String houseId, ArchivedMedia archivedMedia) throws Exception { + String id = houseId.toUpperCase(); + MetadataType mdType = MetadataTypeDetector.GuessMetadataType(id); + String result = null; + + //a groovy nem latja enumnak, hanem az objektum tulajdonsaganak + switch (mdType.toString()) { + case "OctopusPlaceholder": + case "OctopusStory": + result = octopusOutputFolder; + break; + case "TrafficMaterial": + result = materialOutputFolder; + break; + case "TrafficPromo": + result = promoOutputFolder; + break; + case "TrafficAD": + result = advertisementOutputFolder; + break; + case "Generic": + result = genericOutputFolder; + break; + } + return result; + } + + private String getPossiblePath(String id, Path targetPath) throws IOException { + String[] result = new String[] { targetPath.toString() }; + FileVisitor matcherVisitor = new SimpleFileVisitor() { + @Override + public FileVisitResult preVisitDirectory(Path dir, BasicFileAttributes attrs) throws IOException { + String dirName = dir.getFileName().toString(); + if (dirName.startsWith(id + "-") || dirName.equals(id)) { + result[0] = dir.toString(); + return FileVisitResult.TERMINATE; + } + return FileVisitResult.CONTINUE; + } + }; + Files.walkFileTree(targetPath.getParent(), matcherVisitor); + return result[0]; + } + + private Object[] localTargetInit(String localRetrievePath, String outputFolder, String houseId, IJobRuntime jobRuntime) throws IOException { + String id = houseId.toUpperCase(); + String targetPath = getPossiblePath(id, Paths.get(localRetrievePath, outputFolder, id)).toString(); + String targetNamePattern = houseId + TARGETNAMEPATTERN; + try { + EscortFiles.ensureUNCFolder(Paths.get(targetPath)); + } catch (Exception e) { + logger.error(jobRuntime.getSessionMarker(), "A cél mappa '{}' nem létezik és nem hozható létre. A rendszer hibaüzenete: {}", targetPath, + e.getMessage()); + throw e; + } + return new Object[] { targetPath, targetNamePattern, false }; + } +} diff --git a/server/-product/production/OMAR/jobs/steps/PrepareMediaRestoreStep.java b/server/-product/production/OMAR/jobs/steps/PrepareMediaRestoreStep.java new file mode 100644 index 00000000..3708aa3e --- /dev/null +++ b/server/-product/production/OMAR/jobs/steps/PrepareMediaRestoreStep.java @@ -0,0 +1,37 @@ +package user.jobengine.server.steps; + +import java.util.List; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + +import user.commons.StoreUri; +import user.commons.remotestore.RemoteStoreProtocol; +import user.jobengine.db.Media; +import user.jobengine.db.MediaFile; +import user.jobengine.db.Store; + +public class PrepareMediaRestoreStep extends JobStep { + private static final Logger logger = LogManager.getLogger(); + + @StepEntry + public Object[] execute(Media media, String tempStoreName, String tempStoreProtocol) throws Exception { + StoreUri mediaStoreUri = null; + List mediaFiles = media.getMediaFiles(); + if (mediaFiles.size() > 1) + throw new Exception("Media " + media.getId() + " already has proxy"); + for (MediaFile mediaFile : mediaFiles) { + Store store = mediaFile.getStore(); + if (store.isSystem() && !store.isLowres()) { + mediaStoreUri = store.getStoreUri(RemoteStoreProtocol.TSM); + if (mediaStoreUri != null) + break; + } + } + + StoreUri tempStoreUri = getManager().getStoreUri(tempStoreName, Enum.valueOf(RemoteStoreProtocol.class, tempStoreProtocol)); + + return new Object[] { mediaStoreUri, tempStoreUri, media.getMediaFileRealName() }; + } + +} diff --git a/server/-product/production/OMAR/jobs/steps/PrepareRemoteTranscodeStep.java b/server/-product/production/OMAR/jobs/steps/PrepareRemoteTranscodeStep.java new file mode 100644 index 00000000..5116b1ef --- /dev/null +++ b/server/-product/production/OMAR/jobs/steps/PrepareRemoteTranscodeStep.java @@ -0,0 +1,48 @@ +package user.jobengine.server.steps; + +import java.nio.file.Path; +import java.nio.file.Paths; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + +import com.ibm.nosql.json.api.BasicDBObject; + +import user.commons.configuration.SystemConfiguration; + +public class PrepareRemoteTranscodeStep extends JobStep { + static private final Logger logger = LogManager.getLogger(); + private static boolean RANDOMIZE_ARCHIVES = SystemConfiguration.getInstance().value("tsm.randomize-archives", + false); + + @StepEntry + public Object[] execute(String profileName, String fileName) throws Exception { + String hiResRoot = "m:/"; + String lowResRoot = "m:/lowres"; + + Path inputPath = Paths.get(hiResRoot, fileName); + + String realFileName = fileName; + + if (RANDOMIZE_ARCHIVES) + realFileName = realFileName.substring(9); + + String outFileName = realFileName.substring(0, realFileName.lastIndexOf(".")) + ".mp4"; + + Path relativeOutputPath = null; + if (realFileName.length() > 2) + relativeOutputPath = Paths.get(realFileName.substring(0, 1), realFileName.substring(1, 2), + realFileName.substring(2, 3), outFileName); + else + relativeOutputPath = Paths.get("0", outFileName); + + BasicDBObject parameters = new BasicDBObject(); + parameters.put("profile", profileName); + parameters.put("input", inputPath.toString()); + parameters.put("output", Paths.get(lowResRoot, relativeOutputPath.toString()).toString()); + + logger.info("Prepared for remote transode {}", parameters); + return new Object[] { parameters, relativeOutputPath.toString() }; + } + +} diff --git a/server/-product/production/OMAR/jobs/steps/QueryMissingProxyMediaStep.java b/server/-product/production/OMAR/jobs/steps/QueryMissingProxyMediaStep.java new file mode 100644 index 00000000..7df2bed0 --- /dev/null +++ b/server/-product/production/OMAR/jobs/steps/QueryMissingProxyMediaStep.java @@ -0,0 +1,74 @@ +package user.jobengine.server.steps; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + +import com.ibm.nosql.json.api.BasicDBObject; +import com.ibm.nosql.json.api.DB; +import com.ibm.nosql.json.api.DBCollection; + +import user.commons.log4j2.marker.MediaCubeUndoMarker; +import user.commons.nosql.NoSQLUtils; +import user.jobengine.db.IItemManager; +import user.jobengine.db.Media; +import user.jobengine.db.MediaFile; + +public class QueryMissingProxyMediaStep extends JobStep { + private static final Logger logger = LogManager.getLogger(); + + @StepEntry + public Object[] execute() throws Exception { + setDescription("Looking for missing proxy"); + DB db = NoSQLUtils.getNoSQLDB(); + DBCollection collection = db.getCollection("missing_lowres"); + Media media = getFirstUntranscodedMedia(collection); + + if (media == null) { + logger.info(new MediaCubeUndoMarker(getSessionMarker().getSessionID()), "Nincs feldolgozandó hiány."); + cancel(); + return null; + } + + String name = media.getMediaFileRealName(); + collection.save(new BasicDBObject("name", name)); + logger.info(getSessionMarker(), "Processing mediaID: {}", media.getId()); + return new Object[] { media }; + } + + private Media getFirstUntranscodedMedia(DBCollection collection) { + Media[] result = new Media[] { null }; + //MV + String query = "SELECT mediaid FROM VW_MISSING_PROXY_IDS WHERE HOUSEID like 'M%' or HOUSEID like 'P%' or HOUSEID like 'R%' ORDER BY modified DESC"; + + //HTV + //String query = "SELECT mediaid FROM VW_MISSING_PROXY_IDS"; + IItemManager manager = getManager(); + manager.executeQuery(query, rs -> { + try { + long mediaId = rs.getLong(1); + Media media = manager.getMedia(mediaId); + //a nevgeneralas miatt az eredeti MediaFilesName nem jo, a pontos nev kell nekunk + String name = media.getMediaFileRealName(); + //logger.info(getSessionMarker(), "Checking {}", name); + long existing = collection.find(new BasicDBObject("name", name)).count(); + if (existing > 0) { + //logger.info(getSessionMarker(), "{} is on missing_lowres list", name); + return true; + } + // 210617 proxy keszites tiltasa + MediaFile mf = manager.getSystemMediaFile(media); + if (mf.isDisableProxy()) { + //logger.info(getSessionMarker(), "Proxy disabled {}", name); + + return true; + } + + result[0] = media; + } catch (Exception e) { + logger.error(e); + } + return false; + }, null); + return result[0]; + } +} diff --git a/server/-product/production/OMAR/jobs/steps/RemoteJobStep.java b/server/-product/production/OMAR/jobs/steps/RemoteJobStep.java new file mode 100644 index 00000000..d4900d76 --- /dev/null +++ b/server/-product/production/OMAR/jobs/steps/RemoteJobStep.java @@ -0,0 +1,36 @@ +package user.jobengine.server.steps; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + +import com.ibm.nosql.json.api.BasicDBObject; + +import user.jobengine.server.steps.shared.MediaCubeClient; + +public class RemoteJobStep extends JobStep { + private static final Logger logger = LogManager.getLogger(); + + @StepEntry + public Object[] execute(String remoteServer, String template, String name, BasicDBObject remoteJobParameters) throws Exception { + setDescription("Executing on {}, template {}, profile {}", remoteServer, template, remoteJobParameters.getString("profile")); + MediaCubeClient mc = new MediaCubeClient(remoteServer); + long jobId = mc.startjob(template, name, remoteJobParameters); + logger.info(getMarker(), "Started {} on server {}", jobId, remoteServer); + while (true) { + BasicDBObject status = mc.getStatus(jobId); + if (status != null) + setProgress(status.getInt("progress")); + + Thread.sleep(2000); + + String jobStatus = status.getString("status"); + if ("SUSPENDED".equals(jobStatus)) + throw new Exception(status.getString("description")); + + if ("FINISHED".equals(jobStatus)) + break; + } + + return null; + } +} diff --git a/server/-product/production/OMAR/jobs/steps/SafeDeleteRecursiveStep.java b/server/-product/production/OMAR/jobs/steps/SafeDeleteRecursiveStep.java new file mode 100644 index 00000000..238aae23 --- /dev/null +++ b/server/-product/production/OMAR/jobs/steps/SafeDeleteRecursiveStep.java @@ -0,0 +1,416 @@ +package user.jobengine.server.steps; + +import java.io.File; +import java.io.IOException; +import java.nio.file.DirectoryStream; +import java.nio.file.FileVisitResult; +import java.nio.file.FileVisitor; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.nio.file.attribute.BasicFileAttributes; +import java.text.SimpleDateFormat; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.Date; +import java.util.LinkedHashSet; +import java.util.List; +import java.util.Set; + +import org.apache.commons.io.FileUtils; +import org.apache.commons.io.FilenameUtils; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + +import com.ibm.nosql.json.api.BasicDBObject; + +import user.commons.IEntityBase; +import user.commons.RemoteFile; +import user.commons.StoreUri; +import user.commons.mediatool.MediaInfo; +import user.commons.remotestore.RemoteStoreProtocol; +import user.jobengine.db.Item; +import user.jobengine.db.Media; +import user.jobengine.db.MediaFile; +import user.jobengine.db.MediaFileDAO; +import user.jobengine.db.Store; +import user.jobengine.server.steps.shared.EscortFiles; +import user.mediacube.metadata.interfaces.IMetadata; +import user.mediacube.metadata.interfaces.IMetadataListOptions; +import user.mediacube.metadata.interfaces.IMetadataProvider; +import user.mediacube.metadata.interfaces.IMetadataProviderFactory; +import user.mediacube.metadata.interfaces.MetadataProviderType; +import user.mediacube.metadata.interfaces.MetadataType; +import user.mediacube.metadata.interfaces.PlanAirMetadataListOptions; + +public class SafeDeleteRecursiveStep extends JobStep implements FileVisitor { + private static final Logger logger = LogManager.getLogger(); + private static final String ITEM_TITLE = "itemTitle"; + private static final String ITEM_HOUSEID = "itemHouseId"; + private static final String MEDIA_HOUSEID = "mediaHouseId"; + private static final String MEDIA_TITLE = "mediaTitle"; + private static final String MEDIA_DESCRIPTION = "mediaDescription"; + private static final String MEDIA_TYPE = "mediaType"; + private static final String KILLDATEEXT = ".killdate"; + private SimpleDateFormat df = new SimpleDateFormat("yyyy.MM.dd HH:mm:ss"); + + private List skipPathNames = Arrays.asList("!ARCHIVALAS_ALATT", EscortFiles.STATUSFOLDER, + EscortFiles.CONFLICTFOLDER); + private Set includeList; + + private boolean canReadMediaInfo(Path mediaFilePath) { + boolean result = false; + try { + MediaInfo mi = new MediaInfo(mediaFilePath); + mi.process(); + result = true; + } catch (Exception e) { + logger.warn(getSessionMarker(), e.getMessage()); + } + return result; + } + + private boolean checkArchiveItem(ArchiveItem archiveItem, Item item, Media media) { + if (archiveItem == null) + return false; + + String itemHouseId = archiveItem.getItemHouseId() == null ? "" : archiveItem.getItemHouseId(); + if (!itemHouseId.equals(item.getHouseId())) { + logger.error(getSessionMarker(), "ItemHouseId"); + return false; + } + + String itemTitle = archiveItem.getItemTitle() == null ? "" : archiveItem.getItemTitle(); + if (!itemTitle.equals(item.getTitle())) { + logger.error(getSessionMarker(), "ItemTitle"); + return false; + } + String mediaHouseId = archiveItem.getMediaHouseId() == null ? "" : archiveItem.getMediaHouseId(); + if (!mediaHouseId.equals(media.getHouseId())) { + logger.error(getSessionMarker(), "MediaHouseId"); + return false; + } + String mediaTitle = archiveItem.getMediaTitle() == null ? "" : archiveItem.getMediaTitle(); + if (!mediaTitle.equals(media.getTitle())) { + logger.error(getSessionMarker(), "MediaTitle"); + return false; + } + + String mediaDescription = archiveItem.getMediaDescription() == null ? "" : archiveItem.getMediaDescription(); + String storedMediaDescription = media.getDescription() == null ? "" : media.getDescription(); + if (!mediaDescription.equals(storedMediaDescription)) { + logger.error(getSessionMarker(), "MediaDescription"); + return false; + } + + return true; + } + + private ArchiveItem createArchiveItem(Path filePath) throws Exception { + ArchiveItem result = null; + + String fileName = filePath.getFileName().toString(); + String mediaHouseId = FilenameUtils.removeExtension(fileName); + try { + result = getPlanAirMetadata(mediaHouseId); + + if (result != null) + result.setMediaFile(filePath.toString()); + + } catch (Exception e) { + logger.error(getSessionMarker(), "PlanAir metadata error", e); + } + + return result; + } + + @StepEntry + public Object[] execute(String sourcePath) throws Exception { + logger.info(getSessionMarker(), "Starting in {}", sourcePath); + logger.info(getSessionMarker(), "{};{};{};{};{};{};{};{};{};{};{};{};{};{};{};{};{};{};", "Napló időbélyeg", + "Neve", "Elérése", "Mérete", "Létrehozva", "Módosítva", "Archiválva (MC)", "Archiválva (TSM)", + "Méret (TSM)", ".catched", ".killdate", ".nomd", "Archivált (MC)", "Archivált (TSM)", + "Metaadat egyezés", "Méret egyezés", "Törölhető", "Kezelendő"); + + try { + String location = "/opt/test-mediacube/file_list_original.txt"; + includeList = loadIncludeList(location); + Files.walkFileTree(Paths.get(sourcePath), this); + } catch (Exception e) { + logger.error(getSessionMarker(), "Az '{}' mappa elérése sikertelen. A rendszer hibaüzenete: {}", sourcePath, + e.getMessage()); + } + return null; + } + + private ArchiveItem getPlanAirMetadata(String mediaHouseId) throws Exception { + PlanAirMetadataListOptions opt = new PlanAirMetadataListOptions(); + opt.setSearch(mediaHouseId); + opt.setType(MetadataType.Material); + + BasicDBObject json = null; + List data = null; + + IMetadataProviderFactory factory = getService(IMetadataProviderFactory.class); + if (factory == null) + logger.info(getSessionMarker(), "IMetadataProviderFactory is null"); + + IMetadataProvider planairProvider = factory.getProvider(MetadataProviderType.PLANAIR); + if (planairProvider == null) + logger.info(getSessionMarker(), "IMetadataProvider is null"); + + ArchiveItem result = null; + data = planairProvider.list(opt); + if (data.size() != 0) + json = data.get(0).asJSON(); + else { + opt.setType(MetadataType.Promo); + data = planairProvider.list(opt); + if (data.size() != 0) + json = data.get(0).asJSON(); + else { + opt.setType(MetadataType.AD); + data = planairProvider.list(opt); + if (data.size() != 0) + json = data.get(0).asJSON(); + } + } + if (json != null) { + result = new ArchiveItem(); + result.setItemHouseId(json.getString(ITEM_HOUSEID)); + result.setItemTitle(json.getString(ITEM_TITLE)); + result.setMediaHouseId(json.getString(MEDIA_HOUSEID)); + result.setMediaTitle(json.getString(MEDIA_TITLE)); + result.setMediaDescription(json.getString(MEDIA_DESCRIPTION)); + result.setMediaType(json.getString(MEDIA_TYPE)); + } + return result; + } + + @Override + public FileVisitResult postVisitDirectory(Path paramT, IOException paramIOException) throws IOException { + return FileVisitResult.CONTINUE; + } + + @Override + public FileVisitResult preVisitDirectory(Path dir, BasicFileAttributes paramBasicFileAttributes) + throws IOException { + Path dirName = dir.getFileName(); + + if (skipPathNames.contains(dirName.toString())) { + //logger.info(getSessionMarker(), "PreVisit skip {}", dir); + return FileVisitResult.SKIP_SUBTREE; + } //else + //logger.info(getSessionMarker(), "PreVisit {}", dir); + + return FileVisitResult.CONTINUE; + } + + private boolean processPathItem(Path mediaPath) throws Exception { + if (getJobRuntime().isWaitingCancel()) { + cancel(); + return false; + } + + BasicFileAttributes attr = Files.readAttributes(mediaPath, BasicFileAttributes.class); + File mediaFSFile = mediaPath.toFile(); + String fileName = mediaPath.getFileName().toString(); + + if (mediaFSFile.isDirectory()) + return true; + + if (fileName.startsWith(".") || fileName.endsWith(".nomd")) + return true; + + Path nomdFile = Paths.get(mediaPath.toString() + ".nomd"); + ArchiveItem archiveItem = createArchiveItem(mediaPath); + List killDateFiles = getKillDateFiles(mediaPath); + MediaFile mediaFile = getMediaFile(fileName); + + long size = mediaFSFile.length(); + Date lastModifiedDate = new Date(attr.lastModifiedTime().toMillis()); + //Date lastAccesDate = new Date(attr.lastAccessTime().toMillis()); + Date createDate = new Date(attr.creationTime().toMillis()); + boolean catchedExists = EscortFiles.isMediaCatched(mediaPath); + boolean noMDExists = Files.exists(nomdFile); + boolean killdateExists = killDateFiles.size() > 0; + //boolean mediaInfoAvailable = canReadMediaInfo(mediaPath); + boolean mcArchived = mediaFile != null; + boolean metadataEquals = false; + boolean tsmArchived = false; + boolean sizeEquals = false; + Date mcArchivedDate = null; + long tsmSize = 0; + Date tsmBackupDate = null; + boolean includeContains = includeList.contains(mediaPath.toString()); + + if (mcArchived) { + //metadata + Media media = getManager().getMedia(mediaFile.getMediaId()); + mcArchivedDate = media.getCreated(); + Item item = getManager().getItem(media.getItemId()); + metadataEquals = checkArchiveItem(archiveItem, item, media); + + //tsm + String tsmFileName = mediaFile.getRelativePath(); + RemoteFile tsmFile = getTSMFile(tsmFileName); + tsmArchived = tsmFile != null; + if (tsmArchived) { + tsmSize = tsmFile.getSize(); + tsmBackupDate = getTSMBackupDate(tsmFileName); + sizeEquals = tsmSize == size; + } + } + + boolean canDelete = !noMDExists && catchedExists && killdateExists && mcArchived && tsmArchived + && metadataEquals; + Date now = new Date(System.currentTimeMillis()); + logger.info(getSessionMarker(), "{};{};{};{};{};{};{};{};{};{};{};{};{};{};{};{};{};{};", D(now), fileName, + mediaPath.getParent(), size, D(createDate), D(lastModifiedDate), D(mcArchivedDate), D(tsmBackupDate), + tsmSize, YN(catchedExists), YN(killdateExists), YN(noMDExists), YN(mcArchived), YN(tsmArchived), + YN(metadataEquals), YN(sizeEquals), YN(canDelete), YN(includeContains)); + + return true; + } + + private Date getTSMBackupDate(String tsmFileName) throws Exception { + Date result = null; + IMetadataProviderFactory factory = getService(IMetadataProviderFactory.class); + if (factory == null) + return null; + IMetadataProvider provider = factory.getProvider(MetadataProviderType.TSM); + if (provider == null) + return null; + + IMetadataListOptions opt = provider.createOptions(new BasicDBObject("fileName", tsmFileName)); + + List tsmContents = provider.list(opt); + if (tsmContents != null && tsmContents.size() > 0) { + + for (IMetadata md : tsmContents) { + Date backupDate = md.asJSON().getDate("backupDate"); + if (result == null) + result = backupDate; + else { + if (backupDate.after(result)) + result = backupDate; + } + } + } + return result; + } + + private char YN(boolean value) { + return value ? 'Y' : 'N'; + } + + private String D(Date value) { + return value == null ? "" : df.format(value); + } + + @Override + public FileVisitResult visitFile(Path filePath, BasicFileAttributes paramBasicFileAttributes) throws IOException { + //logger.info(getSessionMarker(), "Will checked {}", filePath); + try { + if (!processPathItem(filePath)) + return FileVisitResult.TERMINATE; + } catch (Exception e) { + logger.catching(e); + } + return FileVisitResult.CONTINUE; + } + + @Override + public FileVisitResult visitFileFailed(Path filePath, IOException paramIOException) throws IOException { + logger.info("Error archive {}", filePath); + return FileVisitResult.CONTINUE; + } + + private List getKillDateFiles(Path filePath) { + String killDateFilePattern = String.format("%s.*%s", filePath.getFileName().toString(), KILLDATEEXT); + List result = new ArrayList<>(); + Path statusPath = null; + try { + statusPath = Paths.get(filePath.getParent().toString(), EscortFiles.STATUSFOLDER); + } catch (Exception e) { + logger.catching(e); + return null; + } + File statusPathFile = statusPath.toFile(); + if (statusPathFile.exists() && statusPathFile.isDirectory()) { + try (DirectoryStream stream = Files.newDirectoryStream(statusPath, killDateFilePattern)) { + stream.forEach(p -> result.add(p)); + } catch (Exception e) { + logger.catching(e); + } + } + Collections.sort(result); + return result; + } + + MediaFile getMediaFile(String fileName) { + MediaFile result = null; + MediaFileDAO mfDAO = (MediaFileDAO) getManager().getBaseDAO(MediaFile.class); + List mfList = mfDAO.getByHouseId(fileName); + if (mfList != null && mfList.size() == 1) + result = (MediaFile) mfList.get(0); + return result; + } + + RemoteFile getTSMFile(String mcFileName) { + RemoteFile result = null; + Store tsmStore = getManager().getSystemStore(false); + StoreUri tsmStoreUri = tsmStore.getSourceStoreUri(RemoteStoreProtocol.TSM); + try { + result = tsmStoreUri.getRemoteFile(mcFileName); + } catch (Exception e) { + logger.error(e); + } finally { + tsmStoreUri.cleanUp(); + } + return result; + } + + private Set loadIncludeList(String location) throws IOException { + Set result = new LinkedHashSet<>(); + + Path path = Paths.get(location); + List lines = FileUtils.readLines(path.toFile()); + + String lastDir = null; + for (String line : lines) { + line = line.trim(); + if (line.startsWith("Directory of")) { + lastDir = line; + lastDir = lastDir.replace("Directory of", ""); + lastDir = lastDir.replace("X:", ""); + lastDir = lastDir.replace("\\", "/"); + lastDir = lastDir.trim(); + // if (!lastDir.endsWith(".STATUS")) + // System.out.println(">> " + lastDir); + } + + if (lastDir != null && lastDir.endsWith(".STATUS")) + continue; + + if (line.startsWith("2") && line.length() > 39) { + String file = line.substring(39).trim(); + + if (file.equals(".") || file.equals("..")) + continue; + + String len = line.substring(21, 39).trim(); + if (len.length() != 0) { + String fullpath = "/mnt/POLC/FINISHED_SHOWS" + lastDir + "/" + file; + //System.out.println("/mnt/POLC/FINISHED_SHOWS" + lastDir + "/" + file + " : " + len); + result.add(fullpath); + } + } + + } + + return result; + } + +} diff --git a/server/-product/production/OMAR/jobs/steps/SaveMediaProxy.java b/server/-product/production/OMAR/jobs/steps/SaveMediaProxy.java new file mode 100644 index 00000000..3f0f667d --- /dev/null +++ b/server/-product/production/OMAR/jobs/steps/SaveMediaProxy.java @@ -0,0 +1,22 @@ +package user.jobengine.server.steps; + +import user.jobengine.db.IItemManager; +import user.jobengine.db.Media; +import user.jobengine.db.MediaFile; +import user.jobengine.db.Store; + +public class SaveMediaProxy extends JobStep { + + @StepEntry + public Object[] execute(Media media, String proxyRelativePath) throws Exception { + IItemManager manager = getManager(); + Store store = manager.getCurrentLowresStore(); + MediaFile mediaFile = new MediaFile(); + mediaFile.setMedia(media); + mediaFile.setStore(store); + mediaFile.setFileType(manager.getFileType("Low-res")); + mediaFile.setRelativePath(proxyRelativePath); + manager.add(mediaFile); + return null; + } +} diff --git a/server/-product/production/OMAR/jobs/steps/TSMBackupStep.java b/server/-product/production/OMAR/jobs/steps/TSMBackupStep.java new file mode 100644 index 00000000..b526f16f --- /dev/null +++ b/server/-product/production/OMAR/jobs/steps/TSMBackupStep.java @@ -0,0 +1,239 @@ +package user.jobengine.server.steps; + +import java.io.File; +import java.io.IOException; +import java.sql.Timestamp; +import java.util.Date; +import java.util.List; + +import org.apache.commons.lang.RandomStringUtils; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.apache.logging.log4j.Marker; +import org.apache.logging.log4j.message.Message; +import org.apache.logging.log4j.message.ParameterizedMessage; + +import user.commons.RemoteFile; +import user.commons.StoreUri; +import user.commons.configuration.SystemConfiguration; +import user.commons.mediatool.Timecode; +import user.commons.mediatool.Timecode.Type; +import user.commons.remotestore.IProgressEventListener; +import user.commons.remotestore.IStatusEventListener; +import user.commons.remotestore.ProgressEvent; +import user.commons.remotestore.RemoteStoreProtocol; +import user.commons.remotestore.StatusEvent; +import user.jobengine.db.FileType; +import user.jobengine.db.IItemManager; +import user.jobengine.db.Media; +import user.jobengine.db.MediaFile; +import user.jobengine.db.Store; +import user.jobengine.server.IJobEngine; +import user.jobengine.server.IJobRuntime; +import user.jobengine.server.JobEngineException; +import user.jobengine.server.steps.shared.EscortFiles; +import user.jobengine.server.steps.shared.ItemManagerExtensions; + +public class TSMBackupStep extends JobStep { + private static final String MXFEXT = ".MXF"; + private static final Logger logger = LogManager.getLogger(); + private static boolean RANDOMIZE_ARCHIVES = SystemConfiguration.getInstance().value("tsm.randomize-archives"); + private IItemManager manager; + private File sourceMediaFile; + private Store tsmStore; + private StoreUri targetUri; + private FileType fileType; + private Marker marker; + + @StepEntry + public Object[] execute(ArchiveItem archiveItem, Media mediaCubeMedia, int killDateDays) throws Exception { + marker = getSessionMarker(); + + File sourceMediaFile = new File(archiveItem.getMediaFile()); + String sourceFileName = sourceMediaFile.getName(); + long fileSize = sourceMediaFile.length(); + + try { + Timecode timecode = new Timecode(mediaCubeMedia.getLength(), Type.PAL); + String details = String.format("%s (%s, %d bytes)", sourceFileName, timecode.toString(), fileSize); + logger.info(marker, details); + getJobRuntime().setDescription(details); + } catch (Exception e) { + String details = String.format("%s (%d bytes)", sourceFileName, fileSize); + getJobRuntime().setDescription(details); + } + Timecode timecode = new Timecode(mediaCubeMedia.getLength(), Type.PAL); + String details = String.format("%s (%s, %d bytes)", sourceFileName, timecode.toString(), fileSize); + getJobRuntime().setDescription(details); + try { + + setAndCheck(archiveItem, mediaCubeMedia, getEngine()); + + // TODO mxf helyett az osszes kiterjesztest!!!!! + // A dupla ellenorzes a napon beluli ismetlesek miatt kell + long existingMediaId = archiveItem.getExistingMediaId(); + + if (fileSize == 0 && existingMediaId == 0) { + existingMediaId = ItemManagerExtensions.getExistingRundownMedia(manager, + sourceFileName.replace(MXFEXT, "")); + if (existingMediaId == 0) + existingMediaId = -1; + } + + if (existingMediaId == 0) + existingMediaId = ItemManagerExtensions.getExistingRundownMedia(manager, + sourceFileName.replace(MXFEXT, "")); + String targetFileName; + if (RANDOMIZE_ARCHIVES) { + // a-z, A-Z, 0-9. For example: WRMcpIk7, s57JwCVA + // veletlenszeru neveket adunk! + targetFileName = String.format("%s-%s", RandomStringUtils.randomAlphanumeric(8), sourceFileName); + } else + targetFileName = sourceFileName; + + if (existingMediaId == 0) { + StoreUri sourceUri = manager.createStoreUri(RemoteStoreProtocol.LOCAL, + sourceMediaFile.getParent().toString()); + + final IJobRuntime runtime = getJobRuntime(); + sourceUri.addProgressListener(new IProgressEventListener() { + @Override + public void progressChanged(ProgressEvent evt) { + runtime.incrementProgress(evt.getProgress()); + } + }); + sourceUri.addStatusListener(new IStatusEventListener() { + @Override + public void statusChanged(StatusEvent evt) { + evt.setCancel(!canContinue()); + } + }); + + RemoteFile remoteFile = sourceUri.transferFrom(targetUri, sourceFileName, targetFileName); + } + + if (existingMediaId > 0) + logger.info(marker, "Az '{}' TSM mentése nem szükséges, mert már megtalálható az archívumban.", + sourceFileName); + + // Fel kell szabadítani, hogy a kovetkezo archivalaskor is nekifusson + if (existingMediaId == -1) { + logger.info(marker, + "Az '{}' mentése jelenleg nem lehetséges, mert a szükséges metaadat még nem található meg az archívumban.", + sourceFileName); + if (!archiveItem.removeCatchedFile()) + logger.error(marker, + "Az '{}' állomány .catched jelző állománya nem törölhető. Az újabb archiválási kísérlethez annak kézi eltávolítása szükséges!", + sourceMediaFile.getName()); + mediaCubeMedia.remove(); + } else { + saveMetadata(mediaCubeMedia, sourceMediaFile, targetFileName, existingMediaId, fileSize, + archiveItem.isDisableProxy()); + logger.info(marker, "Az '{}' archiválása sikeres.", sourceFileName); + if (killDateDays != 0) + EscortFiles.createUNCKillDate(sourceMediaFile.getParent(), sourceFileName, killDateDays, marker); + } + + } catch (Exception e) { + logger.catching(e); + Message m = new ParameterizedMessage("Az '{}' állomány archiválása sikertelen. A rendszer hibaüzenete: {}", + details, e.getMessage()); + logger.error(marker, m); + if (!archiveItem.removeCatchedFile()) + logger.error(marker, + "Az '{}' állomány .catched jelző állománya nem törölhető. Az újabb archiválási kísérlethez annak kézi eltávolítása szükséges!", + sourceMediaFile.getName()); + throw new Exception(m.getFormattedMessage()); + } + return null; + } + + private void saveMetadata(Media mediaCubeMedia, File sourceFile, String targetFileName, long existingMediaId, + long fileSize, boolean disableProxy) { + + if (existingMediaId == 0) { + MediaFile mf = manager.createMediaFile(targetFileName, fileType, tsmStore, mediaCubeMedia); + mf.setHouseId(sourceFile.getName()); + mf.setFileSize(fileSize); + // 210617 proxy keszites tiltasa + mf.setDisableProxy(disableProxy); + mf.add(); + } else { + Media existingMedia = manager.getMedia(existingMediaId); + List mediaFiles = existingMedia.getMediaFiles(); + if (mediaFiles != null) { + for (MediaFile mf : mediaFiles) { + mf.setPersister(manager); + mf.setId(0); + mf.setMedia(mediaCubeMedia); + // mivel itt masolat keszul, nem allitunk at semmit +// mf.setFileSize(fileSize); +// mf.setDisableProxy(disableProxy); + mf.add(); + } + } + } + mediaCubeMedia.setPersister(manager); + + // 210614 megis maradjon az aktualis idopont + mediaCubeMedia.setArchived(new Timestamp(new Date().getTime())); + + /* + try { + BasicFileAttributes attr = Files.readAttributes(sourceFile.toPath(), BasicFileAttributes.class); + mediaCubeMedia.setArchived(new Timestamp(attr.creationTime().toMillis())); + } catch (IOException e) { + logger.catching(e); + } + */ + mediaCubeMedia.modify(); + } + + private void setAndCheck(ArchiveItem archiveItem, Media mediaCubeMedia, IJobEngine jobEngine) + throws JobEngineException, IOException { + if (jobEngine == null) { + logger.error(marker, "Az folyamatkezelő réteg nem elérhető."); + throw new NullPointerException("Internal error, missing JobEngine reference."); + } + manager = jobEngine.getItemManager(); + if (manager == null) { + logger.error(marker, "Az adatbáziskezelő réteg nem elérhető."); + throw new NullPointerException("Internal error, missing ItemManager reference."); + } + if (archiveItem == null) { + logger.error(marker, "A folyamat 'archiveItem' bemeneti paramétere üres."); + throw new NullPointerException("System is not configured properly, missing 'archiveItem' input parameter."); + } + sourceMediaFile = new File(archiveItem.getMediaFile()); + if (sourceMediaFile == null) { + logger.error(marker, "A folyamat 'archiveItem' bemeneti paraméter 'mediaFile' értéke üres."); + throw new NullPointerException( + "System is not configured properly, missing 'mediaFile' value in 'archiveItem' input parameter."); + } + if (!sourceMediaFile.exists()) { + logger.error(marker, "A(z) {} állomány nem létezik vagy nem érhető el.", sourceMediaFile.getName()); + throw new IOException(String.format("Input file {} does not exist or unreachable.", sourceMediaFile.getName())); + } + tsmStore = manager.getSystemStore(false); + if (tsmStore == null) { + logger.error(marker, "A TSM rendszer beállítás nem elérhető."); + throw new NullPointerException("System is not configured properly, missing TSM Store."); + } + targetUri = tsmStore.getSourceStoreUri(RemoteStoreProtocol.TSM); + if (targetUri == null) { + logger.error(marker, "A TSM rendszer beállítás paraméterei nem elérhetőek."); + throw new NullPointerException("System is not configured properly, missing TSM StoreUri."); + } + fileType = manager.getFileType("High-res"); + if (fileType == null) { + logger.error(marker, "Adatbázis bejegyzés hiba, a 'High-res' FileType nem található."); + throw new NullPointerException("System is not configured properly, missing 'High-res' FileType."); + } + if (mediaCubeMedia == null) { + logger.error(marker, "A folyamat 'mediaCubeMedia' bemeneti paramétere üres."); + throw new NullPointerException( + "System is not configured properly, 'mediaCubeMedia' input parameter missing."); + } + + } +} diff --git a/server/-product/production/OMAR/jobs/steps/TSMExtendedRetrieveStep.java b/server/-product/production/OMAR/jobs/steps/TSMExtendedRetrieveStep.java new file mode 100644 index 00000000..fd222344 --- /dev/null +++ b/server/-product/production/OMAR/jobs/steps/TSMExtendedRetrieveStep.java @@ -0,0 +1,95 @@ +package user.jobengine.server.steps; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + +import user.commons.StoreUri; +import user.commons.remotestore.RemoteStoreProtocol; +import user.jobengine.db.ArchivedMedia; +import user.jobengine.db.IItemManager; +import user.jobengine.server.IJobEngine; +import user.jobengine.server.IJobRuntime; + +/* +import user.jobengine.server.steps.EscortFiles; +*/ + +public class TSMExtendedRetrieveStep extends TSMRestoreStep { + private static String NEXIO_HOST = System.getProperty("nexio.host"); + // private static String NEXIO_HOST = SystemConfiguration.getInstance().value("services.nexio.host"); + private static final Logger logger = LogManager.getLogger(); + + private boolean useNexioTarget; + private int nexioPort; + private String nexioUserName, nexioPassword; + private String nexioAgency; + + @Override + protected void afterRestore(StoreUri targetUri, String targetPath, int killDateDays, String targetFileName) throws Exception { + if (useNexioTarget) { + EscortFiles.setNEXIOKillDate(killDateDays, targetFileName, nexioAgency, targetUri); + } else { + super.afterRestore(targetUri, targetPath, killDateDays, targetFileName); + } + } + + @Override + protected void beforeRestore(StoreUri targetURI, String targetName) throws Exception { + String newTargetName = targetName; + if (targetName.contains(".")) + newTargetName = targetName.substring(0, targetName.lastIndexOf('.')); + if (useNexioTarget) + if (targetURI.fileExists(newTargetName + ".mxf")) + throw new Exception(String.format("%s-The newly created file name is existed.", getClass().getSimpleName())); + } + + @Override + protected void checkTargetPath(String targetPath) { + if (!useNexioTarget) + super.checkTargetPath(targetPath); + } + + @Override + protected StoreUri createTargetUri(IItemManager manager, String targetPath) throws NullPointerException { + StoreUri result = null; + logger.info(getSessionMarker(), "Create target uri {}", targetPath); + if (useNexioTarget) { + if (NEXIO_HOST == null) { + throw new NullPointerException("Missing system property on 'nexio.host' name"); + } + result = manager.createStoreUri(RemoteStoreProtocol.FTP, NEXIO_HOST); + result.setPortNumber(nexioPort); + result.setUserName(nexioUserName); + result.setPassword(nexioPassword); + } else + result = super.createTargetUri(manager, targetPath); + return result; + } + + @StepEntry + public Object[] execute(ArchivedMedia archivedMedia, String targetPath, String targetNamePattern, String successRecipient, int killDateDays, + String localRetrievePath, String globalRetrievePath, boolean useNexioTarget, String nexioAgency, int nexioPort, String nexioUserName, + String nexioPassword, IJobEngine jobEngine, IJobRuntime jobRuntime) throws Exception { + this.useNexioTarget = useNexioTarget; + this.nexioAgency = nexioAgency; + this.nexioPort = nexioPort; + this.nexioUserName = nexioUserName; + this.nexioPassword = nexioPassword; + if (nexioPort == 0) { + throw new NullPointerException("System is not configured properly, 'nexioPort' input parameter missing."); + } + if (nexioUserName == null) { + throw new NullPointerException("System is not configured properly, 'nexioUserName' input parameter missing."); + } + if (nexioPassword == null) { + throw new NullPointerException("System is not configured properly, 'nexioPassword' input parameter missing."); + } + if (nexioAgency == null) { + throw new NullPointerException("System is not configured properly, 'nexioAgency' input parameter missing."); + } + + return super.execute(archivedMedia.getMedia(), targetPath, targetNamePattern, successRecipient, killDateDays, localRetrievePath, globalRetrievePath, + jobEngine, jobRuntime); + } + +} diff --git a/server/-product/production/OMAR/jobs/steps/TSMRestoreStep.java b/server/-product/production/OMAR/jobs/steps/TSMRestoreStep.java new file mode 100644 index 00000000..26d8607b --- /dev/null +++ b/server/-product/production/OMAR/jobs/steps/TSMRestoreStep.java @@ -0,0 +1,187 @@ +package user.jobengine.server.steps; + +import java.io.IOException; +import java.nio.file.Paths; +import java.text.Normalizer; +import java.util.List; +import java.util.regex.Pattern; + +import org.apache.commons.lang.StringUtils; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.apache.logging.log4j.Marker; +import org.apache.logging.log4j.message.Message; + +import user.commons.LogUtils; +import user.commons.RemoteFile; +import user.commons.StoreUri; +import user.commons.mediatool.Timecode; +import user.commons.mediatool.Timecode.Type; +import user.commons.remotestore.IProgressEventListener; +import user.commons.remotestore.IStatusEventListener; +import user.commons.remotestore.ProgressEvent; +import user.commons.remotestore.RemoteStoreProtocol; +import user.commons.remotestore.StatusEvent; +import user.jobengine.db.IItemManager; +import user.jobengine.db.Media; +import user.jobengine.db.MediaFile; +import user.jobengine.db.Store; +import user.jobengine.server.IJobEngine; +import user.jobengine.server.IJobRuntime; +import user.jobengine.server.steps.shared.EscortFiles; + +public class TSMRestoreStep extends JobStep { + private static final String DOT = "."; + public static final Pattern DIACRITICS_AND_FRIENDS = Pattern.compile("[\\p{InCombiningDiacriticalMarks}\\p{IsLm}\\p{IsSk}]+"); + private static final Logger logger = LogManager.getLogger(); + private IItemManager manager; + private StoreUri targetUri; + private StoreUri sourceUri; + private String sourceFileName; + private Marker marker; + + protected void afterRestore(StoreUri targetUri, String targetPath, int killDateDays, String targetFileName) throws IOException, Exception { + if (killDateDays != 0) + EscortFiles.createUNCKillDate(targetPath, targetFileName, killDateDays, marker); + } + + protected void beforeRestore(StoreUri targetURI, String targetFileName) throws Exception { + } + + protected void checkTargetPath(String targetPath) { + if (StringUtils.isBlank(targetPath)) { + logger.error(marker, "A folyamat 'targetPath' bemeneti paramétere üres."); + throw new NullPointerException("System is not configured properly, 'targetPath' input parameter missing."); + } + } + + protected StoreUri createTargetUri(IItemManager manager, String targetPath) { + return manager.createStoreUri(RemoteStoreProtocol.LOCAL, targetPath); + } + + @StepEntry + public Object[] execute(Media mediaCubeMedia, String targetPath, String targetNamePattern, String successRecipient, int killDateDays, + String localRetrievePath, String globalRetrievePath, IJobEngine jobEngine, IJobRuntime jobRuntime) throws Exception { + marker = jobRuntime.getSessionMarker(); + setAndCheck(mediaCubeMedia, targetPath, targetNamePattern, localRetrievePath, globalRetrievePath, jobEngine); + String targetFileName = String.format(targetNamePattern, sourceFileName); + //20210129 + //targetFileName = getMaximizedFileName(mediaCubeMedia, targetFileName, 120); + + Timecode timecode = new Timecode(mediaCubeMedia.getLength(), Type.PAL); + try { + String details = String.format("%s (%s)", sourceFileName, timecode.toString()); + jobRuntime.setDescription(details); + beforeRestore(targetUri, targetNamePattern); + final IJobRuntime runtime = jobRuntime; + sourceUri.addProgressListener(new IProgressEventListener() { + @Override + public void progressChanged(ProgressEvent evt) { + runtime.incrementProgress(evt.getProgress()); + } + }); + sourceUri.addStatusListener(new IStatusEventListener() { + @Override + public void statusChanged(StatusEvent evt) { + evt.setCancel(!canContinue()); + } + }); + RemoteFile result = sourceUri.transferFrom(targetUri, sourceFileName, targetFileName); + + String globalTargetPath = Paths.get(targetPath, targetFileName) + .getParent() + .toString() + .replace(Paths.get(localRetrievePath).toString(), globalRetrievePath); + + logger.info(marker, + "Az '{}' állomány visszatöltése sikeres volt '{}' néven. A célmappa a ide kattintva nyitható meg.", + sourceFileName, targetFileName, globalTargetPath); + afterRestore(targetUri, targetPath, killDateDays, targetFileName); + + } catch (Exception e) { + Message msg = LogUtils.format("Az '{}' állomány visszatöltése sikertelen. A rendszer hibaüzenete: {}", sourceFileName, e.getMessage()); + logger.error(marker, msg); + // logger.error(jobRuntime.marker, msg); + logger.catching(e); + throw e; + } + + return null; + } + + private String getMaximizedFileName(Media mediaCubeMedia, String targetFileName, int limit) { + String name = targetFileName; + String extension = ""; + if (name.contains(DOT)) { + extension = DOT + name.substring(name.lastIndexOf(DOT) + 1); + name = name.substring(0, name.lastIndexOf(DOT)); + } + String typeName = Normalizer.normalize(mediaCubeMedia.getItemType().getName(), Normalizer.Form.NFD); + typeName = DIACRITICS_AND_FRIENDS.matcher(typeName).replaceAll(""); + typeName = typeName.replace(" ", "_"); + + int allowedSize = limit - typeName.length() - 1 - extension.length(); + if (name.length() > allowedSize) + name = name.substring(0, allowedSize); + + return String.format("%s_%s%s", name, typeName, extension); + } + + private String getSourceFileName(Media mediaCubeMedia, Store store) { + List mediaFiles = mediaCubeMedia.getMediaFiles(); + if (mediaFiles == null) + return null; + for (MediaFile mediaFile : mediaFiles) { + if (mediaFile.getStore().getId() == store.getId()) + return mediaFile.getRelativePath(); + } + return null; + } + + private void setAndCheck(Media mediaCubeMedia, String targetPath, String targetNamePattern, String localRetrievePath, String globalRetrievePath, + IJobEngine jobEngine) { + if (jobEngine == null) { + logger.error(marker, "Az folyamatkezelő réteg nem elérhető."); + throw new NullPointerException("Internal error, missing JobEngine reference."); + } + manager = jobEngine.getItemManager(); + if (manager == null) { + logger.error(marker, "Az adatbáziskezelő réteg nem elérhető."); + throw new NullPointerException("Internal error, missing ItemManager reference."); + } + if (mediaCubeMedia == null) { + logger.error(marker, "A folyamat 'mediaCubeMedia' bemeneti paramétere üres."); + throw new NullPointerException("System is not configured properly, 'mediaCubeMedia' input parameter missing."); + } + checkTargetPath(targetPath); + if (StringUtils.isBlank(targetNamePattern)) { + logger.error(marker, "A folyamat 'targetNamePattern' bemeneti paramétere üres."); + throw new NullPointerException("System is not configured properly, 'targetNamePattern' input parameter missing."); + } + Store tsmStore = manager.getSystemStore(false); + if (tsmStore == null) { + logger.error(marker, "A TSM rendszer beállítás nem elérhető."); + throw new NullPointerException("System is not configured properly, missing TSM Store."); + } + sourceUri = tsmStore.getSourceStoreUri(RemoteStoreProtocol.TSM); + if (sourceUri == null) { + logger.error(marker, "A TSM rendszer beállítás paraméterei nem elérhetőek."); + throw new NullPointerException("System is not configured properly, missing TSM StoreUri."); + } + targetUri = createTargetUri(manager, targetPath); + sourceFileName = getSourceFileName(mediaCubeMedia, tsmStore); + if (sourceFileName == null) { + logger.error(marker, "Adatbázis bejegyzés hiba, a visszatöltendő fájl neve nem található."); + throw new NullPointerException("Database error, missing MediaFile 'relativePath'."); + } + + if (StringUtils.isBlank(localRetrievePath)) { + logger.error(marker, "A folyamat 'localRetrievePath' bemeneti paramétere üres."); + throw new NullPointerException("System is not configured properly, 'localRetrievePath' input parameter missing."); + } + if (StringUtils.isBlank(globalRetrievePath)) { + logger.error(marker, "A folyamat 'globalRetrievePath' bemeneti paramétere üres."); + throw new NullPointerException("System is not configured properly, 'globalRetrievePath' input parameter missing."); + } + } +} diff --git a/server/-product/production/OMAR/jobs/steps/TSMSimpleRestoreStep.java b/server/-product/production/OMAR/jobs/steps/TSMSimpleRestoreStep.java new file mode 100644 index 00000000..ba63212c --- /dev/null +++ b/server/-product/production/OMAR/jobs/steps/TSMSimpleRestoreStep.java @@ -0,0 +1,57 @@ +package user.jobengine.server.steps; + +import java.nio.file.Files; +import java.nio.file.Paths; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + +import user.commons.StoreUri; +import user.commons.mediatool.Timecode; +import user.commons.mediatool.Timecode.Type; +import user.commons.remotestore.IProgressEventListener; +import user.commons.remotestore.IStatusEventListener; +import user.commons.remotestore.ProgressEvent; +import user.commons.remotestore.RemoteStoreProtocol; +import user.commons.remotestore.StatusEvent; +import user.jobengine.db.Media; +import user.jobengine.db.MediaFile; +import user.jobengine.db.Store; + +public class TSMSimpleRestoreStep extends JobStep { + private static final Logger logger = LogManager.getLogger(); + + @StepEntry + public Object[] execute(long mediaId, String targetPath) throws Exception { + String fileName = null; + try { + Media media = getManager().getMedia(mediaId); + MediaFile mediaFile = getManager().getSystemMediaFile(media); + fileName = mediaFile.getRelativePath(); + Timecode timecode = new Timecode(media.getLength(), Type.PAL); + getJobRuntime().setDescription(String.format("%s (%s)", fileName, timecode.toString())); + Store tsmStore = getManager().getSystemStore(false); + StoreUri sourceStoreUri = getManager().getStoreUri(tsmStore.getName(), RemoteStoreProtocol.TSM); + sourceStoreUri.addProgressListener(new IProgressEventListener() { + @Override + public void progressChanged(ProgressEvent evt) { + setProgress(evt.getProgress()); + } + }); + sourceStoreUri.addStatusListener(new IStatusEventListener() { + @Override + public void statusChanged(StatusEvent evt) { + evt.setCancel(!canContinue()); + } + }); + StoreUri targetStoreUri = getManager().createStoreUri(RemoteStoreProtocol.LOCAL, targetPath); + sourceStoreUri.transferFrom(targetStoreUri, fileName, fileName + ".part"); + Files.move(Paths.get(targetPath, fileName + ".part"), Paths.get(targetPath, fileName)); + } catch (Exception e) { + logger.error("Az '{}' állomány visszatöltése sikertelen. A rendszer üzenete: {}", fileName, e.getMessage()); + throw e; + } + + return null; + } +} diff --git a/server/-product/production/OMAR/jobs/steps/TestForkCancelableStep.java b/server/-product/production/OMAR/jobs/steps/TestForkCancelableStep.java new file mode 100644 index 00000000..c1b49f76 --- /dev/null +++ b/server/-product/production/OMAR/jobs/steps/TestForkCancelableStep.java @@ -0,0 +1,35 @@ +package user.jobengine.server.steps; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + +import user.commons.IJob; +import user.commons.JobStatus; +import user.commons.ListUtils; +import user.jobengine.server.IJobEngine; +import user.jobengine.server.IJobRuntime; + +public class TestForkCancelableStep extends JobStep { + private static final String CHILD_TITLE = "Párhuzamosított alfolyamat"; + // private static final String CHILD_TEMPLATE = "fake-concurrent.xml"; + private static final String CHILD_TEMPLATE = "cancelable.xml"; + private static final Logger logger = LogManager.getLogger(); + int count = 3; + + @StepEntry + public Object[] execute(IJobEngine jobEngine, IJobRuntime jobRuntime) throws Exception { + for (int i = 0; i < count; i++) { + IJobRuntime rt = getEngine().submit(null, e -> { + if (e.getStatus().equals(JobStatus.SUSPENDED)) { + IJobRuntime rt = (IJobRuntime) e.getSource(); + logger.info("Cleanup on SUSPEND {}, {}", rt.getId(), rt.isDisableRetry()); + } + }, CHILD_TEMPLATE, "JOB " + i, 0, IJobEngine.DEFAULT_OWNER, ListUtils.asMap("param", i)); + + rt.setRelated("TEST" + rt.getId()); + } + + logger.info("Done"); + return null; + } +} diff --git a/server/-product/production/OMAR/jobs/steps/TranscodeFFAStranStep.java b/server/-product/production/OMAR/jobs/steps/TranscodeFFAStranStep.java new file mode 100644 index 00000000..d09685bc --- /dev/null +++ b/server/-product/production/OMAR/jobs/steps/TranscodeFFAStranStep.java @@ -0,0 +1,151 @@ +package user.jobengine.server.steps; + +import java.io.File; +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; + +import org.apache.commons.io.FilenameUtils; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.apache.logging.log4j.Marker; +import org.apache.logging.log4j.message.Message; +import org.apache.logging.log4j.message.ParameterizedMessage; + +import user.commons.FFAStransAPI; +import user.commons.IFFAStransAPI; +import user.commons.StoreUri; +import user.commons.mediatool.Timecode; +import user.commons.mediatool.Timecode.Type; +import user.commons.remotestore.RemoteStoreProtocol; +import user.jobengine.db.FileType; +import user.jobengine.db.IItemManager; +import user.jobengine.db.Media; +import user.jobengine.db.Store; +import user.jobengine.server.IJobEngine; +import user.jobengine.server.IJobRuntime; + +public class TranscodeFFAStranStep extends JobStep { + private static final int POLL_INTERVALL = 3000; + private static final String MP4EXT = ".MP4"; + private static final String MXFEXT = ".MXF"; + private static final String LOWRES_FILETYPE = "Low-res"; + private static final Logger logger = LogManager.getLogger("TranscodeFFAStranStep"); + private IItemManager manager; + private Store store; + private FileType fileType; + private Media mediaCubeMedia; + private Marker marker; + + @StepEntry + public Object[] execute(ArchiveItem archiveItem, Media mediaCubeMedia, String transcoderAddress, String transcoderTemplateName, + String globalHiresSourcePath, String localLowresTargetPath, boolean deleteSource, IJobEngine jobEngine, IJobRuntime jobRuntime) throws Exception { + + this.marker = jobRuntime.getSessionMarker(); + this.manager = jobEngine.getItemManager(); + this.store = check(manager.getCurrentLowresStore(), "lowres Store"); + this.fileType = check(manager.getFileType(LOWRES_FILETYPE), "lowres FileType"); + this.mediaCubeMedia = check(mediaCubeMedia, "mediaCubeMedia"); + check(archiveItem, "archiveItem"); + check(transcoderAddress, "transcoderAddress"); + check(transcoderTemplateName, "transcoderTemplateName"); + check(globalHiresSourcePath, "globalHiresSourcePath"); + check(localLowresTargetPath, "localLowresTargetPath"); + + File sourceMediaFile = new File(archiveItem.getMediaFile()); + logger.info("Transcoding {}", archiveItem.getMediaFile()); + String sourceFileName = sourceMediaFile.getName(); + Timecode timecode = new Timecode(mediaCubeMedia.getLength(), Type.PAL); + + String details = String.format("%s (%s, %d bytes)", sourceFileName, timecode.toString(), sourceMediaFile.length()); + + StoreUri storeUri = store.getTargetStoreUri(RemoteStoreProtocol.LOCAL); + if (storeUri == null) + throw new Exception("Can not detect proxy folder."); + + String webPath = storeUri.toString(true); + + Path targetPath = null; + try { + String targetFileName = FilenameUtils.removeExtension(sourceFileName) + MP4EXT; + targetPath = Paths.get(localLowresTargetPath, targetFileName); + if (!Files.exists(targetPath)) { + // jobRuntime.setDescription(String.format("%s: %s", jobRuntime.getDescription(), details)); + jobRuntime.setDescription(String.format("%s transzkódolása", details)); + String sourceFile = Paths.get(globalHiresSourcePath, sourceFileName).toString(); + IFFAStransAPI api = new FFAStransAPI(transcoderAddress, p -> { + if (p <= 100) + jobRuntime.incrementProgress(p); + }); + + api.submit(transcoderTemplateName, sourceFile); + api.monitor(POLL_INTERVALL); + } + + //a sikeres transzkod utan nem mindig van ott egybol a fajl + long started = System.currentTimeMillis(); + while (!Files.exists(targetPath)) { + long current = System.currentTimeMillis(); + //max 5 perc varakozas + if (current - started > 5 * 60 * 1000) + throw new Exception("Transcode job target file access timed out"); + Thread.sleep(POLL_INTERVALL); + } + + postprocess(targetPath, webPath); + + } catch (Exception e) { + logger.catching(e); + Message m = new ParameterizedMessage("{} átkódolás hiba: {}", sourceFileName, e.getMessage()); + logger.error(marker, m); + throw new Exception(m.getFormattedMessage()); + } finally { + try { + if (deleteSource && sourceMediaFile != null && sourceMediaFile.exists()) + sourceMediaFile.delete(); + } catch (Exception e) { + logger.catching(e); + } + try { + if (deleteSource && targetPath != null && Files.exists(targetPath)) + Files.delete(targetPath); + } catch (Exception e) { + logger.catching(e); + } + } + return null; + } + + private void postprocess(Path transcodedFilePath, String webPath) throws IOException { + Path lowresPath = null; + try { + String transcodedFileName = transcodedFilePath.getFileName().toString(); + String targetPath = null; + if (transcodedFileName.indexOf(".") > 2) { + Path subdir = Paths.get(transcodedFileName.substring(0, 1), transcodedFileName.substring(1, 2), transcodedFileName.substring(2, 3)); + EscortFiles.ensureUNCFolder(webPath, subdir.toString()); + targetPath = Paths.get(subdir.toString(), transcodedFileName).toString(); + } else { + targetPath = transcodedFileName; + } + lowresPath = Paths.get(webPath, targetPath); + int version = 1; + while (Files.exists(lowresPath)) { + String fileName = transcodedFileName + version + MP4EXT; + lowresPath = Paths.get(lowresPath.toString().replace(transcodedFileName, fileName)); + targetPath = targetPath.replace(transcodedFileName, fileName); + transcodedFileName = fileName; + version++; + } + + Files.move(transcodedFilePath, lowresPath); + manager.createMediaFile(targetPath, fileType, store, mediaCubeMedia).add(); + } catch (IOException e) { + logger.catching(e); + logger.error(marker, "A '{}' állomány mozgatása a '{}' helyre nem sikerült.", transcodedFilePath, lowresPath); + throw e; + } + } + +} diff --git a/server/-product/production/OMAR/jobs/steps/TransferStep.java b/server/-product/production/OMAR/jobs/steps/TransferStep.java new file mode 100644 index 00000000..c10084fa --- /dev/null +++ b/server/-product/production/OMAR/jobs/steps/TransferStep.java @@ -0,0 +1,110 @@ +package user.jobengine.server.steps; + +import java.nio.file.Path; +import java.nio.file.Paths; + +import org.apache.commons.net.ftp.FTPClient; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + +import user.commons.RemoteFile; +import user.commons.StoreUri; +import user.commons.remotestore.FtpDirectoryLister; +import user.commons.remotestore.RemoteStoreProtocol; +import user.jobengine.db.Store; + +public class TransferStep extends JobStep { + private static final String DOT_PART = ".part"; + private static final Logger logger = LogManager.getLogger(); + + private void copy(StoreUri sourceStoreUri, String sourceFileName, StoreUri targetStoreUri, String targetFileName) throws Exception { + String currentTargetFileName = targetFileName; + + boolean renameAfterCopy = false; + boolean renameAfterFTP = false; + if (getTmpExtension() != null) { + if (RemoteStoreProtocol.LOCAL.equals(targetStoreUri.getProtocol())) { + currentTargetFileName += getTmpExtension(); + renameAfterCopy = true; + } + Store targetStore = getManager().getStore(targetStoreUri.getStoreId()); + + if (RemoteStoreProtocol.FTP.equals(targetStoreUri.getProtocol()) && !"NEXIO1".equals(targetStore.getName()) + && !"NEXIO2".equals(targetStore.getName())) { + currentTargetFileName += getTmpExtension(); + renameAfterFTP = true; + } + } + + sourceStoreUri.transferFrom(targetStoreUri, sourceFileName, currentTargetFileName); + + logger.info(getMarker(), "Transfer of {} completed from {} to {}", sourceFileName, sourceStoreUri, currentTargetFileName); + + if (renameAfterCopy) { + Path tmpTargetFile = Paths.get(targetStoreUri.toString(true), currentTargetFileName); + Path targetFile = Paths.get(targetStoreUri.toString(true), targetFileName); + try { + logger.info(getMarker(), "Renaming LOCAL file from {} to {} on {}", currentTargetFileName, targetFileName, sourceStoreUri); + tmpTargetFile.toFile().renameTo(targetFile.toFile()); + } catch (Exception e) { + logger.error(getMarker(), e.getMessage()); + } + } + if (renameAfterFTP) { + try { + FtpDirectoryLister lister = (FtpDirectoryLister) targetStoreUri.getLister(); + FTPClient client = lister.connect(); + logger.info(getMarker(), "Renaming FTP file from {} to {} on {}", currentTargetFileName, targetFileName, sourceStoreUri); + client.rename(currentTargetFileName, targetFileName); + } catch (Exception e) { + logger.error(getMarker(), e.getMessage()); + } finally { + targetStoreUri.cleanUp(); + } + } + } + + @StepEntry + public Object[] execute(StoreUri sourceStoreUri, String sourceFileName, StoreUri targetStoreUri, String targetFileName) throws Exception { + try { + getJobRuntime().setCancelable(false); + + Store sourceStore = getManager().getStore(sourceStoreUri.getStoreId()); + Store targetStore = getManager().getStore(targetStoreUri.getStoreId()); + + setDescription("{} -> {} : {}", sourceStore.getName(), targetStore.getName(), sourceFileName); + + sourceStoreUri.addProgressListener(e -> setProgress(e.getProgress())); + + StoreUri currentTargetStoreUri = getTargetStoreUri(targetStoreUri); + + try { + RemoteFile remoteFile = currentTargetStoreUri.getRemoteFile(targetFileName); + if (remoteFile != null) { + logger.info("File {} already exists on target {}, skipping transfer", targetFileName, targetStore.getName()); + } + } catch (Exception e) { + } + + copy(sourceStoreUri, sourceFileName, currentTargetStoreUri, targetFileName); + } catch (Exception e) { + logger.error(getMarker(), "Error in transfer of {} when copying from {} to {}.", sourceFileName, sourceStoreUri, targetStoreUri); + throw e; + } finally { + getJobRuntime().setDescription(null); + if (sourceStoreUri != null) + sourceStoreUri.cleanUp(); + if (targetStoreUri != null) + targetStoreUri.cleanUp(); + } + return null; + } + + protected StoreUri getTargetStoreUri(StoreUri targetStoreUri) { + return targetStoreUri; + } + + protected String getTmpExtension() { + return DOT_PART; + } +} diff --git a/server/-product/production/OMAR/jobs/steps/UpdateGhostMediaDataStep.java b/server/-product/production/OMAR/jobs/steps/UpdateGhostMediaDataStep.java new file mode 100644 index 00000000..39cb7b30 --- /dev/null +++ b/server/-product/production/OMAR/jobs/steps/UpdateGhostMediaDataStep.java @@ -0,0 +1,89 @@ +package user.jobengine.server.steps; + +import java.util.List; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.apache.logging.log4j.Marker; + +import user.commons.remotestore.RemoteStoreProtocol; +import user.jobengine.db.IItemManager; +import user.jobengine.db.Media; +import user.jobengine.db.MediaFile; +import user.jobengine.db.Store; +import user.jobengine.server.IJobEngine; +import user.jobengine.server.IJobRuntime; +import user.jobengine.server.steps.shared.MetadataType; +import user.jobengine.server.steps.shared.MetadataTypeDetector; + +public class UpdateGhostMediaDataStep extends JobStep { + private static final Logger logger = LogManager.getLogger(); + private Marker marker; + + @StepEntry + public Object[] execute(Media mediaCubeMedia, IJobEngine jobEngine, IJobRuntime jobRuntime) throws Exception { + marker = jobRuntime.getSessionMarker(); + + IItemManager manager = jobEngine.getItemManager(); + //Refresh from db + List mediaFiles = manager.getMedia(mediaCubeMedia.getId()).getMediaFiles(); + if (mediaFiles != null && mediaFiles.size() == 2) { + MediaFile lowres = null; + MediaFile highres = null; + + for (MediaFile mf : mediaFiles) { + if (mf.getStore().getSourceStoreUri(RemoteStoreProtocol.HTTP) != null) + lowres = mf; + else + highres = mf; + } + + if (highres == null) { + logger.info(marker, "Nincs highres mediaId: {}", mediaCubeMedia.getId()); + return null; + } + if (lowres == null) { + logger.info(marker, "Nincs lowres mediaId: {}", mediaCubeMedia.getId()); + return null; + } + + String id = MetadataTypeDetector.truncateExtension(highres.getRelativePath()); + id = MetadataTypeDetector.truncateVersion(id); + boolean detect = MetadataTypeDetector.GuessMetadataType(id) == MetadataType.OctopusPlaceholder + || MetadataTypeDetector.GuessMetadataType(id) == MetadataType.OctopusStory; + if (!detect) { + logger.info(marker, "Nem bejátszó mediaId: {}, file: {}", mediaCubeMedia.getId(), highres.getRelativePath()); + return null; + } + + Store highresStore = manager.getSystemStore(false); + final long sourceMediaId = lowres.getId(); + final long highresMediaFileId = highres.getId(); + final String highresRealtivePath = highres.getRelativePath(); + + manager.executeQuery("SELECT mediaid FROM mediafile WHERE relativepath=? and storeid=? and id!=?", rs -> { + long mediaId = rs.getLong(1); + Media media = manager.getMedia(mediaId); + if (media.getMediaFilesCount() == 1) { + logger.info(marker, "Hiányzó szellem lowres hozzáadása {} alapján", media.getId()); + + MediaFile mf = (MediaFile) manager.get(MediaFile.class, sourceMediaId); + mf.setMedia(media); + mf.setId(0); + manager.add(mf); + media.setLength(mediaCubeMedia.getLength()); + manager.modify(media); + } + return true; + }, st -> { + st.setString(1, highresRealtivePath); + st.setLong(2, highresStore.getId()); + st.setLong(3, highresMediaFileId); + }); + + } + + return null; + } + +} diff --git a/server/-product/production/OMAR/jobs/steps/ValidateProResStep.java b/server/-product/production/OMAR/jobs/steps/ValidateProResStep.java new file mode 100644 index 00000000..e36f784d --- /dev/null +++ b/server/-product/production/OMAR/jobs/steps/ValidateProResStep.java @@ -0,0 +1,32 @@ +package user.jobengine.server.steps; + +import java.util.HashMap; +import java.util.Map; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + +import user.commons.mediaarea.MediaArea; + +public class ValidateProResStep extends JobStep { + private static final Logger logger = LogManager.getLogger(); + + @StepEntry + public Object[] execute(Map proResFiles) { + Map files = new HashMap(); + + if (!proResFiles.isEmpty()) { + for (String fileName : proResFiles.keySet()) { + MediaArea mediaArea = proResFiles.get(fileName); + String videoFormat = mediaArea.getFormat(); + + if (mediaArea.getFrameRate() == 23.976 && videoFormat.equals("PRORES")) { + files.put(fileName, mediaArea); + } + } + } else { + logger.info("proResFiles is empty!"); + } + return new Object[] { files }; + } +} diff --git a/server/-product/production/OMAR/jobs/steps/shared/EscortFiles.java b/server/-product/production/OMAR/jobs/steps/shared/EscortFiles.java new file mode 100644 index 00000000..c888f722 --- /dev/null +++ b/server/-product/production/OMAR/jobs/steps/shared/EscortFiles.java @@ -0,0 +1,348 @@ +package user.jobengine.server.steps.shared; + +import java.io.ByteArrayInputStream; +import java.io.File; +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; +import java.io.StringWriter; +import java.io.UnsupportedEncodingException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.nio.file.attribute.FileAttribute; +import java.nio.file.attribute.PosixFilePermission; +import java.nio.file.attribute.PosixFilePermissions; +import java.text.SimpleDateFormat; +import java.util.Calendar; +import java.util.Date; +import java.util.Set; + +import javax.xml.parsers.DocumentBuilder; +import javax.xml.parsers.DocumentBuilderFactory; +import javax.xml.transform.OutputKeys; +import javax.xml.transform.Transformer; +import javax.xml.transform.TransformerConfigurationException; +import javax.xml.transform.TransformerException; +import javax.xml.transform.TransformerFactory; +import javax.xml.transform.TransformerFactoryConfigurationError; +import javax.xml.transform.dom.DOMSource; +import javax.xml.transform.stream.StreamResult; + +import org.apache.commons.lang.StringUtils; +import org.apache.commons.net.ftp.FTPClient; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.apache.logging.log4j.Marker; +import org.apache.logging.log4j.message.Message; +import org.w3c.dom.DOMImplementation; +import org.w3c.dom.Document; +import org.w3c.dom.Element; + +import com.ibm.nosql.json.api.BasicDBObject; +import com.ibm.nosql.json.util.JSON; + +import user.commons.CalendarUtils; +import user.commons.log4j2.marker.MediaCubeMarker; +import user.commons.StoreUri; +import user.commons.remotestore.FtpDirectoryLister; + +public class EscortFiles { + private static final String RECORDTIMESTAMP = "RecordTimeStamp"; + private static final String MODIFIEDTIMESTAMP = "ModifiedTimeStamp"; + public static final String DOT_CATCHED = ".catched"; + public static final String DOT_JSON = ".json"; + private static final Logger logger = LogManager.getLogger(); + private static final String EXTENDEDAGENCY = "ExtendedAgency"; + private static final String EXTENDEDDESCRIPTION = "ExtendedDescription"; + private static final String KILLDATE = "KillDate"; + private static final String FORMAT_KILLDATE = "MM-dd-yyyy"; + private static final String EXTENDEDID = "extendedId"; + private static final String ID = "ID"; + private static final String KILLDATE_FILENAME = "%s.%s.killdate"; + private static final String FORMAT_KILLDATENAME = "yyyyMMdd"; + public static final String STATUSFOLDER = ".STATUS"; + public static final String CONFLICTFOLDER = ".CONFLICT"; + + public static String composeKillDate(int days) { + Calendar killDate = Calendar.getInstance(); + killDate.add(Calendar.DAY_OF_YEAR, days); + SimpleDateFormat dateFormat = new SimpleDateFormat(FORMAT_KILLDATENAME); + return dateFormat.format(killDate.getTime()); + } + /* + * + * 02-02-2018 TEST + * TEST AGENT AGENT + */ + + private static String composeKillDateFileName(String fileName, int days) { + return String.format(KILLDATE_FILENAME, fileName, composeKillDate(days)); + } + + public static void createCatchedFile(Path escortFile) throws IOException { + Path catchedFilePath = Paths.get(escortFile.toString() + DOT_CATCHED); + Files.createFile(catchedFilePath); + } + + public static void createFellow(String escortFile, String extension) throws IOException { + Files.copy(Paths.get(escortFile), Paths.get(escortFile + "." + extension)); + } + + /*** + * A media eleresi utjan alapjan a .STATUS almappaban letrehozza a .catch fajlt. + * + * @param mediaFile + * @throws IOException + */ + public static void createMediaCatch(Path mediaFile) throws IOException { + Path catchedFile = createMediaCatchFilePath(mediaFile); + ensureUNCFolder(catchedFile.getParent()); + Files.createFile(catchedFile); + } + + public static Path createMediaCatchFilePath(Path mediaFile) { + String fileName = mediaFile.getFileName().toString() + DOT_CATCHED; + return Paths.get(mediaFile.getParent().toString(), STATUSFOLDER, fileName); + } + + public static void createMetadata(String filePath, String fileName, String metadata) throws IOException { + ensureUNCFolder(filePath, STATUSFOLDER); + String metadataFileName = fileName + DOT_JSON; + Path metadataPath = Paths.get(filePath, STATUSFOLDER, metadataFileName); + Files.write(metadataPath, metadata.getBytes()); + } + + public static boolean createMetadataIfNotExists(String filePath, String fileName, String metadata) + throws IOException { + boolean result = false; + if (!EscortFiles.isMetadataExists(filePath, fileName)) { + EscortFiles.createMetadata(filePath, fileName, metadata); + result = true; + } + return result; + } + + public static void createMorpheusXML(String filePath, String fileName, String content) throws IOException { + ensureUNCFolder(filePath, STATUSFOLDER); + Path xmlPath = Paths.get(filePath, fileName); + if (Files.exists(xmlPath)) + throw new IOException(String.format("Az '%s' állomány már létezik.", xmlPath)); + Files.write(xmlPath, content.getBytes()); + } + + public static byte[] createNEXIODatesMeta(String fileName, Date recorded, Date modified) throws Exception { + DocumentBuilderFactory dbf = DocumentBuilderFactory.newInstance(); + DocumentBuilder db = dbf.newDocumentBuilder(); + DOMImplementation impl = db.getDOMImplementation(); + Document xmlDocument = impl.createDocument(null, null, null); + + Element root = xmlDocument.createElement(ID); + root.setAttribute(EXTENDEDID, fileName); + // 07-13-2020 (19:36:52) + // 05-18-2013 (18:52:24) + SimpleDateFormat df = new SimpleDateFormat("MM-dd-yyyy (HH:mm:ss)"); + root.appendChild(xmlDocument.createElement(MODIFIEDTIMESTAMP)) + .appendChild(xmlDocument.createTextNode(df.format(modified))); + root.appendChild(xmlDocument.createElement(RECORDTIMESTAMP)) + .appendChild(xmlDocument.createTextNode(df.format(recorded))); + xmlDocument.appendChild(root); + + return xmDocumentToString(xmlDocument); + } + + public static byte[] createNEXIOKillDateFile(String fileName, Date killDate, String description, String agency) + throws Exception { + DocumentBuilderFactory dbf = DocumentBuilderFactory.newInstance(); + DocumentBuilder db = dbf.newDocumentBuilder(); + DOMImplementation impl = db.getDOMImplementation(); + Document xmlDocument = impl.createDocument(null, null, null); + + Element root = xmlDocument.createElement(ID); + root.setAttribute(EXTENDEDID, fileName); + if (killDate != null) { + String sKillDate = CalendarUtils.toString(CalendarUtils.createCalendar(killDate), FORMAT_KILLDATE); + root.appendChild(xmlDocument.createElement(KILLDATE)).appendChild(xmlDocument.createTextNode(sKillDate)); + } + + if (StringUtils.isNotBlank(description)) + root.appendChild(xmlDocument.createElement(EXTENDEDDESCRIPTION)) + .appendChild(xmlDocument.createTextNode(description)); + if (StringUtils.isNotBlank(agency)) + root.appendChild(xmlDocument.createElement(EXTENDEDAGENCY)).appendChild(xmlDocument.createTextNode(agency)); + xmlDocument.appendChild(root); + + return xmDocumentToString(xmlDocument); + } + + public static Document createNEXIOMeta(byte[] content) throws Exception { + DocumentBuilderFactory dbf = DocumentBuilderFactory.newInstance(); + DocumentBuilder db = dbf.newDocumentBuilder(); + DOMImplementation impl = db.getDOMImplementation(); + Document xmlDocument = null; + + try (InputStream is = new ByteArrayInputStream(content)) { + xmlDocument = db.parse(is); + } catch (Exception e) { + logger.catching(e); + } + + return xmlDocument; + } + + public static void createUNCKillDate(String filePath, String fileName, int days, Marker marker) throws IOException { + ensureUNCFolder(filePath, STATUSFOLDER); + String killDateFileName = composeKillDateFileName(fileName, days); + Path killDatePath = Paths.get(filePath, STATUSFOLDER, killDateFileName); + if (Files.exists(killDatePath)) + logger.warn(marker, "Az '{}' állomány már létezik.", killDatePath); + else + Files.createFile(killDatePath); + } + + @SuppressWarnings("unchecked") + public static T decode(Path escortFile) { + T result = null; + try { + byte[] bytes = Files.readAllBytes(escortFile); + String content = new String(bytes); + result = (T) JSON.parse(content); + } catch (Exception e) { + logger.error("Decode error. System message is: ", e.getMessage()); + } + return result; + } + + public static void ensureUNCFolder(Path filePath) throws IOException { + File folder = filePath.toFile(); + if (!folder.exists() || !folder.isDirectory()) { + try { + Set perms = PosixFilePermissions.fromString("rwxrwxrwx"); + FileAttribute> attr = PosixFilePermissions.asFileAttribute(perms); + Files.createDirectories(filePath, attr); + } catch (Exception e) { + // logger.catching(e); + try { + Files.createDirectories(filePath); + } catch (Exception e1) { + logger.catching(e1); + throw e1; + } + } + } + } + + public static void ensureUNCFolder(String filePath, String folderName) throws IOException { + Path statusPath = Paths.get(filePath, folderName); + ensureUNCFolder(statusPath); + } + + public static boolean isCatchedFileExists(Path escortFile) { + Path catchedFilePath = Paths.get(escortFile.toString() + DOT_CATCHED); + return Files.exists(catchedFilePath); + } + + /*** + * A media elérési útján alapján a .STATUS almappában vizsgálja .catch fajl + * létezését. + * + * @param mediaFile + * @return + */ + public static boolean isMediaCatched(Path mediaFile) { + Path catchedFile = createMediaCatchFilePath(mediaFile); + return Files.exists(catchedFile); + } + + public static boolean isMetadataExists(String filePath, String fileName) throws IOException { + boolean result = false; + String metadataFileName = fileName + DOT_JSON; + Path metadataPath = Paths.get(filePath, STATUSFOLDER, metadataFileName); + result = Files.exists(metadataPath); + return result; + } + + public static void notifyRecipient(Path escortFile, Logger logger, Message msg) { + if (Files.exists(escortFile)) { + try { + BasicDBObject downloadable = EscortFiles.decode(escortFile); + String recipientKey = "recipient"; + if (downloadable.containsKey(recipientKey)) { + String recipient = downloadable.getString(recipientKey); + logger.info(new MediaCubeMarker(recipient, "MediaCube rendszerüzenet"), msg); + } + } catch (Exception e) { + logger.catching(e); + } + + } + } + + public static void remove(Path file) { + try { + file.toFile().delete(); + } catch (Exception e) { + logger.error("Unable to delete {}", file.toAbsolutePath().toString()); + } + } + + public static void removeCatchedFile(Path escortFile) { + remove(Paths.get(escortFile.toString() + DOT_CATCHED)); + } + + /*** + * A media eleresi utjan alapjan a .STATUS almappabol torli a .catch fajlt. + * + * @param mediaFile + * @throws IOException + */ + public static void removeMediaCatch(Path mediaFile) { + Path catchedFile = createMediaCatchFilePath(mediaFile); + remove(catchedFile); + } + + public static void setNEXIOKillDate(int killDateDays, String targetFileName, String nexioAgency, StoreUri targetUri) + throws Exception { + OutputStream outStream = null; + try { + FTPClient targetFTP = ((FtpDirectoryLister) targetUri.getLister()).connect(); + Calendar killDate = CalendarUtils.createCalendar(new Date()); + killDate.add(Calendar.DAY_OF_YEAR, killDateDays); + if (targetFileName.toLowerCase().contains(".mxf")) + targetFileName = targetFileName.substring(0, targetFileName.lastIndexOf('.')); + byte[] killDateFile = EscortFiles.createNEXIOKillDateFile(targetFileName, killDate.getTime(), null, + nexioAgency); + String xml = targetFileName + ".xml"; + outStream = targetFTP.storeFileStream(xml); + if (outStream == null) { + throw new NullPointerException( + "Can not open: " + targetFileName.substring(0, targetFileName.lastIndexOf('.')) + ".xml" + + " Reply:" + targetFTP.getReplyString()); + } + outStream.write(killDateFile); + outStream.flush(); + } catch (Exception e) { + throw e; + } finally { + if (outStream != null) + outStream.close(); + targetUri.cleanUp(); + } + } + + private static byte[] xmDocumentToString(Document xmlDocument) throws TransformerFactoryConfigurationError, + TransformerConfigurationException, TransformerException, IOException, UnsupportedEncodingException { + DOMSource domSource = new DOMSource(xmlDocument); + TransformerFactory tf = TransformerFactory.newInstance(); + Transformer transformer = tf.newTransformer(); + transformer.setOutputProperty(OutputKeys.METHOD, "xml"); + transformer.setOutputProperty(OutputKeys.ENCODING, "UTF-16"); + transformer.setOutputProperty(OutputKeys.INDENT, "yes"); + StringWriter sw = new StringWriter(); + StreamResult sr = new StreamResult(sw); + transformer.transform(domSource, sr); + String result = sw.toString(); + sw.close(); + return result.getBytes("UTF-16"); + } +} diff --git a/server/-product/production/OMAR/jobs/steps/shared/ExternalCommand.java b/server/-product/production/OMAR/jobs/steps/shared/ExternalCommand.java new file mode 100644 index 00000000..307f36d0 --- /dev/null +++ b/server/-product/production/OMAR/jobs/steps/shared/ExternalCommand.java @@ -0,0 +1,79 @@ +package user.jobengine.server.steps.shared; + +import java.io.BufferedReader; +import java.io.InputStreamReader; +import java.util.ArrayList; +import java.util.List; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + +public class ExternalCommand { + private static final Logger logger = LogManager.getLogger(); + private ExternalProfile profile; + + public ExternalCommand(ExternalProfile profile) { + this.profile = profile; + } + + public String execute(String input, String output, boolean firstResponse, IExternalCallback responseCallBack) throws Exception { + List arguments = getArguments(input, output); + List command = new ArrayList<>(); + command.add(profile.getExecutable()); + command.addAll(arguments); + + ProcessBuilder processBuilder = new ProcessBuilder(); + processBuilder.command(command); + + String result = null; + try { + logger.info("Executing : {}", processBuilder.command()); + + Process process = processBuilder.start(); + try (BufferedReader reader = new BufferedReader(new InputStreamReader(process.getInputStream()))) { + String line = null; + while ((line = reader.readLine()) != null) { + logger.debug("Process response: {}", line); + if (responseCallBack != null) + responseCallBack.onResponse(line); + //System.out.println(line); + if (line != null && line.length() > 0) { + result = line; + if (firstResponse) + break; + } + } + int exitCode = process.waitFor(); + if (exitCode != 0) { + StringBuilder msg = new StringBuilder(); + try (BufferedReader errReader = new BufferedReader(new InputStreamReader(process.getErrorStream()))) { + String errline = null; + while ((errline = errReader.readLine()) != null) { + msg.append(errline); + } + } catch (Exception ex) { + } + + throw new Exception("Exited with error code : " + exitCode + ". " + msg); + } + } catch (Exception e) { + throw e; + } + } catch (Exception e) { + logger.error(e); + throw e; + } + + return result; + } + + private List getArguments(String input, String output) { + List result = new ArrayList<>(); + + profile.getArguments().forEach(i -> { + result.add(i.replace("%i", input).replace("%o", output)); + }); + return result; + } + +} \ No newline at end of file diff --git a/server/-product/production/OMAR/jobs/steps/shared/ExternalCommandExecutor.java b/server/-product/production/OMAR/jobs/steps/shared/ExternalCommandExecutor.java new file mode 100644 index 00000000..a34ff7f0 --- /dev/null +++ b/server/-product/production/OMAR/jobs/steps/shared/ExternalCommandExecutor.java @@ -0,0 +1,32 @@ +package user.jobengine.server.steps.shared; + +import user.commons.configuration.SystemConfiguration; + +public class ExternalCommandExecutor { + + public void execute(String profileName, String input, String output, IExternalCallback responseCallBack) throws Exception { + ExternalCommand externalCommand = getExternalCommand(profileName); + externalCommand.execute(input, output, false, responseCallBack); + } + + private ExternalCommand getExternalCommand(String profileName) throws Exception { + ExternalProfilesConfig config = SystemConfiguration.getInstance().load("settings/external-commands.yaml", ExternalProfilesConfig.class); + + if (config == null) + throw new Exception("Missing external-commands.yaml configuration"); + + ExternalProfile selectedProfile = null; + for (ExternalProfile profile : config.getProfiles()) { + if (profileName.equals(profile.getName())) { + selectedProfile = profile; + break; + } + } + + if (selectedProfile == null) + throw new Exception("Missing profile " + profileName + " in external-commands.yaml configuration"); + + return new ExternalCommand(selectedProfile); + } + +} diff --git a/server/-product/production/OMAR/jobs/steps/shared/ExternalProfile.java b/server/-product/production/OMAR/jobs/steps/shared/ExternalProfile.java new file mode 100644 index 00000000..68e22f4c --- /dev/null +++ b/server/-product/production/OMAR/jobs/steps/shared/ExternalProfile.java @@ -0,0 +1,33 @@ +package user.jobengine.server.steps.shared; + +import java.util.List; + +public class ExternalProfile { + private String executable; + private String name; + private List arguments; + + public List getArguments() { + return arguments; + } + + public String getExecutable() { + return executable; + } + + public String getName() { + return name; + } + + public void setArguments(List arguments) { + this.arguments = arguments; + } + + public void setExecutable(String executable) { + this.executable = executable; + } + + public void setName(String name) { + this.name = name; + } +} \ No newline at end of file diff --git a/server/-product/production/OMAR/jobs/steps/shared/ExternalProfilesConfig.java b/server/-product/production/OMAR/jobs/steps/shared/ExternalProfilesConfig.java new file mode 100644 index 00000000..7ef77291 --- /dev/null +++ b/server/-product/production/OMAR/jobs/steps/shared/ExternalProfilesConfig.java @@ -0,0 +1,15 @@ +package user.jobengine.server.steps.shared; + +import java.util.List; + +public class ExternalProfilesConfig { + private List profiles; + + public List getProfiles() { + return profiles; + } + + public void setProfiles(List profiles) { + this.profiles = profiles; + } +} \ No newline at end of file diff --git a/server/-product/production/OMAR/jobs/steps/shared/FileSearchFilterOptions.java b/server/-product/production/OMAR/jobs/steps/shared/FileSearchFilterOptions.java new file mode 100644 index 00000000..b5a8d0e8 --- /dev/null +++ b/server/-product/production/OMAR/jobs/steps/shared/FileSearchFilterOptions.java @@ -0,0 +1,41 @@ +package user.jobengine.server.steps.shared; + +import java.nio.file.Path; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +import com.ibm.nosql.json.api.BasicDBObject; + +public class FileSearchFilterOptions { + + private BasicDBObject filter; + + public FileSearchFilterOptions(BasicDBObject filter) { + this.filter = filter; + } + + public boolean acceptFile(Path file) { + if (filter == null) + return true; + + if (filter.containsKey("fileName")) { + //.*\.(sh|ini|conf|vhost|xml|php)$ + String fileNamePattern = filter.getString("fileName"); + if (fileNamePattern == null || fileNamePattern.trim().length() == 0) + return true; + + Pattern pattern = Pattern.compile(fileNamePattern, Pattern.CASE_INSENSITIVE); + + Matcher matcher = pattern.matcher(file.getFileName().toString()); + if (matcher.find()) + return true; + + } + + return false; + } + + public boolean preAcceptDirectory(Path file) { + return true; + } +} diff --git a/server/-product/production/OMAR/jobs/steps/shared/IExternalCallback.java b/server/-product/production/OMAR/jobs/steps/shared/IExternalCallback.java new file mode 100644 index 00000000..1e071913 --- /dev/null +++ b/server/-product/production/OMAR/jobs/steps/shared/IExternalCallback.java @@ -0,0 +1,5 @@ +package user.jobengine.server.steps.shared; + +public interface IExternalCallback { + void onResponse(String data); +} \ No newline at end of file diff --git a/server/-product/production/OMAR/jobs/steps/shared/ItemManagerExtensions.java b/server/-product/production/OMAR/jobs/steps/shared/ItemManagerExtensions.java new file mode 100644 index 00000000..32f17cb5 --- /dev/null +++ b/server/-product/production/OMAR/jobs/steps/shared/ItemManagerExtensions.java @@ -0,0 +1,94 @@ +package user.jobengine.server.steps.shared; + +import java.nio.file.Path; + +import com.ibm.nosql.json.api.BasicDBObject; + +import user.commons.RemoteFile; +import user.commons.StoreUri; +import user.commons.remotestore.RemoteStoreProtocol; +import user.jobengine.db.IItemManager; +import user.jobengine.db.IResultSetConsumer; +import user.jobengine.db.Store; + +public class ItemManagerExtensions { + + public static BasicDBObject getArchiveInfo(IItemManager manager, long houseid) { + final BasicDBObject[] result = { null }; + StringBuilder query = new StringBuilder(); + query.append("select count(*) as count, sum(length) as duration FROM media"); + query.append(" "); + query.append(String.format("where houseid='%d' and itemtypeid = 82", houseid)); + query.append(" "); + query.append("group by houseid"); + IResultSetConsumer consumer = rs -> { + BasicDBObject o = new BasicDBObject(); + o.put("count", rs.getLong("count")); + o.put("duration", rs.getLong("duration")); + result[0] = o; + return false; + }; + manager.executeQuery(query.toString(), consumer, null); + return result[0]; + } + + public static long getExistingRundownMedia(IItemManager manager, String houseid) { + final long[] result = new long[] { 0 }; + final String[] idToCheck = new String[] { houseid }; + int pos = houseid.lastIndexOf("-"); + //a hivas a CopyForArchiveNEXIOMaterialsStep-bol is johet, ott meg nincs idobelyegezve a nev! + if (pos > 0 && houseid.length() - pos > 4) + idToCheck[0] = houseid.substring(0, pos); + MetadataType metadataType = MetadataTypeDetector.GuessMetadataType(idToCheck[0]); + if (metadataType == MetadataType.OctopusPlaceholder) { + StringBuilder query = new StringBuilder(); + query.append("select mediaid, mediafilehouseid, filename"); + query.append(" "); + query.append(String.format("from vw_rundown_items where mediafilehouseid like '%s%%'", idToCheck[0])); + query.append(" "); + query.append("order by filename, mediaid"); + IResultSetConsumer consumer = rs -> { + String fileName = rs.getString("filename"); + if (idToCheck[0].equals(fileName)) { + result[0] = rs.getLong("mediaid"); + return false; + } else + return true; + }; + manager.executeQuery(query.toString(), consumer, null); + } + + return result[0]; + } + + static public boolean isArchived(IItemManager manager, Path filePath) { + boolean result = false; + String name = filePath.getFileName().toString(); + String[] tsmName = new String[] { null }; + String query = String.format("SELECT relativepath FROM MEDIAFILE WHERE houseid = '%s'", name); + manager.executeQuery(query, rs -> { + tsmName[0] = rs.getString("relativepath"); + return false; + }, null); + + Store tsmStore = manager.getSystemStore(false); + if (tsmStore == null) + throw new NullPointerException("A TSM bejegyzés nem található!"); + + StoreUri tsmStoreUri = tsmStore.getSourceStoreUri(RemoteStoreProtocol.TSM); + if (tsmStoreUri == null) + throw new NullPointerException("A TSM forrás elérése nem található!"); + + if (tsmName[0] != null) { + try { + RemoteFile remoteFile = tsmStoreUri.getRemoteFile(tsmName[0]); + result = remoteFile != null; + } catch (Exception e) { + result = false; + } finally { + tsmStoreUri.cleanUp(); + } + } + return result; + } +} diff --git a/server/-product/production/OMAR/jobs/steps/shared/MediaCubeClient.java b/server/-product/production/OMAR/jobs/steps/shared/MediaCubeClient.java new file mode 100644 index 00000000..0915cc33 --- /dev/null +++ b/server/-product/production/OMAR/jobs/steps/shared/MediaCubeClient.java @@ -0,0 +1,77 @@ +package user.jobengine.server.steps.shared; + +import javax.ws.rs.client.Entity; +import javax.ws.rs.client.Invocation.Builder; +import javax.ws.rs.core.MediaType; +import javax.ws.rs.core.MultivaluedMap; +import javax.ws.rs.core.Response; +import javax.ws.rs.core.Response.Status; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.jboss.resteasy.client.jaxrs.ResteasyClient; +import org.jboss.resteasy.client.jaxrs.ResteasyClientBuilder; +import org.jboss.resteasy.client.jaxrs.ResteasyWebTarget; +import org.jboss.resteasy.specimpl.MultivaluedMapImpl; + +import com.ibm.nosql.json.JSONUtil; +import com.ibm.nosql.json.api.BasicDBObject; + +public class MediaCubeClient { + private static Logger logger = LogManager.getLogger(); + private ResteasyWebTarget webTarget; + + public MediaCubeClient(String address) { + ResteasyClient client = new ResteasyClientBuilder().build(); + webTarget = client.target(address); + } + + BasicDBObject getDbObject(String json) { + BasicDBObject result = (BasicDBObject) JSONUtil.jsonToDbObject(json); + + if (result == null) + throw new NullPointerException("API Result is null!"); + + if (result.containsKey("exception")) { + BasicDBObject e = (BasicDBObject) result.get("exception"); + throw new RuntimeException(e.getString("message")); + } + //{"exception":{"message":"Invalid credentials.","publicName":"AuthenticationFailedException"}} + return result; + } + + public BasicDBObject getStatus(long jobId) { + MultivaluedMap vars = new MultivaluedMapImpl<>(); + vars.add("jobId", jobId); + Response response = query("services/rest/jobengine/jobstatus", vars).get(); + if (response.getStatus() != Status.OK.getStatusCode()) { + logger.error(response.readEntity(String.class)); + System.out.println(response.readEntity(String.class)); + return null; + } + String result = response.readEntity(String.class); + return getDbObject(result); + } + + private Builder query(String path, MultivaluedMap vars) { + ResteasyWebTarget target = webTarget.path(path).queryParams(vars); + Builder result = target.request(); + return result; + } + + public long startjob(String template, String name, BasicDBObject jobParams) throws Exception { + MultivaluedMap vars = new MultivaluedMapImpl<>(); + vars.add("template", template); + vars.add("name", name); + Response response = query("services/rest/jobengine/startjob", vars).post(Entity.entity(jobParams.toString(), MediaType.APPLICATION_JSON)); + + if (response.getStatus() != Status.OK.getStatusCode()) { + logger.error(response.readEntity(String.class)); + return 0; + } + + String resultObject = response.readEntity(String.class); + return Long.parseLong(resultObject); + } + +} diff --git a/server/-product/production/OMAR/jobs/steps/shared/MediaFileSearchFilterOptions.java b/server/-product/production/OMAR/jobs/steps/shared/MediaFileSearchFilterOptions.java new file mode 100644 index 00000000..ee467e5b --- /dev/null +++ b/server/-product/production/OMAR/jobs/steps/shared/MediaFileSearchFilterOptions.java @@ -0,0 +1,42 @@ +package user.jobengine.server.steps.shared; + +import java.nio.file.Path; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +import com.ibm.nosql.json.api.BasicDBObject; + +public class MediaFileSearchFilterOptions extends FileSearchFilterOptions { + + private BasicDBObject filter; + + public MediaFileSearchFilterOptions(BasicDBObject filter) { + super(filter); + } + + @Override + public boolean acceptFile(Path file) { + if (filter == null) + return true; + + if (filter.containsKey("fileName")) { + // .*\.(sh|ini|conf|vhost|xml|php)$ + String fileNamePattern = filter.getString("fileName"); + if (fileNamePattern == null || fileNamePattern.trim().length() == 0) + return true; + + Pattern pattern = Pattern.compile(fileNamePattern, Pattern.CASE_INSENSITIVE); + + Matcher matcher = pattern.matcher(file.getFileName().toString()); + if (matcher.find()) + return true; + } + + return false; + } + + @Override + public boolean preAcceptDirectory(Path file) { + return true; + } +} diff --git a/server/-product/production/OMAR/jobs/steps/shared/MetadataType.java b/server/-product/production/OMAR/jobs/steps/shared/MetadataType.java new file mode 100644 index 00000000..0e27bf34 --- /dev/null +++ b/server/-product/production/OMAR/jobs/steps/shared/MetadataType.java @@ -0,0 +1,5 @@ +package user.jobengine.server.steps.shared; + +public enum MetadataType { + TrafficMaterial, TrafficPromo, TrafficAD, OctopusStory, OctopusPlaceholder, Generic +} diff --git a/server/-product/production/OMAR/jobs/steps/shared/MetadataTypeDetector.java b/server/-product/production/OMAR/jobs/steps/shared/MetadataTypeDetector.java new file mode 100644 index 00000000..9e9e6be9 --- /dev/null +++ b/server/-product/production/OMAR/jobs/steps/shared/MetadataTypeDetector.java @@ -0,0 +1,49 @@ +package user.jobengine.server.steps.shared; + +import org.apache.commons.lang.StringUtils; + +public class MetadataTypeDetector { + + private static final String HYPHEN = "-"; + private static final String DOT = "."; + + private static final String REGEXP_TRAFFICMATERIALID = "^M{1}[0-9]{6}[A-Z]{1}"; + private static final String REGEXP_TRAFFICADID = "^R{1}[0-9]{6}[A-Z]{1}"; + private static final String REGEXP_TRAFFICPROMOID = "^P{1}[0-9]{6}[A-Z]{1}"; + private static final String REGEXP_OCTOPUSSTORYID = "^[0-9]+"; + private static final String REGEXP_OCTOPUSPLACEHOLDERID = "^[0-9]+_[0-9]+"; + private static final String REGEXP_OCTOPUSPLACEHOLDERVERSIONEDID = "^[0-9]+_[0-9]+-[0-9]{3}"; + + public static MetadataType GuessMetadataType(String id) { + if (StringUtils.isBlank(id)) + return MetadataType.Generic; + if (id.matches(REGEXP_TRAFFICMATERIALID)) + return MetadataType.TrafficMaterial; + if (id.matches(REGEXP_TRAFFICPROMOID)) + return MetadataType.TrafficPromo; + if (id.matches(REGEXP_TRAFFICADID)) + return MetadataType.TrafficAD; + if (id.matches(REGEXP_OCTOPUSSTORYID)) + return MetadataType.OctopusStory; + if (id.matches(REGEXP_OCTOPUSPLACEHOLDERID)) + return MetadataType.OctopusPlaceholder; + if (id.matches(REGEXP_OCTOPUSPLACEHOLDERVERSIONEDID)) + return MetadataType.OctopusPlaceholder; + return MetadataType.Generic; + } + + public static String truncateExtension(String name) { + String result = name; + if (result != null && result.contains(DOT)) + result = result.substring(0, result.lastIndexOf(DOT)); + return result; + } + + public static String truncateVersion(String name) { + String result = name; + if (result != null && result.contains(HYPHEN)) + result = result.split(HYPHEN)[0]; + return result; + } + +} diff --git a/server/-product/production/OMAR/jobs/steps/shared/TestLib.java b/server/-product/production/OMAR/jobs/steps/shared/TestLib.java new file mode 100644 index 00000000..d12738ae --- /dev/null +++ b/server/-product/production/OMAR/jobs/steps/shared/TestLib.java @@ -0,0 +1,9 @@ +package user.jobengine.server.steps.shared; + +public class TestLib { + + public void hello() { + TestLib1 lib = new TestLib1(); + lib.helo(); + } +} diff --git a/server/-product/production/OMAR/jobs/steps/shared/TestLib1.java b/server/-product/production/OMAR/jobs/steps/shared/TestLib1.java new file mode 100644 index 00000000..99d47e5f --- /dev/null +++ b/server/-product/production/OMAR/jobs/steps/shared/TestLib1.java @@ -0,0 +1,8 @@ +package user.jobengine.server.steps.shared; + +public class TestLib1 { + + public void hello() { + System.out.println("Hello from lib1"); + } +} diff --git a/server/-product/production/OMAR/jobs/templates/archive-limited.xml b/server/-product/production/OMAR/jobs/templates/archive-limited.xml new file mode 100644 index 00000000..6ea81306 --- /dev/null +++ b/server/-product/production/OMAR/jobs/templates/archive-limited.xml @@ -0,0 +1,43 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/server/-product/production/OMAR/jobs/templates/archive-material.xml b/server/-product/production/OMAR/jobs/templates/archive-material.xml new file mode 100644 index 00000000..973ac9fa --- /dev/null +++ b/server/-product/production/OMAR/jobs/templates/archive-material.xml @@ -0,0 +1,49 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/server/-product/production/OMAR/jobs/templates/archive-ondemand.xml b/server/-product/production/OMAR/jobs/templates/archive-ondemand.xml new file mode 100644 index 00000000..54555f6c --- /dev/null +++ b/server/-product/production/OMAR/jobs/templates/archive-ondemand.xml @@ -0,0 +1,43 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/server/-product/production/OMAR/jobs/templates/archive-recursive.xml b/server/-product/production/OMAR/jobs/templates/archive-recursive.xml new file mode 100644 index 00000000..02b97884 --- /dev/null +++ b/server/-product/production/OMAR/jobs/templates/archive-recursive.xml @@ -0,0 +1,25 @@ + + + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/server/-product/production/OMAR/jobs/templates/batch-retrieve-ondemand.xml b/server/-product/production/OMAR/jobs/templates/batch-retrieve-ondemand.xml new file mode 100644 index 00000000..20332cf0 --- /dev/null +++ b/server/-product/production/OMAR/jobs/templates/batch-retrieve-ondemand.xml @@ -0,0 +1,29 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/server/-product/production/OMAR/jobs/templates/calculatemd5.xml b/server/-product/production/OMAR/jobs/templates/calculatemd5.xml new file mode 100644 index 00000000..637c7638 --- /dev/null +++ b/server/-product/production/OMAR/jobs/templates/calculatemd5.xml @@ -0,0 +1,19 @@ + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/server/-product/production/OMAR/jobs/templates/cancelable.xml b/server/-product/production/OMAR/jobs/templates/cancelable.xml new file mode 100644 index 00000000..2be289b3 --- /dev/null +++ b/server/-product/production/OMAR/jobs/templates/cancelable.xml @@ -0,0 +1,17 @@ + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/server/-product/production/OMAR/jobs/templates/common-copy.xml b/server/-product/production/OMAR/jobs/templates/common-copy.xml new file mode 100644 index 00000000..7abb6fcf --- /dev/null +++ b/server/-product/production/OMAR/jobs/templates/common-copy.xml @@ -0,0 +1,41 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/server/-product/production/OMAR/jobs/templates/create-lowres-ondemand.xml b/server/-product/production/OMAR/jobs/templates/create-lowres-ondemand.xml new file mode 100644 index 00000000..f7922734 --- /dev/null +++ b/server/-product/production/OMAR/jobs/templates/create-lowres-ondemand.xml @@ -0,0 +1,106 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/server/-product/production/OMAR/jobs/templates/create-proxy-ffmpeg.xml b/server/-product/production/OMAR/jobs/templates/create-proxy-ffmpeg.xml new file mode 100644 index 00000000..acb2cb62 --- /dev/null +++ b/server/-product/production/OMAR/jobs/templates/create-proxy-ffmpeg.xml @@ -0,0 +1,25 @@ + + + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/server/-product/production/OMAR/jobs/templates/delete-materials.xml b/server/-product/production/OMAR/jobs/templates/delete-materials.xml new file mode 100644 index 00000000..365fe19c --- /dev/null +++ b/server/-product/production/OMAR/jobs/templates/delete-materials.xml @@ -0,0 +1,17 @@ + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/server/-product/production/OMAR/jobs/templates/dummy-test-job-copy.xml b/server/-product/production/OMAR/jobs/templates/dummy-test-job-copy.xml new file mode 100644 index 00000000..2c5d5a5d --- /dev/null +++ b/server/-product/production/OMAR/jobs/templates/dummy-test-job-copy.xml @@ -0,0 +1,47 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/server/-product/production/OMAR/jobs/templates/dummy-test-job.xml b/server/-product/production/OMAR/jobs/templates/dummy-test-job.xml new file mode 100644 index 00000000..2c5d5a5d --- /dev/null +++ b/server/-product/production/OMAR/jobs/templates/dummy-test-job.xml @@ -0,0 +1,47 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/server/-product/production/OMAR/jobs/templates/integration-test.xml b/server/-product/production/OMAR/jobs/templates/integration-test.xml new file mode 100644 index 00000000..6593e5d7 --- /dev/null +++ b/server/-product/production/OMAR/jobs/templates/integration-test.xml @@ -0,0 +1,6 @@ + + + + + + \ No newline at end of file diff --git a/server/-product/production/OMAR/jobs/templates/metadata-updater.xml b/server/-product/production/OMAR/jobs/templates/metadata-updater.xml new file mode 100644 index 00000000..f516c3e2 --- /dev/null +++ b/server/-product/production/OMAR/jobs/templates/metadata-updater.xml @@ -0,0 +1,6 @@ + + + + + + \ No newline at end of file diff --git a/server/-product/production/OMAR/jobs/templates/migrate-hsm.xml b/server/-product/production/OMAR/jobs/templates/migrate-hsm.xml new file mode 100644 index 00000000..faec8724 --- /dev/null +++ b/server/-product/production/OMAR/jobs/templates/migrate-hsm.xml @@ -0,0 +1,21 @@ + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/server/-product/production/OMAR/jobs/templates/move-jpeg-to-isilon.xml b/server/-product/production/OMAR/jobs/templates/move-jpeg-to-isilon.xml new file mode 100644 index 00000000..ae74dab8 --- /dev/null +++ b/server/-product/production/OMAR/jobs/templates/move-jpeg-to-isilon.xml @@ -0,0 +1,51 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/server/-product/production/OMAR/jobs/templates/prores-archive.xml b/server/-product/production/OMAR/jobs/templates/prores-archive.xml new file mode 100644 index 00000000..2d49bcba --- /dev/null +++ b/server/-product/production/OMAR/jobs/templates/prores-archive.xml @@ -0,0 +1,76 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/server/-product/production/OMAR/jobs/templates/remote-transcode.xml b/server/-product/production/OMAR/jobs/templates/remote-transcode.xml new file mode 100644 index 00000000..ce4c77b7 --- /dev/null +++ b/server/-product/production/OMAR/jobs/templates/remote-transcode.xml @@ -0,0 +1,111 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/server/-product/production/OMAR/jobs/templates/retrieve-ondemand.xml b/server/-product/production/OMAR/jobs/templates/retrieve-ondemand.xml new file mode 100644 index 00000000..b58d8fdf --- /dev/null +++ b/server/-product/production/OMAR/jobs/templates/retrieve-ondemand.xml @@ -0,0 +1,150 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/server/-product/production/OMAR/jobs/templates/retrieve.xml b/server/-product/production/OMAR/jobs/templates/retrieve.xml new file mode 100644 index 00000000..e12fd950 --- /dev/null +++ b/server/-product/production/OMAR/jobs/templates/retrieve.xml @@ -0,0 +1,21 @@ + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/server/-product/production/OMAR/jobs/templates/safe-delete-recursive.xml b/server/-product/production/OMAR/jobs/templates/safe-delete-recursive.xml new file mode 100644 index 00000000..bfa8e371 --- /dev/null +++ b/server/-product/production/OMAR/jobs/templates/safe-delete-recursive.xml @@ -0,0 +1,17 @@ + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/server/-product/production/OMAR/jobs/templates/sync-subtitles.xml b/server/-product/production/OMAR/jobs/templates/sync-subtitles.xml new file mode 100644 index 00000000..f68c2781 --- /dev/null +++ b/server/-product/production/OMAR/jobs/templates/sync-subtitles.xml @@ -0,0 +1,23 @@ + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/server/-product/production/OMAR/jobs/templates/sys-recreate-lowres.xml b/server/-product/production/OMAR/jobs/templates/sys-recreate-lowres.xml new file mode 100644 index 00000000..9ef84b31 --- /dev/null +++ b/server/-product/production/OMAR/jobs/templates/sys-recreate-lowres.xml @@ -0,0 +1,118 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/server/-product/production/OMAR/jobs/templates/test-fork-cancelable.xml b/server/-product/production/OMAR/jobs/templates/test-fork-cancelable.xml new file mode 100644 index 00000000..8c88d84b --- /dev/null +++ b/server/-product/production/OMAR/jobs/templates/test-fork-cancelable.xml @@ -0,0 +1,6 @@ + + + + + + \ No newline at end of file diff --git a/server/-product/production/OMAR/log/.gitignore b/server/-product/production/OMAR/log/.gitignore new file mode 100644 index 00000000..c474102b --- /dev/null +++ b/server/-product/production/OMAR/log/.gitignore @@ -0,0 +1,5 @@ +/delete-materials.log +/markered-mediacube.log +/mediacube-err.log +/mediacube.log +/movtest.log diff --git a/server/-product/production/OMAR/log/2022-05/.gitignore b/server/-product/production/OMAR/log/2022-05/.gitignore new file mode 100644 index 00000000..5e8754b0 --- /dev/null +++ b/server/-product/production/OMAR/log/2022-05/.gitignore @@ -0,0 +1,6 @@ +/markered-mediacube-05-05-2022-1.log.gz +/markered-mediacube-05-06-2022-1.log.gz +/mediacube-05-05-2022-1.log.gz +/mediacube-05-06-2022-1.log.gz +/mediacube-err-05-05-2022-1.log.gz +/mediacube-err-05-06-2022-1.log.gz diff --git a/server/-product/production/OMAR/log/2022-05/markered-mediacube-05-09-2022-1.log.gz b/server/-product/production/OMAR/log/2022-05/markered-mediacube-05-09-2022-1.log.gz new file mode 100644 index 00000000..53250356 Binary files /dev/null and b/server/-product/production/OMAR/log/2022-05/markered-mediacube-05-09-2022-1.log.gz differ diff --git a/server/-product/production/OMAR/log/2022-05/mediacube-05-09-2022-1.log.gz b/server/-product/production/OMAR/log/2022-05/mediacube-05-09-2022-1.log.gz new file mode 100644 index 00000000..b4179fcc Binary files /dev/null and b/server/-product/production/OMAR/log/2022-05/mediacube-05-09-2022-1.log.gz differ diff --git a/server/-product/production/OMAR/log/2022-05/mediacube-err-05-09-2022-1.log.gz b/server/-product/production/OMAR/log/2022-05/mediacube-err-05-09-2022-1.log.gz new file mode 100644 index 00000000..605ced81 Binary files /dev/null and b/server/-product/production/OMAR/log/2022-05/mediacube-err-05-09-2022-1.log.gz differ diff --git a/server/-product/production/OMAR/mediacube.bat b/server/-product/production/OMAR/mediacube.bat new file mode 100644 index 00000000..24a39d6c --- /dev/null +++ b/server/-product/production/OMAR/mediacube.bat @@ -0,0 +1,17 @@ +@echo off +if exist tmp rmdir tmp /s /q +mkdir tmp +java ^ +-Dorg.eclipse.epp.logging.aeri.skipReports=true ^ +-Declipse.ignoreApp=true ^ +-Dosgi.noShutdown=true ^ +-Dlog4j.configurationFile=settings/log4j2.xml ^ +-Djetty.home=settings ^ +-Djetty.etc.config.urls=jetty.xml ^ +-Djava.io.tmpdir=tmp ^ +-Dfile.encoding=UTF-8 ^ +-Dgosh.home=configuration ^ +-jar ../../target/products/MediaCube/linux/gtk/x86_64/plugins/org.eclipse.equinox.launcher_1.3.201.v20161025-1711.jar ^ +-Xms512m ^ +-Xmx1024m ^ +-console diff --git a/server/-product/production/OMAR/settings/application.yaml b/server/-product/production/OMAR/settings/application.yaml new file mode 100644 index 00000000..c870af39 --- /dev/null +++ b/server/-product/production/OMAR/settings/application.yaml @@ -0,0 +1,45 @@ +datasource: + mediacube: + url: jdbc:db2://localhost:50000/mc + user: db2admin + password: password + external-indexer: true + simple-search: true + login-timeout: 3 + pool-size: 10 + mediacube-nosql: + url: jdbc:db2://localhost:50000/mc + user: db2admin + password: password + schema: test + login-timeout: 3 + hsm: + url: jdbc:db2://10.11.1.89:51500/tsmdb1 + user: tsminst1 + password: tsminst1 + planair: + url: jdbc:sqlserver://10.11.254.86;databaseName=PA_Vivantis; + user: MAM + password: VDani +services: + ffmpeg: + executable-location: /opt/ffmpeg/ffmpeg + mediacube: + proxy-root: /data/mediacube + nexio: +# host: 10.10.1.55 +# collection-name: nexioclips +# use-mos-gateway: true + disabled: true +jobs: + validate-transfers: false + copy-buffer-size: 32768 + scheduled-execution-disabled: true +tsm: + randomize-archives: false + delimiter: / + node-name: JOBENGINE + fs-name: /JOBENGINE + alternate-fs-name: /JOBENGINE + hl-name: /JOBENGINE + \ No newline at end of file diff --git a/server/-product/production/OMAR/settings/dsm.opt b/server/-product/production/OMAR/settings/dsm.opt new file mode 100644 index 00000000..90fcbdd2 --- /dev/null +++ b/server/-product/production/OMAR/settings/dsm.opt @@ -0,0 +1 @@ +SErvername tsm.in.useribm.hu diff --git a/server/-product/production/OMAR/settings/dsmopt.lock b/server/-product/production/OMAR/settings/dsmopt.lock new file mode 100644 index 00000000..e69de29b diff --git a/server/-product/production/OMAR/settings/external-commands.yaml b/server/-product/production/OMAR/settings/external-commands.yaml new file mode 100644 index 00000000..5f3a3a63 --- /dev/null +++ b/server/-product/production/OMAR/settings/external-commands.yaml @@ -0,0 +1,27 @@ +profiles: +- + name: proxy + executable: /Programs/ffmpeg/bin/ffmpeg.exe + arguments: + - -y + - -v + - error + - -progress + - pipe:1 + - -i + - "%i" + - "%o" +- + name: length + executable: /Programs/ffmpeg/bin/ffprobe.exe + arguments: + - -v + - error + - -select_streams + - v:0 + - -show_entries + - stream=nb_frames + - -of + - default=noprint_wrappers=1:nokey=1 + - "%i" + \ No newline at end of file diff --git a/server/-product/production/OMAR/settings/jetty.xml b/server/-product/production/OMAR/settings/jetty.xml new file mode 100644 index 00000000..30bdeb08 --- /dev/null +++ b/server/-product/production/OMAR/settings/jetty.xml @@ -0,0 +1,45 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/server/-product/production/OMAR/settings/log4j2.xml b/server/-product/production/OMAR/settings/log4j2.xml new file mode 100644 index 00000000..5f514527 --- /dev/null +++ b/server/-product/production/OMAR/settings/log4j2.xml @@ -0,0 +1,105 @@ + + + + + + log + ${logPath}/mediacube.log + ${logPath}/movtest.log + ${logPath}/$${date:yyyy-MM}/movtest-%d{MM-dd-yyyy}-%i.log.gz + ${logPath}/delete-materials.log + ${logPath}/$${date:yyyy-MM}/delete-materials-%d{MM-dd-yyyy}-%i.log.gz + ${logPath}/$${date:yyyy-MM}/mediacube-%d{MM-dd-yyyy}-%i.log.gz + ${logPath}/markered-mediacube.log + ${logPath}/$${date:yyyy-MM}/markered-mediacube-%d{MM-dd-yyyy}-%i.log.gz + ${logPath}/mediacube-err.log + ${logPath}/$${date:yyyy-MM}/mediacube-err-%d{MM-dd-yyyy}-%i.log.gz + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/server/-product/production/OMAR/settings/maestro.yaml b/server/-product/production/OMAR/settings/maestro.yaml new file mode 100644 index 00000000..b8091a2f --- /dev/null +++ b/server/-product/production/OMAR/settings/maestro.yaml @@ -0,0 +1,38 @@ +sourceStoreUri: + name: Default + protocol: LOCAL + uri: "/mnt/NLE" + fileFilter: "*.mxf" + showDirectories: true +alternateSourceStoreUris: +- name: NLE1 + protocol: LOCAL + uri: "/mnt/NLE/NLE1" + fileFilter: "*.mxf" + showDirectories: true +- name: NLE2 + protocol: LOCAL + uri: "/mnt/NLE/NLE2" + fileFilter: "*.mxf" + showDirectories: true +- name: NLE3 + protocol: LOCAL + uri: "/mnt/NLE/NLE3" + fileFilter: "*.mxf" + showDirectories: true +- name: NLE4 + protocol: LOCAL + uri: "/mnt/NLE/NLE4" + fileFilter: "*.mxf" + showDirectories: true +- name: POLC + protocol: LOCAL + uri: "/mnt/POLC" + fileFilter: "*.mxf" + showDirectories: true +targets: +- name: FINISHED_SHOWS + killDateDays: 7 + storeUri: + protocol: LOCAL + uri: "/mnt/PROMISE/FINISHED_SHOWS" diff --git a/server/-product/production/OMAR/settings/mediacube.yaml b/server/-product/production/OMAR/settings/mediacube.yaml new file mode 100644 index 00000000..347ca896 --- /dev/null +++ b/server/-product/production/OMAR/settings/mediacube.yaml @@ -0,0 +1,53 @@ +jobQueuePollInterval: 1000 +disableHelp: true +maestroDisabled: false +alternateRetrieveSelector: false +disableStatistics: true +disableEditor: false +topTypeFilters: +- name: Hír bejátszó +- name: Hír nyers +- name: Visszarögzített +- name: Egyéb +bottomTypeFilters: +- name: Műsor +- name: Műsor nyers +- name: Promo +- name: Promo nyers +- name: Reklám +- name: Reklám nyers +authentication: + authEnabled: true + defaultUser: root + defaultPassword: password + adHost: intra.mediavivantis.hu + adNonSecurePort: 3268 + adBaseDn: DC=intra,DC=mediavivantis,DC=hu + adAdminMap: + - G_MV_U_MUSZAK + - G_MV_U_INGEST + adSubmitterMap: + - G_ECH_U_INFORMATIKUSOK + - G_ECH_U_MUSZAKVEZETOK + - ECH-ISILON-ADMINS + adEditorMap: + - G_ECH_U_INFORMATIKUSOK + - G_ECH_U_MUSZAKVEZETOK + - ECH-ISILON-ADMINS + localAccounts: + - user: user + password: 5F4DCC3B5AA765D61D8327DEB882CF99 + email: + - user: lebony + password: 4E25B117B14D86D7DCECB4E433CF932C + email: + - user: root + password: 5F4DCC3B5AA765D61D8327DEB882CF99 + email: vasary@elgekko.net + localAdmins: + - root + - admin + localSubmitters: + - lebony + localEditors: + - editor