From 9d949c8ddea428c24eb2bdcc1afbb7da00153b92 Mon Sep 17 00:00:00 2001 From: "vasary.daniel" Date: Thu, 29 Jul 2021 18:59:14 +0000 Subject: [PATCH] git-tfs-id: [http://tfs.userrendszerhaz.hu:8080/tfs/DefaultCollection]$/MediaCube;C32286 --- ...ive-checker.xml => PB-archive-checker.xml} | 0 .../MEDIAVIVANTIS/jobs/steps/DeleteFile.java | 22 ++ .../jobs/steps/IntegrationTestStep.java | 128 +++++++ .../steps/PrepareRemoteTranscodeStep.java | 48 +++ .../jobs/steps/SaveMediaProxy.java | 22 ++ .../jobs/steps/TransferStep.java | 110 ++++++ .../jobs/steps/shared/EscortFiles.java | 348 ++++++++++++++++++ .../jobs/steps/shared/ExternalCommand.java | 79 ++++ .../steps/shared/ExternalCommandExecutor.java | 32 ++ .../jobs/steps/shared/ExternalProfile.java | 33 ++ .../steps/shared/ExternalProfilesConfig.java | 15 + .../jobs/steps/shared/IExternalCallback.java | 5 + .../steps/shared/ItemManagerExtensions.java | 94 +++++ .../jobs/steps/shared/MediaCubeClient.java | 77 ++++ .../jobs/steps/shared/MetadataType.java | 5 + .../steps/shared/MetadataTypeDetector.java | 49 +++ .../server/steps/EscortFileUpdaterStep.java | 57 +++ .../steps/PBARCHIVETransferFromStep.java | 11 +- .../server/steps/PBArchiveCheckerStep.java | 66 ++-- 19 files changed, 1154 insertions(+), 47 deletions(-) rename server/-product/production/AMC/jobs/templates/{PBARCHIVE-archive-checker.xml => PB-archive-checker.xml} (100%) create mode 100644 server/-product/production/MEDIAVIVANTIS/jobs/steps/DeleteFile.java create mode 100644 server/-product/production/MEDIAVIVANTIS/jobs/steps/IntegrationTestStep.java create mode 100644 server/-product/production/MEDIAVIVANTIS/jobs/steps/PrepareRemoteTranscodeStep.java create mode 100644 server/-product/production/MEDIAVIVANTIS/jobs/steps/SaveMediaProxy.java create mode 100644 server/-product/production/MEDIAVIVANTIS/jobs/steps/TransferStep.java create mode 100644 server/-product/production/MEDIAVIVANTIS/jobs/steps/shared/EscortFiles.java create mode 100644 server/-product/production/MEDIAVIVANTIS/jobs/steps/shared/ExternalCommand.java create mode 100644 server/-product/production/MEDIAVIVANTIS/jobs/steps/shared/ExternalCommandExecutor.java create mode 100644 server/-product/production/MEDIAVIVANTIS/jobs/steps/shared/ExternalProfile.java create mode 100644 server/-product/production/MEDIAVIVANTIS/jobs/steps/shared/ExternalProfilesConfig.java create mode 100644 server/-product/production/MEDIAVIVANTIS/jobs/steps/shared/IExternalCallback.java create mode 100644 server/-product/production/MEDIAVIVANTIS/jobs/steps/shared/ItemManagerExtensions.java create mode 100644 server/-product/production/MEDIAVIVANTIS/jobs/steps/shared/MediaCubeClient.java create mode 100644 server/-product/production/MEDIAVIVANTIS/jobs/steps/shared/MetadataType.java create mode 100644 server/-product/production/MEDIAVIVANTIS/jobs/steps/shared/MetadataTypeDetector.java create mode 100644 server/user.jobengine.executors/src/user/jobengine/server/steps/EscortFileUpdaterStep.java diff --git a/server/-product/production/AMC/jobs/templates/PBARCHIVE-archive-checker.xml b/server/-product/production/AMC/jobs/templates/PB-archive-checker.xml similarity index 100% rename from server/-product/production/AMC/jobs/templates/PBARCHIVE-archive-checker.xml rename to server/-product/production/AMC/jobs/templates/PB-archive-checker.xml diff --git a/server/-product/production/MEDIAVIVANTIS/jobs/steps/DeleteFile.java b/server/-product/production/MEDIAVIVANTIS/jobs/steps/DeleteFile.java new file mode 100644 index 00000000..539faf74 --- /dev/null +++ b/server/-product/production/MEDIAVIVANTIS/jobs/steps/DeleteFile.java @@ -0,0 +1,22 @@ +package user.jobengine.server.steps; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + +import user.commons.RemoteFile; +import user.commons.StoreUri; + +public class DeleteFile extends JobStep { + private static final Logger logger = LogManager.getLogger(); + + @StepEntry + public Object[] execute(String fileName, StoreUri tempStoreUri) throws Exception { + try { + RemoteFile remoteFile = tempStoreUri.getRemoteFile(fileName); + tempStoreUri.delete(remoteFile); + } catch (Exception e) { + logger.warn(getJobRuntime().getSessionMarker(), e.getMessage()); + } + return null; + } +} diff --git a/server/-product/production/MEDIAVIVANTIS/jobs/steps/IntegrationTestStep.java b/server/-product/production/MEDIAVIVANTIS/jobs/steps/IntegrationTestStep.java new file mode 100644 index 00000000..317edfed --- /dev/null +++ b/server/-product/production/MEDIAVIVANTIS/jobs/steps/IntegrationTestStep.java @@ -0,0 +1,128 @@ +package user.jobengine.server.steps; + +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.concurrent.CountDownLatch; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + +import user.commons.ListUtils; +import user.jobengine.db.ArchivedMedia; +import user.jobengine.db.Item; +import user.jobengine.db.Media; +import user.jobengine.server.steps.shared.ItemManagerExtensions; + +public class IntegrationTestStep extends JobStep { + private static final Logger logger = LogManager.getLogger(); + private ArchivedMedia archivedMedia; + + private void _00_test_cancelable() throws Exception { + CountDownLatch finishLatch = new CountDownLatch(1); + getEngine().submit(null, e -> { + if (e.isRuntimeTerminated()) + finishLatch.countDown(); + }, "cancelable.xml", "Test cancelable", ListUtils.asMap("param", 1)); + finishLatch.await(); + logger.info("_00_test_cancelable SUCCESS"); + } + + private void _01_test_retrieve_ondemand() throws Exception { + archivedMedia = new ArchivedMedia(); + Media media = getManager().getMedia(15285); + Item item = getManager().getItem(media.getItemId()); + archivedMedia.setItem(item); + archivedMedia.setMedia(media); + + CountDownLatch finishLatch = new CountDownLatch(1); + getEngine().submit(null, e -> { + if (e.isRuntimeTerminated()) + finishLatch.countDown(); + + }, "retrieve-ondemand.xml", "Test retrieve-ondemand", + ListUtils.asMap("globalRetrievePath", "file://10.11.1.100", "localRetrievePath", "/mediacube/data", "materialOutputFolder", "/", + "promoOutputFolder", "/", "advertisementOutputFolder", "/", "octopusOutputFolder", "/", "genericOutputFolder", "/", + "onlineOutputFolder", "/", "killDateDays", -1, "nexioAgency", "ARCHIVE_RESTORE", "nexioPort", 2098, "nexioUserName", "administrator", + "nexioPassword", "system", "archivedMedia", archivedMedia, "successRecipient", "vasary@elgekko.net", "houseId", + archivedMedia.getMedia().getHouseId(), "targetPathType", "0")); + + finishLatch.await(); + Path output = Paths.get("/mediacube/data", archivedMedia.getMedia().getHouseId(), + archivedMedia.getMedia().getHouseId() + "-ARCH-" + archivedMedia.getMedia().getMediaFileRealName()); + if (!output.toFile().exists()) + throw new Exception("File not exists: " + output); + + logger.info("_01_test_retrieve_ondemand SUCCESS"); + } + + private void _02_test_archive_material() throws Exception { + Path input = Paths.get("/mediacube/data", archivedMedia.getMedia().getHouseId(), + archivedMedia.getMedia().getHouseId() + "-ARCH-" + archivedMedia.getMedia().getMediaFileRealName()); + + String outputName = "IntegrationTest_" + System.currentTimeMillis() + ".mxf"; + Path ouptput = Paths.get(input.getParent().toString(), outputName); + Files.copy(input, ouptput); + if (!ouptput.toFile().exists()) + throw new Exception("File not exists: " + ouptput); + + ArchiveItem archiveItem = new ArchiveItem(); + archiveItem.setMediaFile(ouptput.toString()); + archiveItem.setItemHouseId(outputName); + archiveItem.setItemTitle(outputName); + archiveItem.setMediaHouseId(outputName); + archiveItem.setMediaTitle(outputName); + archiveItem.setMediaType("Generic"); + + CountDownLatch finishLatch = new CountDownLatch(1); + getEngine().submit(null, e -> { + if (e.isRuntimeTerminated()) + finishLatch.countDown(); + }, "archive-material.xml", "Test archive-material", ListUtils.asMap("archiveItem", archiveItem, "killDateDays", 0)); + finishLatch.await(); + + if (!ItemManagerExtensions.isArchived(getManager(), ouptput)) + throw new Exception("File not archived: " + ouptput); + + Files.delete(ouptput); + + logger.info("_02_test_archive_material SUCCESS"); + } + + private void _03_test_delete_materials() throws Exception { + Path output = Paths.get("/mediacube/data", archivedMedia.getMedia().getHouseId(), + archivedMedia.getMedia().getHouseId() + "-ARCH-" + archivedMedia.getMedia().getMediaFileRealName()); + String source = output.getParent().toString(); + + CountDownLatch finishLatch = new CountDownLatch(1); + getEngine().submit(null, e -> { + if (e.isRuntimeTerminated()) + finishLatch.countDown(); + }, "delete-materials.xml", "Test delete-materials", ListUtils.asMap("sourcePath", source, "skipArchiveCheck", true)); + finishLatch.await(); + + if (output.toFile().exists()) + throw new Exception("File exists: " + output); + } + + @StepEntry + public Object[] execute() throws Exception { + getJobRuntime().setDescription("_00_test_cancelable"); + _00_test_cancelable(); + setProgress(25); + + getJobRuntime().setDescription("_01_test_retrieve_ondemand"); + _01_test_retrieve_ondemand(); + setProgress(50); + + getJobRuntime().setDescription("_02_test_archive_material"); + _02_test_archive_material(); + setProgress(75); + + getJobRuntime().setDescription("_03_test_delete_materials"); + _03_test_delete_materials(); + setProgress(100); + return null; + } + +} diff --git a/server/-product/production/MEDIAVIVANTIS/jobs/steps/PrepareRemoteTranscodeStep.java b/server/-product/production/MEDIAVIVANTIS/jobs/steps/PrepareRemoteTranscodeStep.java new file mode 100644 index 00000000..5116b1ef --- /dev/null +++ b/server/-product/production/MEDIAVIVANTIS/jobs/steps/PrepareRemoteTranscodeStep.java @@ -0,0 +1,48 @@ +package user.jobengine.server.steps; + +import java.nio.file.Path; +import java.nio.file.Paths; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + +import com.ibm.nosql.json.api.BasicDBObject; + +import user.commons.configuration.SystemConfiguration; + +public class PrepareRemoteTranscodeStep extends JobStep { + static private final Logger logger = LogManager.getLogger(); + private static boolean RANDOMIZE_ARCHIVES = SystemConfiguration.getInstance().value("tsm.randomize-archives", + false); + + @StepEntry + public Object[] execute(String profileName, String fileName) throws Exception { + String hiResRoot = "m:/"; + String lowResRoot = "m:/lowres"; + + Path inputPath = Paths.get(hiResRoot, fileName); + + String realFileName = fileName; + + if (RANDOMIZE_ARCHIVES) + realFileName = realFileName.substring(9); + + String outFileName = realFileName.substring(0, realFileName.lastIndexOf(".")) + ".mp4"; + + Path relativeOutputPath = null; + if (realFileName.length() > 2) + relativeOutputPath = Paths.get(realFileName.substring(0, 1), realFileName.substring(1, 2), + realFileName.substring(2, 3), outFileName); + else + relativeOutputPath = Paths.get("0", outFileName); + + BasicDBObject parameters = new BasicDBObject(); + parameters.put("profile", profileName); + parameters.put("input", inputPath.toString()); + parameters.put("output", Paths.get(lowResRoot, relativeOutputPath.toString()).toString()); + + logger.info("Prepared for remote transode {}", parameters); + return new Object[] { parameters, relativeOutputPath.toString() }; + } + +} diff --git a/server/-product/production/MEDIAVIVANTIS/jobs/steps/SaveMediaProxy.java b/server/-product/production/MEDIAVIVANTIS/jobs/steps/SaveMediaProxy.java new file mode 100644 index 00000000..3f0f667d --- /dev/null +++ b/server/-product/production/MEDIAVIVANTIS/jobs/steps/SaveMediaProxy.java @@ -0,0 +1,22 @@ +package user.jobengine.server.steps; + +import user.jobengine.db.IItemManager; +import user.jobengine.db.Media; +import user.jobengine.db.MediaFile; +import user.jobengine.db.Store; + +public class SaveMediaProxy extends JobStep { + + @StepEntry + public Object[] execute(Media media, String proxyRelativePath) throws Exception { + IItemManager manager = getManager(); + Store store = manager.getCurrentLowresStore(); + MediaFile mediaFile = new MediaFile(); + mediaFile.setMedia(media); + mediaFile.setStore(store); + mediaFile.setFileType(manager.getFileType("Low-res")); + mediaFile.setRelativePath(proxyRelativePath); + manager.add(mediaFile); + return null; + } +} diff --git a/server/-product/production/MEDIAVIVANTIS/jobs/steps/TransferStep.java b/server/-product/production/MEDIAVIVANTIS/jobs/steps/TransferStep.java new file mode 100644 index 00000000..c10084fa --- /dev/null +++ b/server/-product/production/MEDIAVIVANTIS/jobs/steps/TransferStep.java @@ -0,0 +1,110 @@ +package user.jobengine.server.steps; + +import java.nio.file.Path; +import java.nio.file.Paths; + +import org.apache.commons.net.ftp.FTPClient; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + +import user.commons.RemoteFile; +import user.commons.StoreUri; +import user.commons.remotestore.FtpDirectoryLister; +import user.commons.remotestore.RemoteStoreProtocol; +import user.jobengine.db.Store; + +public class TransferStep extends JobStep { + private static final String DOT_PART = ".part"; + private static final Logger logger = LogManager.getLogger(); + + private void copy(StoreUri sourceStoreUri, String sourceFileName, StoreUri targetStoreUri, String targetFileName) throws Exception { + String currentTargetFileName = targetFileName; + + boolean renameAfterCopy = false; + boolean renameAfterFTP = false; + if (getTmpExtension() != null) { + if (RemoteStoreProtocol.LOCAL.equals(targetStoreUri.getProtocol())) { + currentTargetFileName += getTmpExtension(); + renameAfterCopy = true; + } + Store targetStore = getManager().getStore(targetStoreUri.getStoreId()); + + if (RemoteStoreProtocol.FTP.equals(targetStoreUri.getProtocol()) && !"NEXIO1".equals(targetStore.getName()) + && !"NEXIO2".equals(targetStore.getName())) { + currentTargetFileName += getTmpExtension(); + renameAfterFTP = true; + } + } + + sourceStoreUri.transferFrom(targetStoreUri, sourceFileName, currentTargetFileName); + + logger.info(getMarker(), "Transfer of {} completed from {} to {}", sourceFileName, sourceStoreUri, currentTargetFileName); + + if (renameAfterCopy) { + Path tmpTargetFile = Paths.get(targetStoreUri.toString(true), currentTargetFileName); + Path targetFile = Paths.get(targetStoreUri.toString(true), targetFileName); + try { + logger.info(getMarker(), "Renaming LOCAL file from {} to {} on {}", currentTargetFileName, targetFileName, sourceStoreUri); + tmpTargetFile.toFile().renameTo(targetFile.toFile()); + } catch (Exception e) { + logger.error(getMarker(), e.getMessage()); + } + } + if (renameAfterFTP) { + try { + FtpDirectoryLister lister = (FtpDirectoryLister) targetStoreUri.getLister(); + FTPClient client = lister.connect(); + logger.info(getMarker(), "Renaming FTP file from {} to {} on {}", currentTargetFileName, targetFileName, sourceStoreUri); + client.rename(currentTargetFileName, targetFileName); + } catch (Exception e) { + logger.error(getMarker(), e.getMessage()); + } finally { + targetStoreUri.cleanUp(); + } + } + } + + @StepEntry + public Object[] execute(StoreUri sourceStoreUri, String sourceFileName, StoreUri targetStoreUri, String targetFileName) throws Exception { + try { + getJobRuntime().setCancelable(false); + + Store sourceStore = getManager().getStore(sourceStoreUri.getStoreId()); + Store targetStore = getManager().getStore(targetStoreUri.getStoreId()); + + setDescription("{} -> {} : {}", sourceStore.getName(), targetStore.getName(), sourceFileName); + + sourceStoreUri.addProgressListener(e -> setProgress(e.getProgress())); + + StoreUri currentTargetStoreUri = getTargetStoreUri(targetStoreUri); + + try { + RemoteFile remoteFile = currentTargetStoreUri.getRemoteFile(targetFileName); + if (remoteFile != null) { + logger.info("File {} already exists on target {}, skipping transfer", targetFileName, targetStore.getName()); + } + } catch (Exception e) { + } + + copy(sourceStoreUri, sourceFileName, currentTargetStoreUri, targetFileName); + } catch (Exception e) { + logger.error(getMarker(), "Error in transfer of {} when copying from {} to {}.", sourceFileName, sourceStoreUri, targetStoreUri); + throw e; + } finally { + getJobRuntime().setDescription(null); + if (sourceStoreUri != null) + sourceStoreUri.cleanUp(); + if (targetStoreUri != null) + targetStoreUri.cleanUp(); + } + return null; + } + + protected StoreUri getTargetStoreUri(StoreUri targetStoreUri) { + return targetStoreUri; + } + + protected String getTmpExtension() { + return DOT_PART; + } +} diff --git a/server/-product/production/MEDIAVIVANTIS/jobs/steps/shared/EscortFiles.java b/server/-product/production/MEDIAVIVANTIS/jobs/steps/shared/EscortFiles.java new file mode 100644 index 00000000..e5040995 --- /dev/null +++ b/server/-product/production/MEDIAVIVANTIS/jobs/steps/shared/EscortFiles.java @@ -0,0 +1,348 @@ +package user.jobengine.server.steps.shared; + +import java.io.ByteArrayInputStream; +import java.io.File; +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; +import java.io.StringWriter; +import java.io.UnsupportedEncodingException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.nio.file.attribute.FileAttribute; +import java.nio.file.attribute.PosixFilePermission; +import java.nio.file.attribute.PosixFilePermissions; +import java.text.SimpleDateFormat; +import java.util.Calendar; +import java.util.Date; +import java.util.Set; + +import javax.xml.parsers.DocumentBuilder; +import javax.xml.parsers.DocumentBuilderFactory; +import javax.xml.transform.OutputKeys; +import javax.xml.transform.Transformer; +import javax.xml.transform.TransformerConfigurationException; +import javax.xml.transform.TransformerException; +import javax.xml.transform.TransformerFactory; +import javax.xml.transform.TransformerFactoryConfigurationError; +import javax.xml.transform.dom.DOMSource; +import javax.xml.transform.stream.StreamResult; + +import org.apache.commons.lang.StringUtils; +import org.apache.commons.net.ftp.FTPClient; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.apache.logging.log4j.Marker; +import org.apache.logging.log4j.message.Message; +import org.w3c.dom.DOMImplementation; +import org.w3c.dom.Document; +import org.w3c.dom.Element; + +import com.ibm.nosql.json.api.BasicDBObject; +import com.ibm.nosql.json.util.JSON; + +import user.commons.CalendarUtils; +import user.commons.MediaCubeMarker; +import user.commons.StoreUri; +import user.commons.remotestore.FtpDirectoryLister; + +public class EscortFiles { + private static final String RECORDTIMESTAMP = "RecordTimeStamp"; + private static final String MODIFIEDTIMESTAMP = "ModifiedTimeStamp"; + public static final String DOT_CATCHED = ".catched"; + public static final String DOT_JSON = ".json"; + private static final Logger logger = LogManager.getLogger(); + private static final String EXTENDEDAGENCY = "ExtendedAgency"; + private static final String EXTENDEDDESCRIPTION = "ExtendedDescription"; + private static final String KILLDATE = "KillDate"; + private static final String FORMAT_KILLDATE = "MM-dd-yyyy"; + private static final String EXTENDEDID = "extendedId"; + private static final String ID = "ID"; + private static final String KILLDATE_FILENAME = "%s.%s.killdate"; + private static final String FORMAT_KILLDATENAME = "yyyyMMdd"; + public static final String STATUSFOLDER = ".STATUS"; + public static final String CONFLICTFOLDER = ".CONFLICT"; + + public static String composeKillDate(int days) { + Calendar killDate = Calendar.getInstance(); + killDate.add(Calendar.DAY_OF_YEAR, days); + SimpleDateFormat dateFormat = new SimpleDateFormat(FORMAT_KILLDATENAME); + return dateFormat.format(killDate.getTime()); + } + /* + * + * 02-02-2018 TEST + * TEST AGENT AGENT + */ + + private static String composeKillDateFileName(String fileName, int days) { + return String.format(KILLDATE_FILENAME, fileName, composeKillDate(days)); + } + + public static void createCatchedFile(Path escortFile) throws IOException { + Path catchedFilePath = Paths.get(escortFile.toString() + DOT_CATCHED); + Files.createFile(catchedFilePath); + } + + public static void createFellow(String escortFile, String extension) throws IOException { + Files.copy(Paths.get(escortFile), Paths.get(escortFile + "." + extension)); + } + + /*** + * A media eleresi utjan alapjan a .STATUS almappaban letrehozza a .catch fajlt. + * + * @param mediaFile + * @throws IOException + */ + public static void createMediaCatch(Path mediaFile) throws IOException { + Path catchedFile = createMediaCathFilePath(mediaFile); + ensureUNCFolder(catchedFile.getParent()); + Files.createFile(catchedFile); + } + + public static Path createMediaCathFilePath(Path mediaFile) { + String fileName = mediaFile.getFileName().toString() + DOT_CATCHED; + return Paths.get(mediaFile.getParent().toString(), STATUSFOLDER, fileName); + } + + public static void createMetadata(String filePath, String fileName, String metadata) throws IOException { + ensureUNCFolder(filePath, STATUSFOLDER); + String metadataFileName = fileName + DOT_JSON; + Path metadataPath = Paths.get(filePath, STATUSFOLDER, metadataFileName); + Files.write(metadataPath, metadata.getBytes()); + } + + public static boolean createMetadataIfNotExists(String filePath, String fileName, String metadata) + throws IOException { + boolean result = false; + if (!EscortFiles.isMetadataExists(filePath, fileName)) { + EscortFiles.createMetadata(filePath, fileName, metadata); + result = true; + } + return result; + } + + public static void createMorpheusXML(String filePath, String fileName, String content) throws IOException { + ensureUNCFolder(filePath, STATUSFOLDER); + Path xmlPath = Paths.get(filePath, fileName); + if (Files.exists(xmlPath)) + throw new IOException(String.format("Az '%s' állomány már létezik.", xmlPath)); + Files.write(xmlPath, content.getBytes()); + } + + public static byte[] createNEXIODatesMeta(String fileName, Date recorded, Date modified) throws Exception { + DocumentBuilderFactory dbf = DocumentBuilderFactory.newInstance(); + DocumentBuilder db = dbf.newDocumentBuilder(); + DOMImplementation impl = db.getDOMImplementation(); + Document xmlDocument = impl.createDocument(null, null, null); + + Element root = xmlDocument.createElement(ID); + root.setAttribute(EXTENDEDID, fileName); + // 07-13-2020 (19:36:52) + // 05-18-2013 (18:52:24) + SimpleDateFormat df = new SimpleDateFormat("MM-dd-yyyy (HH:mm:ss)"); + root.appendChild(xmlDocument.createElement(MODIFIEDTIMESTAMP)) + .appendChild(xmlDocument.createTextNode(df.format(modified))); + root.appendChild(xmlDocument.createElement(RECORDTIMESTAMP)) + .appendChild(xmlDocument.createTextNode(df.format(recorded))); + xmlDocument.appendChild(root); + + return xmDocumentToString(xmlDocument); + } + + public static byte[] createNEXIOKillDateFile(String fileName, Date killDate, String description, String agency) + throws Exception { + DocumentBuilderFactory dbf = DocumentBuilderFactory.newInstance(); + DocumentBuilder db = dbf.newDocumentBuilder(); + DOMImplementation impl = db.getDOMImplementation(); + Document xmlDocument = impl.createDocument(null, null, null); + + Element root = xmlDocument.createElement(ID); + root.setAttribute(EXTENDEDID, fileName); + if (killDate != null) { + String sKillDate = CalendarUtils.toString(CalendarUtils.createCalendar(killDate), FORMAT_KILLDATE); + root.appendChild(xmlDocument.createElement(KILLDATE)).appendChild(xmlDocument.createTextNode(sKillDate)); + } + + if (StringUtils.isNotBlank(description)) + root.appendChild(xmlDocument.createElement(EXTENDEDDESCRIPTION)) + .appendChild(xmlDocument.createTextNode(description)); + if (StringUtils.isNotBlank(agency)) + root.appendChild(xmlDocument.createElement(EXTENDEDAGENCY)).appendChild(xmlDocument.createTextNode(agency)); + xmlDocument.appendChild(root); + + return xmDocumentToString(xmlDocument); + } + + public static Document createNEXIOMeta(byte[] content) throws Exception { + DocumentBuilderFactory dbf = DocumentBuilderFactory.newInstance(); + DocumentBuilder db = dbf.newDocumentBuilder(); + DOMImplementation impl = db.getDOMImplementation(); + Document xmlDocument = null; + + try (InputStream is = new ByteArrayInputStream(content)) { + xmlDocument = db.parse(is); + } catch (Exception e) { + logger.catching(e); + } + + return xmlDocument; + } + + public static void createUNCKillDate(String filePath, String fileName, int days, Marker marker) throws IOException { + ensureUNCFolder(filePath, STATUSFOLDER); + String killDateFileName = composeKillDateFileName(fileName, days); + Path killDatePath = Paths.get(filePath, STATUSFOLDER, killDateFileName); + if (Files.exists(killDatePath)) + logger.warn(marker, "Az '{}' állomány már létezik.", killDatePath); + else + Files.createFile(killDatePath); + } + + @SuppressWarnings("unchecked") + public static T decode(Path escortFile) { + T result = null; + try { + byte[] bytes = Files.readAllBytes(escortFile); + String content = new String(bytes); + result = (T) JSON.parse(content); + } catch (Exception e) { + logger.error("Decode error. System message is: ", e.getMessage()); + } + return result; + } + + public static void ensureUNCFolder(Path filePath) throws IOException { + File folder = filePath.toFile(); + if (!folder.exists() || !folder.isDirectory()) { + try { + Set perms = PosixFilePermissions.fromString("rwxrwxrwx"); + FileAttribute> attr = PosixFilePermissions.asFileAttribute(perms); + Files.createDirectories(filePath, attr); + } catch (Exception e) { + // logger.catching(e); + try { + Files.createDirectories(filePath); + } catch (Exception e1) { + logger.catching(e1); + throw e1; + } + } + } + } + + public static void ensureUNCFolder(String filePath, String folderName) throws IOException { + Path statusPath = Paths.get(filePath, folderName); + ensureUNCFolder(statusPath); + } + + public static boolean isCatchedFileExists(Path escortFile) { + Path catchedFilePath = Paths.get(escortFile.toString() + DOT_CATCHED); + return catchedFilePath.toFile().exists(); + } + + /*** + * A media eleresi utjan alapjan a .STATUS almappaban vizsgalja .catch fajl + * letezeset. + * + * @param mediaFile + * @return + */ + public static boolean isMediaCatched(Path mediaFile) { + Path catchedFile = createMediaCathFilePath(mediaFile); + return catchedFile.toFile().exists(); + } + + public static boolean isMetadataExists(String filePath, String fileName) throws IOException { + boolean result = false; + String metadataFileName = fileName + DOT_JSON; + Path metadataPath = Paths.get(filePath, STATUSFOLDER, metadataFileName); + result = metadataPath.toFile().exists(); + return result; + } + + public static void notifyRecipient(Path escortFile, Logger logger, Message msg) { + if (escortFile.toFile().exists()) { + try { + BasicDBObject downloadable = EscortFiles.decode(escortFile); + String recipientKey = "recipient"; + if (downloadable.containsKey(recipientKey)) { + String recipient = downloadable.getString(recipientKey); + logger.info(new MediaCubeMarker(recipient, "MediaCube rendszerüzenet"), msg); + } + } catch (Exception e) { + logger.catching(e); + } + + } + } + + public static void remove(Path file) { + try { + file.toFile().delete(); + } catch (Exception e) { + logger.error("Unable to delete {}", file.toAbsolutePath().toString()); + } + } + + public static void removeCatchedFile(Path escortFile) { + remove(Paths.get(escortFile.toString() + DOT_CATCHED)); + } + + /*** + * A media eleresi utjan alapjan a .STATUS almappabol torli a .catch fajlt. + * + * @param mediaFile + * @throws IOException + */ + public static void removeMediaCatch(Path mediaFile) { + Path catchedFile = createMediaCathFilePath(mediaFile); + remove(catchedFile); + } + + public static void setNEXIOKillDate(int killDateDays, String targetFileName, String nexioAgency, StoreUri targetUri) + throws Exception { + OutputStream outStream = null; + try { + FTPClient targetFTP = ((FtpDirectoryLister) targetUri.getLister()).connect(); + Calendar killDate = CalendarUtils.createCalendar(new Date()); + killDate.add(Calendar.DAY_OF_YEAR, killDateDays); + if (targetFileName.toLowerCase().contains(".mxf")) + targetFileName = targetFileName.substring(0, targetFileName.lastIndexOf('.')); + byte[] killDateFile = EscortFiles.createNEXIOKillDateFile(targetFileName, killDate.getTime(), null, + nexioAgency); + String xml = targetFileName + ".xml"; + outStream = targetFTP.storeFileStream(xml); + if (outStream == null) { + throw new NullPointerException( + "Can not open: " + targetFileName.substring(0, targetFileName.lastIndexOf('.')) + ".xml" + + " Reply:" + targetFTP.getReplyString()); + } + outStream.write(killDateFile); + outStream.flush(); + } catch (Exception e) { + throw e; + } finally { + if (outStream != null) + outStream.close(); + targetUri.cleanUp(); + } + } + + private static byte[] xmDocumentToString(Document xmlDocument) throws TransformerFactoryConfigurationError, + TransformerConfigurationException, TransformerException, IOException, UnsupportedEncodingException { + DOMSource domSource = new DOMSource(xmlDocument); + TransformerFactory tf = TransformerFactory.newInstance(); + Transformer transformer = tf.newTransformer(); + transformer.setOutputProperty(OutputKeys.METHOD, "xml"); + transformer.setOutputProperty(OutputKeys.ENCODING, "UTF-16"); + transformer.setOutputProperty(OutputKeys.INDENT, "yes"); + StringWriter sw = new StringWriter(); + StreamResult sr = new StreamResult(sw); + transformer.transform(domSource, sr); + String result = sw.toString(); + sw.close(); + return result.getBytes("UTF-16"); + } +} diff --git a/server/-product/production/MEDIAVIVANTIS/jobs/steps/shared/ExternalCommand.java b/server/-product/production/MEDIAVIVANTIS/jobs/steps/shared/ExternalCommand.java new file mode 100644 index 00000000..307f36d0 --- /dev/null +++ b/server/-product/production/MEDIAVIVANTIS/jobs/steps/shared/ExternalCommand.java @@ -0,0 +1,79 @@ +package user.jobengine.server.steps.shared; + +import java.io.BufferedReader; +import java.io.InputStreamReader; +import java.util.ArrayList; +import java.util.List; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + +public class ExternalCommand { + private static final Logger logger = LogManager.getLogger(); + private ExternalProfile profile; + + public ExternalCommand(ExternalProfile profile) { + this.profile = profile; + } + + public String execute(String input, String output, boolean firstResponse, IExternalCallback responseCallBack) throws Exception { + List arguments = getArguments(input, output); + List command = new ArrayList<>(); + command.add(profile.getExecutable()); + command.addAll(arguments); + + ProcessBuilder processBuilder = new ProcessBuilder(); + processBuilder.command(command); + + String result = null; + try { + logger.info("Executing : {}", processBuilder.command()); + + Process process = processBuilder.start(); + try (BufferedReader reader = new BufferedReader(new InputStreamReader(process.getInputStream()))) { + String line = null; + while ((line = reader.readLine()) != null) { + logger.debug("Process response: {}", line); + if (responseCallBack != null) + responseCallBack.onResponse(line); + //System.out.println(line); + if (line != null && line.length() > 0) { + result = line; + if (firstResponse) + break; + } + } + int exitCode = process.waitFor(); + if (exitCode != 0) { + StringBuilder msg = new StringBuilder(); + try (BufferedReader errReader = new BufferedReader(new InputStreamReader(process.getErrorStream()))) { + String errline = null; + while ((errline = errReader.readLine()) != null) { + msg.append(errline); + } + } catch (Exception ex) { + } + + throw new Exception("Exited with error code : " + exitCode + ". " + msg); + } + } catch (Exception e) { + throw e; + } + } catch (Exception e) { + logger.error(e); + throw e; + } + + return result; + } + + private List getArguments(String input, String output) { + List result = new ArrayList<>(); + + profile.getArguments().forEach(i -> { + result.add(i.replace("%i", input).replace("%o", output)); + }); + return result; + } + +} \ No newline at end of file diff --git a/server/-product/production/MEDIAVIVANTIS/jobs/steps/shared/ExternalCommandExecutor.java b/server/-product/production/MEDIAVIVANTIS/jobs/steps/shared/ExternalCommandExecutor.java new file mode 100644 index 00000000..a34ff7f0 --- /dev/null +++ b/server/-product/production/MEDIAVIVANTIS/jobs/steps/shared/ExternalCommandExecutor.java @@ -0,0 +1,32 @@ +package user.jobengine.server.steps.shared; + +import user.commons.configuration.SystemConfiguration; + +public class ExternalCommandExecutor { + + public void execute(String profileName, String input, String output, IExternalCallback responseCallBack) throws Exception { + ExternalCommand externalCommand = getExternalCommand(profileName); + externalCommand.execute(input, output, false, responseCallBack); + } + + private ExternalCommand getExternalCommand(String profileName) throws Exception { + ExternalProfilesConfig config = SystemConfiguration.getInstance().load("settings/external-commands.yaml", ExternalProfilesConfig.class); + + if (config == null) + throw new Exception("Missing external-commands.yaml configuration"); + + ExternalProfile selectedProfile = null; + for (ExternalProfile profile : config.getProfiles()) { + if (profileName.equals(profile.getName())) { + selectedProfile = profile; + break; + } + } + + if (selectedProfile == null) + throw new Exception("Missing profile " + profileName + " in external-commands.yaml configuration"); + + return new ExternalCommand(selectedProfile); + } + +} diff --git a/server/-product/production/MEDIAVIVANTIS/jobs/steps/shared/ExternalProfile.java b/server/-product/production/MEDIAVIVANTIS/jobs/steps/shared/ExternalProfile.java new file mode 100644 index 00000000..68e22f4c --- /dev/null +++ b/server/-product/production/MEDIAVIVANTIS/jobs/steps/shared/ExternalProfile.java @@ -0,0 +1,33 @@ +package user.jobengine.server.steps.shared; + +import java.util.List; + +public class ExternalProfile { + private String executable; + private String name; + private List arguments; + + public List getArguments() { + return arguments; + } + + public String getExecutable() { + return executable; + } + + public String getName() { + return name; + } + + public void setArguments(List arguments) { + this.arguments = arguments; + } + + public void setExecutable(String executable) { + this.executable = executable; + } + + public void setName(String name) { + this.name = name; + } +} \ No newline at end of file diff --git a/server/-product/production/MEDIAVIVANTIS/jobs/steps/shared/ExternalProfilesConfig.java b/server/-product/production/MEDIAVIVANTIS/jobs/steps/shared/ExternalProfilesConfig.java new file mode 100644 index 00000000..7ef77291 --- /dev/null +++ b/server/-product/production/MEDIAVIVANTIS/jobs/steps/shared/ExternalProfilesConfig.java @@ -0,0 +1,15 @@ +package user.jobengine.server.steps.shared; + +import java.util.List; + +public class ExternalProfilesConfig { + private List profiles; + + public List getProfiles() { + return profiles; + } + + public void setProfiles(List profiles) { + this.profiles = profiles; + } +} \ No newline at end of file diff --git a/server/-product/production/MEDIAVIVANTIS/jobs/steps/shared/IExternalCallback.java b/server/-product/production/MEDIAVIVANTIS/jobs/steps/shared/IExternalCallback.java new file mode 100644 index 00000000..1e071913 --- /dev/null +++ b/server/-product/production/MEDIAVIVANTIS/jobs/steps/shared/IExternalCallback.java @@ -0,0 +1,5 @@ +package user.jobengine.server.steps.shared; + +public interface IExternalCallback { + void onResponse(String data); +} \ No newline at end of file diff --git a/server/-product/production/MEDIAVIVANTIS/jobs/steps/shared/ItemManagerExtensions.java b/server/-product/production/MEDIAVIVANTIS/jobs/steps/shared/ItemManagerExtensions.java new file mode 100644 index 00000000..32f17cb5 --- /dev/null +++ b/server/-product/production/MEDIAVIVANTIS/jobs/steps/shared/ItemManagerExtensions.java @@ -0,0 +1,94 @@ +package user.jobengine.server.steps.shared; + +import java.nio.file.Path; + +import com.ibm.nosql.json.api.BasicDBObject; + +import user.commons.RemoteFile; +import user.commons.StoreUri; +import user.commons.remotestore.RemoteStoreProtocol; +import user.jobengine.db.IItemManager; +import user.jobengine.db.IResultSetConsumer; +import user.jobengine.db.Store; + +public class ItemManagerExtensions { + + public static BasicDBObject getArchiveInfo(IItemManager manager, long houseid) { + final BasicDBObject[] result = { null }; + StringBuilder query = new StringBuilder(); + query.append("select count(*) as count, sum(length) as duration FROM media"); + query.append(" "); + query.append(String.format("where houseid='%d' and itemtypeid = 82", houseid)); + query.append(" "); + query.append("group by houseid"); + IResultSetConsumer consumer = rs -> { + BasicDBObject o = new BasicDBObject(); + o.put("count", rs.getLong("count")); + o.put("duration", rs.getLong("duration")); + result[0] = o; + return false; + }; + manager.executeQuery(query.toString(), consumer, null); + return result[0]; + } + + public static long getExistingRundownMedia(IItemManager manager, String houseid) { + final long[] result = new long[] { 0 }; + final String[] idToCheck = new String[] { houseid }; + int pos = houseid.lastIndexOf("-"); + //a hivas a CopyForArchiveNEXIOMaterialsStep-bol is johet, ott meg nincs idobelyegezve a nev! + if (pos > 0 && houseid.length() - pos > 4) + idToCheck[0] = houseid.substring(0, pos); + MetadataType metadataType = MetadataTypeDetector.GuessMetadataType(idToCheck[0]); + if (metadataType == MetadataType.OctopusPlaceholder) { + StringBuilder query = new StringBuilder(); + query.append("select mediaid, mediafilehouseid, filename"); + query.append(" "); + query.append(String.format("from vw_rundown_items where mediafilehouseid like '%s%%'", idToCheck[0])); + query.append(" "); + query.append("order by filename, mediaid"); + IResultSetConsumer consumer = rs -> { + String fileName = rs.getString("filename"); + if (idToCheck[0].equals(fileName)) { + result[0] = rs.getLong("mediaid"); + return false; + } else + return true; + }; + manager.executeQuery(query.toString(), consumer, null); + } + + return result[0]; + } + + static public boolean isArchived(IItemManager manager, Path filePath) { + boolean result = false; + String name = filePath.getFileName().toString(); + String[] tsmName = new String[] { null }; + String query = String.format("SELECT relativepath FROM MEDIAFILE WHERE houseid = '%s'", name); + manager.executeQuery(query, rs -> { + tsmName[0] = rs.getString("relativepath"); + return false; + }, null); + + Store tsmStore = manager.getSystemStore(false); + if (tsmStore == null) + throw new NullPointerException("A TSM bejegyzés nem található!"); + + StoreUri tsmStoreUri = tsmStore.getSourceStoreUri(RemoteStoreProtocol.TSM); + if (tsmStoreUri == null) + throw new NullPointerException("A TSM forrás elérése nem található!"); + + if (tsmName[0] != null) { + try { + RemoteFile remoteFile = tsmStoreUri.getRemoteFile(tsmName[0]); + result = remoteFile != null; + } catch (Exception e) { + result = false; + } finally { + tsmStoreUri.cleanUp(); + } + } + return result; + } +} diff --git a/server/-product/production/MEDIAVIVANTIS/jobs/steps/shared/MediaCubeClient.java b/server/-product/production/MEDIAVIVANTIS/jobs/steps/shared/MediaCubeClient.java new file mode 100644 index 00000000..0915cc33 --- /dev/null +++ b/server/-product/production/MEDIAVIVANTIS/jobs/steps/shared/MediaCubeClient.java @@ -0,0 +1,77 @@ +package user.jobengine.server.steps.shared; + +import javax.ws.rs.client.Entity; +import javax.ws.rs.client.Invocation.Builder; +import javax.ws.rs.core.MediaType; +import javax.ws.rs.core.MultivaluedMap; +import javax.ws.rs.core.Response; +import javax.ws.rs.core.Response.Status; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.jboss.resteasy.client.jaxrs.ResteasyClient; +import org.jboss.resteasy.client.jaxrs.ResteasyClientBuilder; +import org.jboss.resteasy.client.jaxrs.ResteasyWebTarget; +import org.jboss.resteasy.specimpl.MultivaluedMapImpl; + +import com.ibm.nosql.json.JSONUtil; +import com.ibm.nosql.json.api.BasicDBObject; + +public class MediaCubeClient { + private static Logger logger = LogManager.getLogger(); + private ResteasyWebTarget webTarget; + + public MediaCubeClient(String address) { + ResteasyClient client = new ResteasyClientBuilder().build(); + webTarget = client.target(address); + } + + BasicDBObject getDbObject(String json) { + BasicDBObject result = (BasicDBObject) JSONUtil.jsonToDbObject(json); + + if (result == null) + throw new NullPointerException("API Result is null!"); + + if (result.containsKey("exception")) { + BasicDBObject e = (BasicDBObject) result.get("exception"); + throw new RuntimeException(e.getString("message")); + } + //{"exception":{"message":"Invalid credentials.","publicName":"AuthenticationFailedException"}} + return result; + } + + public BasicDBObject getStatus(long jobId) { + MultivaluedMap vars = new MultivaluedMapImpl<>(); + vars.add("jobId", jobId); + Response response = query("services/rest/jobengine/jobstatus", vars).get(); + if (response.getStatus() != Status.OK.getStatusCode()) { + logger.error(response.readEntity(String.class)); + System.out.println(response.readEntity(String.class)); + return null; + } + String result = response.readEntity(String.class); + return getDbObject(result); + } + + private Builder query(String path, MultivaluedMap vars) { + ResteasyWebTarget target = webTarget.path(path).queryParams(vars); + Builder result = target.request(); + return result; + } + + public long startjob(String template, String name, BasicDBObject jobParams) throws Exception { + MultivaluedMap vars = new MultivaluedMapImpl<>(); + vars.add("template", template); + vars.add("name", name); + Response response = query("services/rest/jobengine/startjob", vars).post(Entity.entity(jobParams.toString(), MediaType.APPLICATION_JSON)); + + if (response.getStatus() != Status.OK.getStatusCode()) { + logger.error(response.readEntity(String.class)); + return 0; + } + + String resultObject = response.readEntity(String.class); + return Long.parseLong(resultObject); + } + +} diff --git a/server/-product/production/MEDIAVIVANTIS/jobs/steps/shared/MetadataType.java b/server/-product/production/MEDIAVIVANTIS/jobs/steps/shared/MetadataType.java new file mode 100644 index 00000000..0e27bf34 --- /dev/null +++ b/server/-product/production/MEDIAVIVANTIS/jobs/steps/shared/MetadataType.java @@ -0,0 +1,5 @@ +package user.jobengine.server.steps.shared; + +public enum MetadataType { + TrafficMaterial, TrafficPromo, TrafficAD, OctopusStory, OctopusPlaceholder, Generic +} diff --git a/server/-product/production/MEDIAVIVANTIS/jobs/steps/shared/MetadataTypeDetector.java b/server/-product/production/MEDIAVIVANTIS/jobs/steps/shared/MetadataTypeDetector.java new file mode 100644 index 00000000..9e9e6be9 --- /dev/null +++ b/server/-product/production/MEDIAVIVANTIS/jobs/steps/shared/MetadataTypeDetector.java @@ -0,0 +1,49 @@ +package user.jobengine.server.steps.shared; + +import org.apache.commons.lang.StringUtils; + +public class MetadataTypeDetector { + + private static final String HYPHEN = "-"; + private static final String DOT = "."; + + private static final String REGEXP_TRAFFICMATERIALID = "^M{1}[0-9]{6}[A-Z]{1}"; + private static final String REGEXP_TRAFFICADID = "^R{1}[0-9]{6}[A-Z]{1}"; + private static final String REGEXP_TRAFFICPROMOID = "^P{1}[0-9]{6}[A-Z]{1}"; + private static final String REGEXP_OCTOPUSSTORYID = "^[0-9]+"; + private static final String REGEXP_OCTOPUSPLACEHOLDERID = "^[0-9]+_[0-9]+"; + private static final String REGEXP_OCTOPUSPLACEHOLDERVERSIONEDID = "^[0-9]+_[0-9]+-[0-9]{3}"; + + public static MetadataType GuessMetadataType(String id) { + if (StringUtils.isBlank(id)) + return MetadataType.Generic; + if (id.matches(REGEXP_TRAFFICMATERIALID)) + return MetadataType.TrafficMaterial; + if (id.matches(REGEXP_TRAFFICPROMOID)) + return MetadataType.TrafficPromo; + if (id.matches(REGEXP_TRAFFICADID)) + return MetadataType.TrafficAD; + if (id.matches(REGEXP_OCTOPUSSTORYID)) + return MetadataType.OctopusStory; + if (id.matches(REGEXP_OCTOPUSPLACEHOLDERID)) + return MetadataType.OctopusPlaceholder; + if (id.matches(REGEXP_OCTOPUSPLACEHOLDERVERSIONEDID)) + return MetadataType.OctopusPlaceholder; + return MetadataType.Generic; + } + + public static String truncateExtension(String name) { + String result = name; + if (result != null && result.contains(DOT)) + result = result.substring(0, result.lastIndexOf(DOT)); + return result; + } + + public static String truncateVersion(String name) { + String result = name; + if (result != null && result.contains(HYPHEN)) + result = result.split(HYPHEN)[0]; + return result; + } + +} diff --git a/server/user.jobengine.executors/src/user/jobengine/server/steps/EscortFileUpdaterStep.java b/server/user.jobengine.executors/src/user/jobengine/server/steps/EscortFileUpdaterStep.java new file mode 100644 index 00000000..5dcb5239 --- /dev/null +++ b/server/user.jobengine.executors/src/user/jobengine/server/steps/EscortFileUpdaterStep.java @@ -0,0 +1,57 @@ +package user.jobengine.server.steps; + +import java.io.FileNotFoundException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + +import user.commons.DownloadableMedia; +import user.commons.StoreUri; +import user.commons.mediatool.MediaInfo; +import user.commons.remotestore.RemoteStoreProtocol; +import user.jobengine.server.steps.shared.EscortFiles; + +public class EscortFileUpdaterStep extends JobStep { + private static final Logger logger = LogManager.getLogger(); + + @StepEntry + public Object[] execute(StoreUri sourceStoreUri, String fileName, String escortFile) throws Exception { + if (!RemoteStoreProtocol.LOCAL.equals(sourceStoreUri.getProtocol())) + throw new Exception("Store URI protocol must be local."); + Path filePath = Paths.get(sourceStoreUri.toString(true), fileName); + if (!filePath.toFile().exists()) + throw new FileNotFoundException("File not exists: " + filePath.toString()); + + Path escortFilePath = Paths.get(escortFile); + DownloadableMedia downloadable = null; + try { + downloadable = EscortFiles.decode(escortFilePath); + } catch (Exception e) { + logger.error(getSessionMarker(), e.getMessage()); + } + + if (downloadable != null) + storeCurrentFrames(filePath, downloadable, escortFilePath); + + return null; + } + + private void storeCurrentFrames(Path filePath, DownloadableMedia downloadable, Path escortFilePath) { + MediaInfo mi = null; + try { + mi = new MediaInfo(filePath); + mi.process(); + long frames = mi.getFrames(); + downloadable.put("frames", frames); + Files.write(escortFilePath, downloadable.toPrettyString("").getBytes()); + logger.info(getMarker(), "{} length {} is saved to escort file", filePath, frames, + escortFilePath.getFileName().toString()); + } catch (Exception e) { + logger.error(getSessionMarker(), e.getMessage()); + } + } + +} diff --git a/server/user.jobengine.executors/src/user/jobengine/server/steps/PBARCHIVETransferFromStep.java b/server/user.jobengine.executors/src/user/jobengine/server/steps/PBARCHIVETransferFromStep.java index d47c2b79..7ff4702f 100644 --- a/server/user.jobengine.executors/src/user/jobengine/server/steps/PBARCHIVETransferFromStep.java +++ b/server/user.jobengine.executors/src/user/jobengine/server/steps/PBARCHIVETransferFromStep.java @@ -1,19 +1,12 @@ package user.jobengine.server.steps; -import java.nio.file.Path; -import java.nio.file.Paths; - -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; - import user.commons.StoreUri; public class PBARCHIVETransferFromStep extends TransferStep { - private static final Logger logger = LogManager.getLogger(); - @Override @StepEntry - public Object[] execute(StoreUri sourceStoreUri, String sourceFileName, StoreUri targetStoreUri, String targetFileName) throws Exception { + public Object[] execute(StoreUri sourceStoreUri, String sourceFileName, StoreUri targetStoreUri, + String targetFileName) throws Exception { Object[] result = null; result = super.execute(sourceStoreUri, sourceFileName, targetStoreUri, targetFileName); return result; diff --git a/server/user.jobengine.executors/src/user/jobengine/server/steps/PBArchiveCheckerStep.java b/server/user.jobengine.executors/src/user/jobengine/server/steps/PBArchiveCheckerStep.java index 071ed7cf..75b1a357 100644 --- a/server/user.jobengine.executors/src/user/jobengine/server/steps/PBArchiveCheckerStep.java +++ b/server/user.jobengine.executors/src/user/jobengine/server/steps/PBArchiveCheckerStep.java @@ -3,38 +3,28 @@ package user.jobengine.server.steps; import java.io.IOException; import java.nio.file.Paths; import java.sql.Timestamp; -import java.text.SimpleDateFormat; import java.time.Duration; import java.time.Instant; import java.util.ArrayList; -import java.util.Iterator; import java.util.List; import org.apache.commons.net.ftp.FTPClient; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; -import org.w3c.dom.stylesheets.LinkStyle; import user.commons.DownloadableMedia; import user.commons.RemoteFile; import user.commons.StoreUri; -import user.commons.nexio.api.Clip; import user.commons.nexio.api.ClipNotFoundException; -import user.commons.nexio.api.Controller; -import user.commons.nexio.api.Mediabase; import user.commons.nexio.server.protocol.ProtocolException; import user.commons.remotestore.FtpDirectoryLister; -import user.commons.remotestore.IDirectoryLister; import user.commons.remotestore.RemoteStoreProtocol; import user.jobengine.db.Media; -import user.jobengine.db.MediaFile; import user.jobengine.server.steps.shared.EscortFiles; public class PBArchiveCheckerStep extends JobStep { private static final String DOT_MXF = ".mxf"; private static final Logger logger = LogManager.getLogger(); - private SimpleDateFormat df = new SimpleDateFormat("yyyyMMdd-HHmmss"); - private boolean checkModifiedByTime(String title, Timestamp modified, int lastModifiedHours) { long modifiedHours = Duration.between(modified.toInstant(), Instant.now()).toHours(); @@ -43,7 +33,7 @@ public class PBArchiveCheckerStep extends JobStep { return false; } if (modifiedHours < lastModifiedHours) { - logger.debug(getMarker(), "{} modification time is too cloose for now", title); + logger.info(getMarker(), "{} modification time is too cloose for now", title); return false; } @@ -67,24 +57,25 @@ public class PBArchiveCheckerStep extends JobStep { FTPClient client = lister.connect(); //a StoreUri alapu listazas nem mukodik, miota a working directory beallitas fix List remoteFiles = new ArrayList<>(); - + String[] listNames = client.listNames(); for (String f : listNames) { logger.info(getMarker(), "File name is {}", f); - + try { RemoteFile remoteFile = lister.get(f); if (remoteFile == null) logger.info(getMarker(), "Remote file for {} is not available", f); else remoteFiles.add(remoteFile); - - } catch (Exception iex){ + + } catch (Exception iex) { logger.error(getMarker(), iex.getMessage()); } } - - logger.info(getMarker(), "Found {} files in working dir {}", remoteFiles.size(), client.printWorkingDirectory()); + + logger.info(getMarker(), "Found {} files in working dir {}", remoteFiles.size(), + client.printWorkingDirectory()); processClips(sourceStoreName, sourceStoreUri, targetStoreUri, outputPath, remoteFiles, lastModifiedHours); } catch (Exception e) { @@ -100,9 +91,9 @@ public class PBArchiveCheckerStep extends JobStep { return null; } - private void processClips(String storeName, StoreUri sourceStoreUri, StoreUri targetStoreUri, String outputPath, List remoteFiles, - int lastModifiedHours) throws ClipNotFoundException, IOException, ProtocolException { - int processed = 0; + private void processClips(String storeName, StoreUri sourceStoreUri, StoreUri targetStoreUri, String outputPath, + List remoteFiles, int lastModifiedHours) + throws ClipNotFoundException, IOException, ProtocolException { for (RemoteFile remoteFile : remoteFiles) { if (getJobRuntime().isWaitingCancel()) { @@ -111,46 +102,45 @@ public class PBArchiveCheckerStep extends JobStep { break; } try { - + String title = null; Timestamp created = null; Timestamp modified = null; title = remoteFile.getName(); + + int dotIdx = title.lastIndexOf("."); + if (dotIdx > -1) + title = title.substring(0, dotIdx); + created = remoteFile.getCreated(); modified = remoteFile.getModify(); - logger.info(getMarker(), "{} with extension {} created {} modified {}", title, remoteFile.getExtension(), created, modified); + logger.info(getMarker(), "{} created {} modified {}", title, created, modified); Media media = getManager().getMedia(title); boolean createEscort = false; - if (!createEscort) - continue; - - if (checkModifiedByTime(title, modified, lastModifiedHours)) { + if (checkModifiedByTime(title, modified, lastModifiedHours)) createEscort = true; - logger.info(getMarker(), "{} not archived yet", title); - } - long mediaId = 0; - if (media != null) - mediaId = media.getId(); + if (!createEscort) + continue; if (createEscort) { + long mediaId = 0; + if (media != null) + mediaId = media.getId(); String fileName = title + DOT_MXF; - DownloadableMedia downloadable = DownloadableMedia.create(title, fileName, modified, created, 0, 0, sourceStoreUri.getId(), - targetStoreUri.getId(), mediaId); + DownloadableMedia downloadable = DownloadableMedia.create(title, fileName, modified, created, 0, 0, + sourceStoreUri.getId(), targetStoreUri.getId(), mediaId); String escortFileName = storeName + "." + downloadable.getString("fileName"); - if (EscortFiles.createMetadataIfNotExists(outputPath, escortFileName, downloadable.toPrettyString(""))) { + if (EscortFiles.createMetadataIfNotExists(outputPath, escortFileName, + downloadable.toPrettyString(""))) { logger.info(getMarker(), "Archive status file created for {}", fileName); } else { logger.info(getMarker(), "Archive status file already exists for {}", fileName); } - } - processed++; - if (processed % 100 == 0) - logger.info(getMarker(), "Processed {} records", processed); } catch (Exception e) { logger.error(getSessionMarker(), e.getMessage()); continue; -- 2.54.0