From 9edf7a15cf00b8086b0492ab5b7d6f7476a6ad3c Mon Sep 17 00:00:00 2001 From: "vasary.daniel" Date: Tue, 27 Jul 2021 12:40:45 +0000 Subject: [PATCH] git-tfs-id: [http://tfs.userrendszerhaz.hu:8080/tfs/DefaultCollection]$/MediaCube;C32285 --- .../jobs/steps/PBARCHIVETransferFromStep.java | 21 ++ .../AMC/jobs/steps/PBArchiveCheckerStep.java | 187 ++++++++++++++++++ .../templates/PBARCHIVE-archive-checker.xml | 26 +++ .../PBARCHIVE-validate-and-archive.xml | 94 +++++++++ .../tests/SupportMVDeletedFinishedShows.java | 72 +++++++ .../steps/PBARCHIVETransferFromStep.java | 21 ++ .../server/steps/PBArchiveCheckerStep.java | 161 +++++++++++++++ .../scripts/038_create_pbarchive_store.sql | 20 ++ 8 files changed, 602 insertions(+) create mode 100644 server/-product/production/AMC/jobs/steps/PBARCHIVETransferFromStep.java create mode 100644 server/-product/production/AMC/jobs/steps/PBArchiveCheckerStep.java create mode 100644 server/-product/production/AMC/jobs/templates/PBARCHIVE-archive-checker.xml create mode 100644 server/-product/production/AMC/jobs/templates/PBARCHIVE-validate-and-archive.xml create mode 100644 server/hu.user.mediacube.executors.tests/src/hu/user/mediacube/executors/tests/SupportMVDeletedFinishedShows.java create mode 100644 server/user.jobengine.executors/src/user/jobengine/server/steps/PBARCHIVETransferFromStep.java create mode 100644 server/user.jobengine.executors/src/user/jobengine/server/steps/PBArchiveCheckerStep.java create mode 100644 server/user.jobengine.osgi.db/migrations/scripts/038_create_pbarchive_store.sql diff --git a/server/-product/production/AMC/jobs/steps/PBARCHIVETransferFromStep.java b/server/-product/production/AMC/jobs/steps/PBARCHIVETransferFromStep.java new file mode 100644 index 00000000..d47c2b79 --- /dev/null +++ b/server/-product/production/AMC/jobs/steps/PBARCHIVETransferFromStep.java @@ -0,0 +1,21 @@ +package user.jobengine.server.steps; + +import java.nio.file.Path; +import java.nio.file.Paths; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + +import user.commons.StoreUri; + +public class PBARCHIVETransferFromStep extends TransferStep { + private static final Logger logger = LogManager.getLogger(); + + @Override + @StepEntry + public Object[] execute(StoreUri sourceStoreUri, String sourceFileName, StoreUri targetStoreUri, String targetFileName) throws Exception { + Object[] result = null; + result = super.execute(sourceStoreUri, sourceFileName, targetStoreUri, targetFileName); + return result; + } +} diff --git a/server/-product/production/AMC/jobs/steps/PBArchiveCheckerStep.java b/server/-product/production/AMC/jobs/steps/PBArchiveCheckerStep.java new file mode 100644 index 00000000..8beafb23 --- /dev/null +++ b/server/-product/production/AMC/jobs/steps/PBArchiveCheckerStep.java @@ -0,0 +1,187 @@ +package user.jobengine.server.steps; + +import java.io.IOException; +import java.nio.file.Paths; +import java.sql.Timestamp; +import java.text.SimpleDateFormat; +import java.time.Duration; +import java.time.Instant; +import java.util.Iterator; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + +import user.commons.DownloadableMedia; +import user.commons.StoreUri; +import user.commons.nexio.api.Clip; +import user.commons.nexio.api.ClipNotFoundException; +import user.commons.nexio.api.Controller; +import user.commons.nexio.api.Mediabase; +import user.commons.nexio.server.protocol.ProtocolException; +import user.commons.remotestore.RemoteStoreProtocol; +import user.jobengine.db.Media; +import user.jobengine.db.MediaFile; + +public class NEXIOArchiveCheckerStep extends JobStep { + private static final String DOT_MXF = ".mxf"; + private static final Logger logger = LogManager.getLogger(); + private SimpleDateFormat df = new SimpleDateFormat("yyyyMMdd-HHmmss"); + + private boolean check(Timestamp modified, Timestamp created, int duration, Media media, int videoFormat, int lastModifiedHours) { + String title = media.getTitle(); + + //formatum + if (!(videoFormat == 3 || videoFormat == 19)) { + logger.debug(getMarker(), "{} unsupported video format", videoFormat); + return false; + } + + if (!checkModifiedByTime(title, modified, lastModifiedHours)) + return false; + + //utolso modositas + MediaFile mediaFile = getManager().getSystemMediaFile(media); + boolean isModified = true; + if (mediaFile == null) { + logger.info(getMarker(), "{} system media file not available", title); + return false; + } + + String mcMod = df.format(mediaFile.getLastModified()); + String nexioMod = df.format(modified); + isModified = mcMod.compareTo(nexioMod) < 0; + if (isModified) { + logger.info(getMarker(), "{} modification time: MC {} < NEXIO {}", title, mcMod, nexioMod); + return true; + } + + boolean isLengthChanged = media.getLength() > 0 && Math.abs(media.getLength() - duration) > 3; + if (isLengthChanged) { + logger.info(getMarker(), "{} length: MC {} != NEXIO {}", title, media.getLength(), duration); + return true; + } + + return false; + } + + private boolean checkModifiedByTime(String title, Timestamp modified, int lastModifiedHours) { + //utolso modositas a mosthoz kepest + //The result of this method can be a negative period if the end is before the start. + long modifiedHours = Duration.between(modified.toInstant(), Instant.now()).toHours(); + if (modifiedHours < 0) { + logger.info(getMarker(), "{} modification time greather than now", title); + return false; + } + if (modifiedHours < lastModifiedHours) { + logger.debug(getMarker(), "{} modification time is too cloose for now", title); + return false; + } + + return true; + } + + @StepEntry + public Object[] execute(String sourceStoreName, String escortStoreName, int lastModifiedHours) throws Exception { + Controller controller = null; + + try { + getJobRuntime().setRelated(sourceStoreName + " archive"); + + StoreUri nexioStoreUri = getManager().getStoreUri(sourceStoreName, RemoteStoreProtocol.NEXIO); + StoreUri sourceStoreUri = getManager().getStoreUri(sourceStoreName, RemoteStoreProtocol.FTP); + StoreUri targetStoreUri = getManager().getStoreUri("TSM", RemoteStoreProtocol.TSM); + StoreUri escortStoreUri = getManager().getStoreUri(escortStoreName, RemoteStoreProtocol.LOCAL); + String outputPath = Paths.get(escortStoreUri.toString(true)).toString(); + + logger.info(getMarker(), "Connecting mediabase {}", nexioStoreUri.toString()); + if (nexioStoreUri.getPortNumber() == 0) + controller = new Controller(nexioStoreUri.getUri(), nexioStoreUri.getPortNumber()); + else + controller = new Controller(nexioStoreUri.getUri()); + controller.connect(); + Mediabase mediabase = controller.getMediabase(); + + Iterator clips = mediabase.getClips(); + logger.info(getMarker(), "Processing clips"); + + processClips(sourceStoreName, sourceStoreUri, targetStoreUri, outputPath, clips, lastModifiedHours); + } catch (Exception e) { + logger.error(getSessionMarker(), e.getMessage() == null ? "Unknown error" : e.getMessage()); + throw e; + } finally { + setProgress(100); + if (controller != null) + controller.disconnect(); + } + + return null; + } + + private void processClips(String storeName, StoreUri sourceStoreUri, StoreUri targetStoreUri, String outputPath, Iterator clips, + int lastModifiedHours) throws ClipNotFoundException, IOException, ProtocolException { + int processed = 0; + while (clips.hasNext()) { + + if (getJobRuntime().isWaitingCancel()) { + cancel(); + break; + } + + Clip clip = clips.next(); + + try { + String title = null; + Timestamp created = null; + Timestamp modified = null; + int frames = 0; + int videoFormat = 0; + frames = clip.getDuration(); + + //ures clip + if (frames == 1) + continue; + + title = clip.getXid().get(); + created = Timestamp.from(clip.getRecordDateTimestamp().toInstant()); + modified = Timestamp.from(clip.getModifiedTimestamp().toInstant()); + videoFormat = clip.getVideoFormat(); + + Media media = getManager().getMedia(title); + logger.debug(getMarker(), "Checking {}", title); + boolean createEscort = false; + + long mediaId = 0; + if (media == null) { + if (checkModifiedByTime(title, modified, lastModifiedHours)) { + createEscort = true; + logger.info(getMarker(), "{} not archived yet", title); + } + } else { + mediaId = media.getId(); + createEscort = check(modified, created, frames, media, videoFormat, lastModifiedHours); + } + + if (createEscort && !title.contains("*")) { + String fileName = title + DOT_MXF; + DownloadableMedia downloadable = DownloadableMedia.create(title, fileName, modified, created, frames, 0, sourceStoreUri.getId(), + targetStoreUri.getId(), mediaId); + String escortFileName = storeName + "." + downloadable.getString("fileName"); + if (EscortFiles.createMetadataIfNotExists(outputPath, escortFileName, downloadable.toPrettyString(""))) { + logger.info(getMarker(), "Archive status file created for {}", fileName); + } else { + logger.info(getMarker(), "Archive status file already exists for {}", fileName); + } + + } + + processed++; + if (processed % 100 == 0) + logger.info(getMarker(), "Processed {} records", processed); + } catch (Exception e) { + logger.error(getSessionMarker(), e.getMessage()); + continue; + } + } + } + +} diff --git a/server/-product/production/AMC/jobs/templates/PBARCHIVE-archive-checker.xml b/server/-product/production/AMC/jobs/templates/PBARCHIVE-archive-checker.xml new file mode 100644 index 00000000..54d36065 --- /dev/null +++ b/server/-product/production/AMC/jobs/templates/PBARCHIVE-archive-checker.xml @@ -0,0 +1,26 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/server/-product/production/AMC/jobs/templates/PBARCHIVE-validate-and-archive.xml b/server/-product/production/AMC/jobs/templates/PBARCHIVE-validate-and-archive.xml new file mode 100644 index 00000000..2b4a7d71 --- /dev/null +++ b/server/-product/production/AMC/jobs/templates/PBARCHIVE-validate-and-archive.xml @@ -0,0 +1,94 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/server/hu.user.mediacube.executors.tests/src/hu/user/mediacube/executors/tests/SupportMVDeletedFinishedShows.java b/server/hu.user.mediacube.executors.tests/src/hu/user/mediacube/executors/tests/SupportMVDeletedFinishedShows.java new file mode 100644 index 00000000..f8bcd866 --- /dev/null +++ b/server/hu.user.mediacube.executors.tests/src/hu/user/mediacube/executors/tests/SupportMVDeletedFinishedShows.java @@ -0,0 +1,72 @@ +package hu.user.mediacube.executors.tests; + +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.List; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.junit.AfterClass; +import org.junit.BeforeClass; +import org.junit.Test; + +import user.jobengine.db.IItemManager; +import user.jobengine.db.ItemManager; +import user.jobengine.db.Media; + +/*** + * MediaVivantis delete-materials.log-bol kepzett file vizsgalata + * pl. cat delete-materials-12-30-2020-1.log delete-materials-04-16-2021-1.log | grep 'killdate bejegyzés alapján sikeresen' > deleted_finished_shows.txt + * @author elgekko + */ +public class SupportMVDeletedFinishedShows { + Logger logger = LogManager.getLogger(); + + protected static IItemManager manager = null; + + @BeforeClass + static public void setUpConnection() throws Exception { + manager = new ItemManager(); + manager.connect(); + } + + @AfterClass + static public void tearDownConnection() throws Exception { + manager.disconnect(); + } + + @Test + public void checkDeletedFinishedShows() throws Exception { + // Store systemStore = manager.getSystemStore(false); + // logger.info(systemStore.getName()); + Path src = Paths.get("/opt/mv/deleted_finished_shows.txt"); + List lines = Files.readAllLines(src); + + int p = 0; + for (String line : lines) { + String mediaFileName = line.substring(62, line.indexOf(' ', 62)); + //logger.info(media); + + String sql = "select mediaid from mediafile where houseid=?"; + + manager.executeQuery(sql, rs -> { + long mediaid = rs.getLong(1); + //logger.info("{} {}", mediaFileName, mediaid); + Media media = manager.getMedia(mediaid); + if (media.getMediaFilesCount() != 2) { + logger.info("{} {}", mediaFileName, mediaid); + } + return true; + }, st -> { + st.setString(1, mediaFileName); + }); + p++; + + if (p % 100 == 0) + logger.info("Processed {}", p); + } + + logger.info("Processed {} lines", lines.size()); + } +} diff --git a/server/user.jobengine.executors/src/user/jobengine/server/steps/PBARCHIVETransferFromStep.java b/server/user.jobengine.executors/src/user/jobengine/server/steps/PBARCHIVETransferFromStep.java new file mode 100644 index 00000000..d47c2b79 --- /dev/null +++ b/server/user.jobengine.executors/src/user/jobengine/server/steps/PBARCHIVETransferFromStep.java @@ -0,0 +1,21 @@ +package user.jobengine.server.steps; + +import java.nio.file.Path; +import java.nio.file.Paths; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + +import user.commons.StoreUri; + +public class PBARCHIVETransferFromStep extends TransferStep { + private static final Logger logger = LogManager.getLogger(); + + @Override + @StepEntry + public Object[] execute(StoreUri sourceStoreUri, String sourceFileName, StoreUri targetStoreUri, String targetFileName) throws Exception { + Object[] result = null; + result = super.execute(sourceStoreUri, sourceFileName, targetStoreUri, targetFileName); + return result; + } +} diff --git a/server/user.jobengine.executors/src/user/jobengine/server/steps/PBArchiveCheckerStep.java b/server/user.jobengine.executors/src/user/jobengine/server/steps/PBArchiveCheckerStep.java new file mode 100644 index 00000000..071ed7cf --- /dev/null +++ b/server/user.jobengine.executors/src/user/jobengine/server/steps/PBArchiveCheckerStep.java @@ -0,0 +1,161 @@ +package user.jobengine.server.steps; + +import java.io.IOException; +import java.nio.file.Paths; +import java.sql.Timestamp; +import java.text.SimpleDateFormat; +import java.time.Duration; +import java.time.Instant; +import java.util.ArrayList; +import java.util.Iterator; +import java.util.List; + +import org.apache.commons.net.ftp.FTPClient; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.w3c.dom.stylesheets.LinkStyle; + +import user.commons.DownloadableMedia; +import user.commons.RemoteFile; +import user.commons.StoreUri; +import user.commons.nexio.api.Clip; +import user.commons.nexio.api.ClipNotFoundException; +import user.commons.nexio.api.Controller; +import user.commons.nexio.api.Mediabase; +import user.commons.nexio.server.protocol.ProtocolException; +import user.commons.remotestore.FtpDirectoryLister; +import user.commons.remotestore.IDirectoryLister; +import user.commons.remotestore.RemoteStoreProtocol; +import user.jobengine.db.Media; +import user.jobengine.db.MediaFile; +import user.jobengine.server.steps.shared.EscortFiles; + +public class PBArchiveCheckerStep extends JobStep { + private static final String DOT_MXF = ".mxf"; + private static final Logger logger = LogManager.getLogger(); + private SimpleDateFormat df = new SimpleDateFormat("yyyyMMdd-HHmmss"); + + + private boolean checkModifiedByTime(String title, Timestamp modified, int lastModifiedHours) { + long modifiedHours = Duration.between(modified.toInstant(), Instant.now()).toHours(); + if (modifiedHours < 0) { + logger.info(getMarker(), "{} modification time greather than now", title); + return false; + } + if (modifiedHours < lastModifiedHours) { + logger.debug(getMarker(), "{} modification time is too cloose for now", title); + return false; + } + + return true; + } + + @StepEntry + public Object[] execute(String sourceStoreName, String escortStoreName, int lastModifiedHours) throws Exception { + logger.info(getMarker(), "Started {}", sourceStoreName); + StoreUri sourceStoreUri = null; + try { + getJobRuntime().setRelated(sourceStoreName + " archive"); + + sourceStoreUri = getManager().getStoreUri(sourceStoreName, RemoteStoreProtocol.FTP); + + StoreUri targetStoreUri = getManager().getStoreUri("TSM", RemoteStoreProtocol.TSM); + StoreUri escortStoreUri = getManager().getStoreUri(escortStoreName, RemoteStoreProtocol.LOCAL); + String outputPath = Paths.get(escortStoreUri.toString(true)).toString(); + + FtpDirectoryLister lister = (FtpDirectoryLister) sourceStoreUri.getLister(); + FTPClient client = lister.connect(); + //a StoreUri alapu listazas nem mukodik, miota a working directory beallitas fix + List remoteFiles = new ArrayList<>(); + + String[] listNames = client.listNames(); + for (String f : listNames) { + logger.info(getMarker(), "File name is {}", f); + + try { + RemoteFile remoteFile = lister.get(f); + if (remoteFile == null) + logger.info(getMarker(), "Remote file for {} is not available", f); + else + remoteFiles.add(remoteFile); + + } catch (Exception iex){ + logger.error(getMarker(), iex.getMessage()); + } + } + + logger.info(getMarker(), "Found {} files in working dir {}", remoteFiles.size(), client.printWorkingDirectory()); + processClips(sourceStoreName, sourceStoreUri, targetStoreUri, outputPath, remoteFiles, lastModifiedHours); + + } catch (Exception e) { + logger.error(getMarker(), e.getMessage()); + logger.error(getSessionMarker(), e.getMessage() == null ? "Unknown error" : e.getMessage()); + throw e; + } finally { + if (sourceStoreUri != null) + sourceStoreUri.cleanUp(); + setProgress(100); + } + + return null; + } + + private void processClips(String storeName, StoreUri sourceStoreUri, StoreUri targetStoreUri, String outputPath, List remoteFiles, + int lastModifiedHours) throws ClipNotFoundException, IOException, ProtocolException { + int processed = 0; + for (RemoteFile remoteFile : remoteFiles) { + + if (getJobRuntime().isWaitingCancel()) { + //logger.info("CANCELED"); + cancel(); + break; + } + try { + + String title = null; + Timestamp created = null; + Timestamp modified = null; + title = remoteFile.getName(); + created = remoteFile.getCreated(); + modified = remoteFile.getModify(); + + logger.info(getMarker(), "{} with extension {} created {} modified {}", title, remoteFile.getExtension(), created, modified); + Media media = getManager().getMedia(title); + boolean createEscort = false; + + if (!createEscort) + continue; + + if (checkModifiedByTime(title, modified, lastModifiedHours)) { + createEscort = true; + logger.info(getMarker(), "{} not archived yet", title); + } + + long mediaId = 0; + if (media != null) + mediaId = media.getId(); + + if (createEscort) { + String fileName = title + DOT_MXF; + DownloadableMedia downloadable = DownloadableMedia.create(title, fileName, modified, created, 0, 0, sourceStoreUri.getId(), + targetStoreUri.getId(), mediaId); + String escortFileName = storeName + "." + downloadable.getString("fileName"); + if (EscortFiles.createMetadataIfNotExists(outputPath, escortFileName, downloadable.toPrettyString(""))) { + logger.info(getMarker(), "Archive status file created for {}", fileName); + } else { + logger.info(getMarker(), "Archive status file already exists for {}", fileName); + } + + } + + processed++; + if (processed % 100 == 0) + logger.info(getMarker(), "Processed {} records", processed); + } catch (Exception e) { + logger.error(getSessionMarker(), e.getMessage()); + continue; + } + } + } + +} diff --git a/server/user.jobengine.osgi.db/migrations/scripts/038_create_pbarchive_store.sql b/server/user.jobengine.osgi.db/migrations/scripts/038_create_pbarchive_store.sql new file mode 100644 index 00000000..8eb79b60 --- /dev/null +++ b/server/user.jobengine.osgi.db/migrations/scripts/038_create_pbarchive_store.sql @@ -0,0 +1,20 @@ +-- // Create VOD store +-- Migration SQL that makes the change goes here. + +INSERT INTO STORE (NAME, ISSYSTEM, ISLOWRES) VALUES ('PBARCHIVE', 'N', 'N') +@ + + + +-- @DELIMITER ; +INSERT INTO STOREURI(STOREID,PROTOCOL,DELIVERY,URI,ISSTREAM,ISSOURCE,ISTARGET,USERNAME,PASSWORD,ROOTPATH,PORTNUMBER) VALUES +((SELECT ID FROM STORE WHERE NAME='PBARCHIVE'),'FTP',null,'isilon.bcplay.local','N','Y','N','mcube','K6NV5aFemp','ARCHIVE',null) +; +-- @DELIMITER @ + +-- //@UNDO +-- SQL to undo the change goes here. +DELETE FROM STOREURI WHERE STOREID=(SELECT ID FROM STORE WHERE NAME='PBARCHIVE') +@ +DELETE FROM STORE WHERE NAME='PBARCHIVE' +@ -- 2.54.0