From 1b337c7ebec75cf6b2ea7f3bb0184d5c8dc62e85 Mon Sep 17 00:00:00 2001 From: =?utf8?q?V=C3=A1s=C3=A1ry=20D=C3=A1niel?= Date: Tue, 12 Dec 2017 11:23:01 +0000 Subject: [PATCH] git-tfs-id: [http://tfs.userrendszerhaz.hu:8080/tfs/DefaultCollection]$/MediaCube;C30802 --- .../run-mediacube-server-bsh.launch | 2 +- .../server/steps/ArchiveListBuilderStep.java | 4 +- .../CopyForArchiveNEXIOMaterialsStep.java | 65 +++++++++++++------ .../server/steps/MetadataTransformStep.java | 1 + .../jobengine/server/steps/TSMBackupStep.java | 6 +- .../server/steps/TSMRestoreStep.java | 6 +- .../nexio/NexioClipEventDispatcher.java | 12 ++-- .../jobengine/server/steps/ArchiveItem.java | 9 +++ .../{KillDateFile.java => EscortFiles.java} | 40 ++++++------ .../jobengine/server/steps/FileArchive.java | 28 ++++++++ .../server/steps/RundownArchive.java | 2 +- .../jobengine/server/steps/StoryArchive.java | 37 ++++++----- 12 files changed, 137 insertions(+), 75 deletions(-) rename server/user.jobengine.osgi.server/src/user/jobengine/server/steps/{KillDateFile.java => EscortFiles.java} (61%) create mode 100644 server/user.jobengine.osgi.server/src/user/jobengine/server/steps/FileArchive.java diff --git a/server/-configuration/run-mediacube-server-bsh.launch b/server/-configuration/run-mediacube-server-bsh.launch index f957593d..7b162214 100644 --- a/server/-configuration/run-mediacube-server-bsh.launch +++ b/server/-configuration/run-mediacube-server-bsh.launch @@ -19,7 +19,7 @@ - + diff --git a/server/user.jobengine.executors/src/user/jobengine/server/steps/ArchiveListBuilderStep.java b/server/user.jobengine.executors/src/user/jobengine/server/steps/ArchiveListBuilderStep.java index 0f5e7f11..8f68e6fa 100644 --- a/server/user.jobengine.executors/src/user/jobengine/server/steps/ArchiveListBuilderStep.java +++ b/server/user.jobengine.executors/src/user/jobengine/server/steps/ArchiveListBuilderStep.java @@ -16,6 +16,7 @@ import org.apache.logging.log4j.Logger; import com.ibm.nosql.json.JSONUtil; import com.ibm.nosql.json.api.BasicDBObject; +import user.commons.nosql.NoSQLUtils; import user.jobengine.server.IJobEngine; import user.jobengine.server.IJobRuntime; @@ -39,6 +40,7 @@ public class ArchiveListBuilderStep extends JobStep { public static final String MEDIA_HOUSEID = "mediaHouseId"; public static final String MEDIA_TITLE = "mediaTitle"; public static final String MEDIA_DESCRIPTION = "mediaDescription"; + private static final String DURATION = "duration"; private ArchiveItem createArchiveItem(Path jsonFilePath, Path mediaFilePath, Path catchedFilePath) { ArchiveItem result = null; @@ -57,7 +59,7 @@ public class ArchiveListBuilderStep extends JobStep { result.setMediaDescription(getMetadata(dbObject, MEDIA_DESCRIPTION)); result.setMediaFile(mediaFilePath.toString()); result.setCatchedFile(catchedFilePath.toString()); - + result.setDuration(NoSQLUtils.asLong(dbObject, DURATION)); } catch (IOException e) { logger.catching(e); return null; diff --git a/server/user.jobengine.executors/src/user/jobengine/server/steps/CopyForArchiveNEXIOMaterialsStep.java b/server/user.jobengine.executors/src/user/jobengine/server/steps/CopyForArchiveNEXIOMaterialsStep.java index ec5b12bb..3570e102 100644 --- a/server/user.jobengine.executors/src/user/jobengine/server/steps/CopyForArchiveNEXIOMaterialsStep.java +++ b/server/user.jobengine.executors/src/user/jobengine/server/steps/CopyForArchiveNEXIOMaterialsStep.java @@ -1,6 +1,7 @@ package user.jobengine.server.steps; import java.io.IOException; +import java.io.OutputStream; import java.net.InetAddress; import java.net.URI; import java.net.URISyntaxException; @@ -35,6 +36,9 @@ import user.jobengine.server.JobEngineException; import user.jobengine.server.steps.MetadataTypeDetector.MetadataType; public class CopyForArchiveNEXIOMaterialsStep extends JobStep { + private static final String UTF_8 = "utf-8"; + private static final String JSON_EXT = ".json"; + private static final String DURATION = "duration"; private static final String MXFEXT = ".MXF"; private static final Logger logger = LogManager.getLogger(); private static final String NEXIOCLIPS = "nexioclips"; @@ -66,23 +70,32 @@ public class CopyForArchiveNEXIOMaterialsStep extends JobStep { private void copy(RundownArchive rundownArchive) throws Exception { for (StoryArchive storyArchive : rundownArchive.getStoryArchives()) { - for (String fileName : storyArchive.getFileNames()) { - copyFile(fileName, rundownArchive, storyArchive); + for (FileArchive fileArchive : storyArchive.getFileArchives()) { + copyFile(fileArchive, rundownArchive, storyArchive); } } } - private void copyFile(String fileName, RundownArchive rundownArchive, StoryArchive storyArchive) throws Exception { + private void copyFile(FileArchive fileArchive, RundownArchive rundownArchive, StoryArchive storyArchive) throws Exception { FTPClient source = ((FtpDirectoryLister) RemoteFileHandler.createLister(sourceUri)).connect(); - FTPClient target = ((FtpDirectoryLister) RemoteFileHandler.createLister(sourceUri)).connect(); + FTPClient target = ((FtpDirectoryLister) RemoteFileHandler.createLister(targetUri)).connect(); + String fileName = fileArchive.getFileName(); transferFile(source, target, fileName); - BasicDBObject metadata = createMetadata(rundownArchive, storyArchive); - transferMetadata(target, metadata); - createTargetKillDateFile(target, fileName); + BasicDBObject metadata = createMetadata(rundownArchive, storyArchive, fileArchive); + try { + if (!target.changeWorkingDirectory(EscortFiles.STATUSFOLDER)) + target.makeDirectory(EscortFiles.STATUSFOLDER); + if (!target.changeWorkingDirectory(EscortFiles.STATUSFOLDER)) + throw new Exception("!STATUSFOLDER"); + } catch (Exception e) { + logger.catching(e); + throw e; + } + transferMetadata(target, fileName, metadata); createSourceKillDateFile(source, fileName); } - private BasicDBObject createMetadata(RundownArchive rundownArchive, StoryArchive storyArchive) { + private BasicDBObject createMetadata(RundownArchive rundownArchive, StoryArchive storyArchive, FileArchive fileArchive) { // { // "itemHouseId": "43", // "itemTitle": "Hazahúzó", @@ -102,15 +115,14 @@ public class CopyForArchiveNEXIOMaterialsStep extends JobStep { result.put("mediaHouseId", storyArchive.getMediaHouseId()); result.put("mediaTitle", storyArchive.getMediaTitle()); result.put("mediaDescription", storyArchive.getMediaDesc()); + + result.put("duration", fileArchive.getDuration()); return result; } private void createSourceKillDateFile(FTPClient source, String fileName) { } - private void createTargetKillDateFile(FTPClient target, String fileName) { - } - @StepEntry public Object[] execute(int nexioPort, String nexioUserName, String nexioPassword, String archiveFtp, String archiveUserName, String archivePassword, int daysBeforeNow, IJobEngine jobEngine, IJobRuntime jobRuntime) throws Exception { @@ -143,21 +155,22 @@ public class CopyForArchiveNEXIOMaterialsStep extends JobStep { return null; } - private String processMosObject(BasicDBObject rundown, BasicDBObject story, BasicDBObject mosObject) throws Exception { + private FileArchive processMosObject(BasicDBObject rundown, BasicDBObject story, BasicDBObject mosObject) throws Exception { String mosID = mosObject.getString(IOctopusAPI.OBJ_ID); if (MetadataTypeDetector.GuessMetadataType(mosID) != MetadataType.OCTOPUSPLACEHOLDER) { logger.warn("Skipping MOS object {}", mosID); return null; } DBCollection clips = db.getCollection(NEXIOCLIPS); - DBObject clip = clips.findOne(new BasicDBObject(LONGNAMEID, mosID)); + BasicDBObject clip = (BasicDBObject) clips.findOne(new BasicDBObject(LONGNAMEID, mosID)); if (clip == null) { logger.info("File NOT exists {}", mosID); - throw new Exception(String.format("File NOT exists %s")); + throw new Exception(String.format("File NOT exists %s", mosID)); } else { logger.info("File exists {}", mosID); } - return mosID + MXFEXT; + long duration = NoSQLUtils.asLong(clip, DURATION); + return new FileArchive(mosID + MXFEXT, duration); } private RundownArchive processRundow(DBObject r) throws Exception { @@ -210,18 +223,19 @@ public class CopyForArchiveNEXIOMaterialsStep extends JobStep { return null; StoryArchive storyArchive = null; for (BasicDBObject mosObject : mosObjects) { - String fileName = processMosObject(rundown, story, mosObject); - if (fileName == null) + FileArchive fileArchive = processMosObject(rundown, story, mosObject); + if (fileArchive == null) continue; if (storyArchive == null) { storyArchive = new StoryArchive(); storyArchive.setMediaHouseId(parentStoryID); storyArchive.setMediaTitle(story.getString(IOctopusAPI.NAME)); storyArchive.setMediaDesc(story.getString(IOctopusAPI.SCRIPT_CONTENT)); + } - storyArchive.addFile(fileName); + storyArchive.addFileArchive(fileArchive); } - return null; + return storyArchive; } @@ -278,10 +292,11 @@ public class CopyForArchiveNEXIOMaterialsStep extends JobStep { if (!source.setFileType(FTP.BINARY_FILE_TYPE)) throw new Exception("!SOURCE TYPE"); + source.retr(fileName); + if (!target.setFileType(FTP.BINARY_FILE_TYPE)) throw new Exception("!TARGET TYPE"); - source.retr(fileName); target.stor(fileName); while (true) { @@ -291,10 +306,18 @@ public class CopyForArchiveNEXIOMaterialsStep extends JobStep { String replyText = source.getReplyString(); if ("Transaction completed".equals(replyText)) break; + Thread.sleep(500); } } - private void transferMetadata(FTPClient target, BasicDBObject metadata) { + private void transferMetadata(FTPClient target, String fileName, BasicDBObject metadata) throws Exception { + try (OutputStream outStream = target.storeFileStream(fileName + JSON_EXT)) { + outStream.write(metadata.toString().getBytes(UTF_8)); + outStream.flush(); + } catch (Exception e) { + logger.catching(e); + throw e; + } } } diff --git a/server/user.jobengine.executors/src/user/jobengine/server/steps/MetadataTransformStep.java b/server/user.jobengine.executors/src/user/jobengine/server/steps/MetadataTransformStep.java index 29dfa805..0a0c9ac0 100644 --- a/server/user.jobengine.executors/src/user/jobengine/server/steps/MetadataTransformStep.java +++ b/server/user.jobengine.executors/src/user/jobengine/server/steps/MetadataTransformStep.java @@ -35,6 +35,7 @@ public class MetadataTransformStep extends JobStep { archiveItem.getItemHouseId()); mediaCubeMedia = itemManager.createMedia(ITEMTYPENAME, archiveItem.getMediaTitle(), archiveItem.getMediaDescription(), archiveItem.getMediaHouseId()); + mediaCubeMedia.setLength(archiveItem.getDuration()); mediaCubeItem.appendMedia(mediaCubeMedia); itemManager.mergeItemStructure(mediaCubeItem); diff --git a/server/user.jobengine.executors/src/user/jobengine/server/steps/TSMBackupStep.java b/server/user.jobengine.executors/src/user/jobengine/server/steps/TSMBackupStep.java index 44a827a1..64e2384c 100644 --- a/server/user.jobengine.executors/src/user/jobengine/server/steps/TSMBackupStep.java +++ b/server/user.jobengine.executors/src/user/jobengine/server/steps/TSMBackupStep.java @@ -61,10 +61,8 @@ public class TSMBackupStep extends JobStep { saveMetadata(mediaCubeMedia, sourceFileName); logger.info(getMarker(), "Az '{}' állomány archiválása sikeres.", sourceMediaFile.getName()); - if (killDateDays > 0) { - KillDateFile killDateFile = new KillDateFile(sourceMediaFile.getParent().toString(), sourceMediaFile.getName().toString()); - killDateFile.create(killDateDays, getMarker()); - } + if (killDateDays > 0) + EscortFiles.createUNCKillDate(sourceMediaFile.getParent().toString(), sourceMediaFile.getName().toString(), killDateDays, getMarker()); } catch (Exception e) { logger.catching(e); logger.error(getMarker(), "Az '{}' állomány archiválása sikertelen. A rendszer hibaüzenete: {}", sourceMediaFile.getName(), e.getMessage()); diff --git a/server/user.jobengine.executors/src/user/jobengine/server/steps/TSMRestoreStep.java b/server/user.jobengine.executors/src/user/jobengine/server/steps/TSMRestoreStep.java index e41cbe62..7ce84421 100644 --- a/server/user.jobengine.executors/src/user/jobengine/server/steps/TSMRestoreStep.java +++ b/server/user.jobengine.executors/src/user/jobengine/server/steps/TSMRestoreStep.java @@ -56,10 +56,8 @@ public class TSMRestoreStep extends JobStep { logger.info(new MediaCubeMarker(successRecipient), successMessage, sourceFileName, targetPath, targetFileName); logger.info(getMarker(), successMessage, sourceFileName, targetPath, targetFileName); - if (killDateDays > 0) { - KillDateFile killDateFile = new KillDateFile(targetPath, targetFileName); - killDateFile.create(killDateDays, getMarker()); - } + if (killDateDays > 0) + EscortFiles.createUNCKillDate(targetPath, targetFileName, killDateDays, getMarker()); } catch (Exception e) { logger.error(getMarker(), "Az '{}' állomány visszatöltése sikertelen. A rendszer hibaüzenete: {}", sourceFileName, e.getMessage()); diff --git a/server/user.jobengine.osgi.commons/src/user/commons/nexio/NexioClipEventDispatcher.java b/server/user.jobengine.osgi.commons/src/user/commons/nexio/NexioClipEventDispatcher.java index 05063f39..1f217724 100644 --- a/server/user.jobengine.osgi.commons/src/user/commons/nexio/NexioClipEventDispatcher.java +++ b/server/user.jobengine.osgi.commons/src/user/commons/nexio/NexioClipEventDispatcher.java @@ -287,14 +287,14 @@ public class NexioClipEventDispatcher implements ClipEventListener { } /* - + int c = connection.read(buffer, 0, 2); //2 bytes hosszan az idokozben hozzaadott id-k szamossaga if (c < 2) { throw new ProtocolException("c, 2, 2"); } //MSB, LSB - + return nofIds; */ return ret; @@ -411,8 +411,8 @@ public class NexioClipEventDispatcher implements ClipEventListener { logger.info("NEXIO server connected"); this.progressListenerList = new EventListenerList(); - this.dropAllClipsFromMongo(); - this.ensureIndexes(); + // this.dropAllClipsFromMongo(); + // this.ensureIndexes(); boolean[] useMOSGateway = { false }; try { useMOSGateway[0] = Boolean.parseBoolean(System.getProperty(NEXIO_USE_MOS_GATEWAY)); @@ -421,8 +421,8 @@ public class NexioClipEventDispatcher implements ClipEventListener { executor.submit(() -> { try { if (useMOSGateway[0]) { - NexioDataMiner dm = new NexioDataMiner(); - dm.transferClips(); + // NexioDataMiner dm = new NexioDataMiner(); + // dm.transferClips(); } else { copyClipsIntoMongo(); } diff --git a/server/user.jobengine.osgi.server/src/user/jobengine/server/steps/ArchiveItem.java b/server/user.jobengine.osgi.server/src/user/jobengine/server/steps/ArchiveItem.java index 5b807c62..883e32c6 100644 --- a/server/user.jobengine.osgi.server/src/user/jobengine/server/steps/ArchiveItem.java +++ b/server/user.jobengine.osgi.server/src/user/jobengine/server/steps/ArchiveItem.java @@ -13,11 +13,16 @@ public class ArchiveItem implements Serializable { private String mediaDescription; private String mediaFile = null; private String catchedFile = null; + private long duration; public String getCatchedFile() { return catchedFile; } + public long getDuration() { + return duration; + } + public String getItemDescription() { return itemDescription; } @@ -60,6 +65,10 @@ public class ArchiveItem implements Serializable { this.catchedFile = catchedFile; } + public void setDuration(long duration) { + this.duration = duration; + } + public void setItemDescription(String itemDescription) { this.itemDescription = itemDescription; } diff --git a/server/user.jobengine.osgi.server/src/user/jobengine/server/steps/KillDateFile.java b/server/user.jobengine.osgi.server/src/user/jobengine/server/steps/EscortFiles.java similarity index 61% rename from server/user.jobengine.osgi.server/src/user/jobengine/server/steps/KillDateFile.java rename to server/user.jobengine.osgi.server/src/user/jobengine/server/steps/EscortFiles.java index 76d46eba..5bea5ed4 100644 --- a/server/user.jobengine.osgi.server/src/user/jobengine/server/steps/KillDateFile.java +++ b/server/user.jobengine.osgi.server/src/user/jobengine/server/steps/EscortFiles.java @@ -16,35 +16,37 @@ import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.apache.logging.log4j.Marker; -public class KillDateFile { +public class EscortFiles { private static final Logger logger = LogManager.getLogger(); private static final String KILLDATE_NAME = "%s.%s.killdate"; - private static final String YYYY_M_MDD = "yyyyMMdd"; - private static final String STATUSFOLDER = ".STATUS"; - private String filePath; - private String fileName; + private static final String YYYYMMDD = "yyyyMMdd"; + public static final String STATUSFOLDER = ".STATUS"; - public KillDateFile(String filePath, String fileName) { - this.filePath = filePath; - this.fileName = fileName; + private static String composeKillDateFileName(String fileName, int days) { + Calendar killDate = Calendar.getInstance(); + killDate.add(Calendar.DAY_OF_YEAR, days); + SimpleDateFormat dateFormat = new SimpleDateFormat(YYYYMMDD); + return String.format(KILLDATE_NAME, fileName, dateFormat.format(killDate.getTime())); + } + + public static void createUNCKillDate(String filePath, String fileName, int days, Marker marker) throws IOException { + ensureUNCFolder(filePath, STATUSFOLDER, marker); + String killDateFileName = composeKillDateFileName(fileName, days); + Path killDatePath = Paths.get(filePath, STATUSFOLDER, killDateFileName); + if (Files.exists(killDatePath)) + logger.warn(marker, "Az '{}' állomány már létezik.", killDatePath); + else + Files.createFile(killDatePath); } - public void create(int days, Marker marker) throws IOException { - Path statusPath = Paths.get(filePath, STATUSFOLDER); + public static void ensureUNCFolder(String filePath, String folderName, Marker marker) throws IOException { + Path statusPath = Paths.get(filePath, folderName); File statusFolder = statusPath.toFile(); if (!statusFolder.exists() || !statusFolder.isDirectory()) { Set perms = PosixFilePermissions.fromString("rwxrwxrwx"); FileAttribute> attr = PosixFilePermissions.asFileAttribute(perms); Files.createDirectory(statusPath, attr); } - Calendar killDate = Calendar.getInstance(); - killDate.add(Calendar.DAY_OF_YEAR, days); - SimpleDateFormat dateFormat = new SimpleDateFormat(YYYY_M_MDD); - String killDateFileName = String.format(KILLDATE_NAME, fileName, dateFormat.format(killDate.getTime())); - Path killDatePath = Paths.get(filePath, STATUSFOLDER, killDateFileName); - if (Files.exists(killDatePath)) - logger.warn(marker, "Az '{}' állomány már létezik.", killDatePath); - else - Files.createFile(killDatePath); } + } diff --git a/server/user.jobengine.osgi.server/src/user/jobengine/server/steps/FileArchive.java b/server/user.jobengine.osgi.server/src/user/jobengine/server/steps/FileArchive.java new file mode 100644 index 00000000..b8f518d5 --- /dev/null +++ b/server/user.jobengine.osgi.server/src/user/jobengine/server/steps/FileArchive.java @@ -0,0 +1,28 @@ +package user.jobengine.server.steps; + +public class FileArchive { + private String fileName; + private long duration; + + public FileArchive(String fileName, long duration) { + this.fileName = fileName; + this.duration = duration; + } + + public long getDuration() { + return duration; + } + + public String getFileName() { + return fileName; + } + + public void setDuration(long duration) { + this.duration = duration; + } + + public void setFileName(String fileName) { + this.fileName = fileName; + } + +} \ No newline at end of file diff --git a/server/user.jobengine.osgi.server/src/user/jobengine/server/steps/RundownArchive.java b/server/user.jobengine.osgi.server/src/user/jobengine/server/steps/RundownArchive.java index 8623c393..5328cef3 100644 --- a/server/user.jobengine.osgi.server/src/user/jobengine/server/steps/RundownArchive.java +++ b/server/user.jobengine.osgi.server/src/user/jobengine/server/steps/RundownArchive.java @@ -3,7 +3,7 @@ package user.jobengine.server.steps; import java.util.ArrayList; import java.util.List; -class RundownArchive { +public class RundownArchive { private String itemHouseId; private String itemTitle; private String itemDesc; diff --git a/server/user.jobengine.osgi.server/src/user/jobengine/server/steps/StoryArchive.java b/server/user.jobengine.osgi.server/src/user/jobengine/server/steps/StoryArchive.java index 180b91c5..2eb5a29a 100644 --- a/server/user.jobengine.osgi.server/src/user/jobengine/server/steps/StoryArchive.java +++ b/server/user.jobengine.osgi.server/src/user/jobengine/server/steps/StoryArchive.java @@ -3,43 +3,44 @@ package user.jobengine.server.steps; import java.util.ArrayList; import java.util.List; -class StoryArchive { +public class StoryArchive { private String mediaHouseId; private String mediaTitle; private String mediaDesc; - private List fileNames; + private List fileArchives; - public void addFile(String fileName) { - if (fileNames == null) - fileNames = new ArrayList<>(); - fileNames.add(fileName); + public void addFileArchive(FileArchive fileArchive) { + if (fileArchives == null) + fileArchives = new ArrayList<>(); + fileArchives.add(fileArchive); } - public List getFileNames() { - return fileNames; + public List getFileArchives() { + return fileArchives; } - public String getMediaHouseId() { - return mediaHouseId; + public String getMediaDesc() { + return mediaDesc; } - public void setMediaHouseId(String mediaHouseId) { - this.mediaHouseId = mediaHouseId; + public String getMediaHouseId() { + return mediaHouseId; } public String getMediaTitle() { return mediaTitle; } - public void setMediaTitle(String mediaTitle) { - this.mediaTitle = mediaTitle; + public void setMediaDesc(String mediaDesc) { + this.mediaDesc = mediaDesc; } - public String getMediaDesc() { - return mediaDesc; + public void setMediaHouseId(String mediaHouseId) { + this.mediaHouseId = mediaHouseId; } - public void setMediaDesc(String mediaDesc) { - this.mediaDesc = mediaDesc; + public void setMediaTitle(String mediaTitle) { + this.mediaTitle = mediaTitle; } + } \ No newline at end of file -- 2.54.0