From 9db52810f225495d5a7bb29c6169cbe74c44e346 Mon Sep 17 00:00:00 2001 From: =?utf8?q?V=C3=A1s=C3=A1ry=20D=C3=A1niel?= Date: Fri, 18 Jan 2019 13:15:59 +0000 Subject: [PATCH] =?utf8?q?#120=20JSON=20k=C3=BCld=C3=A9se=20a=20visszajelz?= =?utf8?q?=C5=91=20email=20t=C3=B6rzs=C3=A9ben?= MIME-Version: 1.0 Content-Type: text/plain; charset=utf8 Content-Transfer-Encoding: 8bit git-tfs-id: [http://tfs.userrendszerhaz.hu:8080/tfs/DefaultCollection]$/MediaCube;C31388 --- .../Configuration/configuration - Copy.json | 2 +- .../Maestro/Configuration/configuration.json | 69 ++++++++++--------- client/Maestro/MaestroForm.Target.cs | 19 +++-- .../MaestroShared/Commons/PatternNameMaker.cs | 14 +++- .../Targets/UNCTargetProcessor.cs | 3 +- client/MediaCubeClient/MediaCubeApi.cs | 2 +- .../mediacube/executors/tests/Support.java | 2 + .../server/steps/BatchRetrieveForkStep.java | 5 ++ .../CopyForArchiveNEXIOMaterialsStep.java | 29 +++++--- .../server/steps/TSMRestoreStep.java | 6 +- .../src/user/jobengine/server/JobRuntime.java | 5 +- 11 files changed, 96 insertions(+), 60 deletions(-) diff --git a/client/Maestro/Configuration/configuration - Copy.json b/client/Maestro/Configuration/configuration - Copy.json index ad4c3fb1..000156f8 100644 --- a/client/Maestro/Configuration/configuration - Copy.json +++ b/client/Maestro/Configuration/configuration - Copy.json @@ -1,6 +1,6 @@ { "title": "Development", - "active": true, + "active": false, "startInTray": false, "enableCustomMetadataId": true, "filter": "avi", diff --git a/client/Maestro/Configuration/configuration.json b/client/Maestro/Configuration/configuration.json index cf268ea6..b26d0065 100644 --- a/client/Maestro/Configuration/configuration.json +++ b/client/Maestro/Configuration/configuration.json @@ -44,25 +44,25 @@ "version": 0, "redefineSegments": true }, - { - "$type": "MediaCubeMetadata", - "server": { - "address": "http://10.10.1.27/services/rest/jobengine", - "userName": "mediacube", - "password": "Dn8t4gfHcK98o8hyPgLDhr5SgSji4JCxsfpMJsODikUp3nXgrM0UNCi45lLAK8ZOnmEneO44P9qpJ4QDqhctN6MxZodjJgdZTyoZKmSa+ECzEzLr/wPYNgxVaXrVotEy", - "timeout": 1000 - }, - "wsserver": { - "address": "ws://10.10.1.29:88/services/wsapi", - "timeout": 1000 - }, - "jobTemplate": "retrieve-material.xml", - "archiveFolder": "file://10.10.1.105/TESZT/TC/ARCHIVE", - "restoreFolder": "file://10.10.1.105/BRAAVOS/ARCHIVE_RESTORE", - "restoreNamePattern": "%s_%GUID%", - "serverRestoreFolder": "/mnt/ISILON/ARCHIVE_RESTORE", - "killDateDays": 1 - } +{ + "$type": "MediaCubeMetadata", + "server": { + "address": "http://localhost:8888/services/rest/jobengine", + "userName": "mediacube", + "password": "Dn8t4gfHcK98o8hyPgLDhr5SgSji4JCxsfpMJsODikUp3nXgrM0UNCi45lLAK8ZOnmEneO44P9qpJ4QDqhctN6MxZodjJgdZTyoZKmSa+ECzEzLr/wPYNgxVaXrVotEy", + "timeout": 1000 + }, + "wsserver": { + "address": "ws://10.10.1.27/services/wsapi", + "timeout": 1000 + }, + "jobTemplate": "retrieve-material.xml", + "archiveFolder": "file://10.10.1.105/TESZT/TC/ARCHIVE", + "restoreFolder": "file://10.10.1.105/BRAAVOS/ARCHIVE_RESTORE", + "restoreNamePattern": "%s_%GUID%", + "serverRestoreFolder": "/mnt/ISILON/ARCHIVE_RESTORE", + "killDateDays": 1 +} ], "targets": [ { @@ -101,20 +101,23 @@ "timeout": 1000 } }, - { - "label": "Teszt", - "processor": "FTPTargetProcessor", - "outputFormat": "%ID%-%SOURCENAME%", - "tag": "Betöltés", - "killDateDays": 7, - "saveArchiveMetadata": true, - "remote": { - "address": "ftp://10.10.1.105/TESZT/CHECK", - "userName": "mediacube", - "password": "Dn8t4gfHcK98o8hyPgLDhr5SgSji4JCxsfpMJsODikUp3nXgrM0UNCi45lLAK8ZOnmEneO44P9qpJ4QDqhctN6MxZodjJgdZTyoZKmSa+ECzEzLr/wPYNgxVaXrVotEy", - "timeout": 1000 - } - }, + { + "label": "Teszt", + "processor": "FTPTargetProcessor", + "outputFormat": "%ID%-%SOURCENAME%", + "tag": "Betöltés", + "killDateDays": 7, + "saveArchiveMetadata": false, + "sendEmailOnSuccess": true, + "successEmailPattern": "A %ID% - %ITEMTITLE% anyag felkerült a szerverre.\r\nJSON: %JSON%", + "successEmailRecipient": "vasary@elgekko.net", + "remote": { + "address": "ftp://10.10.1.105/TESZT/CHECK", + "userName": "mediacube", + "password": "Dn8t4gfHcK98o8hyPgLDhr5SgSji4JCxsfpMJsODikUp3nXgrM0UNCi45lLAK8ZOnmEneO44P9qpJ4QDqhctN6MxZodjJgdZTyoZKmSa+ECzEzLr/wPYNgxVaXrVotEy", + "timeout": 1000 + } + }, { "label": "Napi megtekintőbe", "processor": "FTPTargetProcessor", diff --git a/client/Maestro/MaestroForm.Target.cs b/client/Maestro/MaestroForm.Target.cs index 7641a9ea..b2df67a9 100644 --- a/client/Maestro/MaestroForm.Target.cs +++ b/client/Maestro/MaestroForm.Target.cs @@ -187,21 +187,26 @@ namespace Maestro { result.MetadataText = SelectedMetadata?.MetadataText; result.InputFileName = sourceItem.Name; result.MovieSegments = MovieSegments?.ToList(); - result.ArchiveMetadata = ArchiveMetadata.DeepClone(ArchiveMetadata); result.VariantID = selectedMetadata == null ? 0 : selectedMetadata.VariantID; result.MetadataKind = selectedMetadata == null ? MetadataType.None : selectedMetadata.Kind; result.UserName = TrayApplicationContext.UserName; result.MessageBus = MessageBus; result.CreateDate = createDate; + //TODO kell duplan? result.Frames = sourceItem.Frames; result.TargetOrder = Configuration.Targets.ToList().IndexOf(target); - result.ArchiveMetadata.userName = result.UserName; result.SelectedSegments = SelectedMetadata?.Selected; - if (SelectedMetadata.Kind == MetadataType.MediaCube && result.ArchiveMetadata != null) { - result.ArchiveMetadata.itemHouseId = PatternNameMaker.Get(result.ArchiveMetadata.itemHouseId, result.ID, result.InputFileName, null, null, result.MetadataText); - result.ArchiveMetadata.itemTitle = PatternNameMaker.Get(result.ArchiveMetadata.itemTitle, result.ID, result.InputFileName, null, null, result.MetadataText); - result.ArchiveMetadata.mediaHouseId = PatternNameMaker.Get(result.ArchiveMetadata.mediaHouseId, result.ID, result.InputFileName, null, null, result.MetadataText); - result.ArchiveMetadata.mediaTitle = PatternNameMaker.Get(result.ArchiveMetadata.mediaTitle, result.ID, result.InputFileName, null, null, result.MetadataText); + + if (ArchiveMetadata != null) { + result.ArchiveMetadata = ArchiveMetadata.DeepClone(ArchiveMetadata); + result.ArchiveMetadata.duration = sourceItem.Frames; + result.ArchiveMetadata.userName = result.UserName; + if (SelectedMetadata.Kind == MetadataType.MediaCube) { + result.ArchiveMetadata.itemHouseId = PatternNameMaker.Get(result.ArchiveMetadata.itemHouseId, result.ID, result.InputFileName, null, null, result.MetadataText); + result.ArchiveMetadata.itemTitle = PatternNameMaker.Get(result.ArchiveMetadata.itemTitle, result.ID, result.InputFileName, null, null, result.MetadataText); + result.ArchiveMetadata.mediaHouseId = PatternNameMaker.Get(result.ArchiveMetadata.mediaHouseId, result.ID, result.InputFileName, null, null, result.MetadataText); + result.ArchiveMetadata.mediaTitle = PatternNameMaker.Get(result.ArchiveMetadata.mediaTitle, result.ID, result.InputFileName, null, null, result.MetadataText); + } } return result; } diff --git a/client/MaestroShared/Commons/PatternNameMaker.cs b/client/MaestroShared/Commons/PatternNameMaker.cs index 6cdeaa35..60dbd5a6 100644 --- a/client/MaestroShared/Commons/PatternNameMaker.cs +++ b/client/MaestroShared/Commons/PatternNameMaker.cs @@ -12,6 +12,9 @@ namespace MaestroShared.Commons { private const string PATTERN_USERNAME = "%USERNAME%"; private const string PATTERN_COMPUTERNAME = "%COMPUTERNAME%"; private const string PATTERN_ID = "%ID%"; + private const string PATTERN_JSON = "%JSON%"; + private const string PATTERN_ITEMTITLE = "%ITEMTITLE%"; + private const string PATTERN_MEDIATITLE = "%MEDIATITLE%"; private const string PATTERN_IDROOT = "%IDROOT%"; private const string PATTERN_GUID = "%GUID%"; private const string PATTERN_SOURCENAME = "%SOURCENAME%"; @@ -41,15 +44,22 @@ namespace MaestroShared.Commons { return result; } - static public string Get(string pattern, string id, string inputName, string outputName, string userName, string text, DateTime? storedDateTime = null) { + static public string Get(string pattern, string id, string inputName, string outputName, string userName, string text, DateTime? storedDateTime = null, string json = null, string itemTitle = null, string mediaTitle = null) { string idRoot = id != null && id.Contains(UNDERSCORE) ? id.Split(UNDERSCORE[0])[0] : id; DateTime dt = storedDateTime == null ? DateTime.Now : (DateTime)storedDateTime; + var trimmedJSON = json; + if (json != null) { + trimmedJSON = json.Replace("\r", "").Replace("\n", ""); + } string result = pattern .Replace(PATTERN_ID, id) .Replace(PATTERN_IDROOT, idRoot) .Replace(PATTERN_GUID, ((ShortGuid)Guid.NewGuid()).ToString()) .Replace(PATTERN_TIMESTAMP, dt.ToString(DATETIME_FORMAT, CultureInfo.InvariantCulture)) - .Replace(PATTERN_DATESTAMP, dt.ToString(DATE_FORMAT_NODOTS, CultureInfo.InvariantCulture)); + .Replace(PATTERN_DATESTAMP, dt.ToString(DATE_FORMAT_NODOTS, CultureInfo.InvariantCulture)) + .Replace(PATTERN_JSON, trimmedJSON) + .Replace(PATTERN_ITEMTITLE, itemTitle) + .Replace(PATTERN_MEDIATITLE, mediaTitle); if (!String.IsNullOrEmpty(inputName)) { result = result.Replace(PATTERN_SOURCENAME, Normalize(Path.GetFileNameWithoutExtension(inputName))); diff --git a/client/MaestroShared/Targets/UNCTargetProcessor.cs b/client/MaestroShared/Targets/UNCTargetProcessor.cs index bf3d6510..dfe3609e 100644 --- a/client/MaestroShared/Targets/UNCTargetProcessor.cs +++ b/client/MaestroShared/Targets/UNCTargetProcessor.cs @@ -374,7 +374,6 @@ namespace MaestroShared.Targets { EnsureDirectoryExistence(statusWorkDir); string fileName = String.Format(METADATA_FILE, OutputName); //logger.Debug("Creating METADATA file {0}", fileName); - Parameters.ArchiveMetadata.duration = Parameters.Frames; byte[] content = Encoding.UTF8.GetBytes(Parameters.ArchiveMetadata.ToString()); string ouputPath = GetOutputFilePath(statusWorkDir, fileName); UploadContent(ouputPath, content); @@ -420,7 +419,7 @@ namespace MaestroShared.Targets { } protected string GetDynamicText(string pattern) { - return PatternNameMaker.Get(pattern, ID, InputName, Output, Parameters.UserName, Parameters.MetadataText, Parameters.CreateDate); + return PatternNameMaker.Get(pattern, ID, InputName, Output, Parameters.UserName, Parameters.MetadataText, Parameters.CreateDate, Parameters.ArchiveMetadata?.ToString(), Parameters.ArchiveMetadata?.itemTitle, Parameters.ArchiveMetadata?.mediaTitle); } private string CreateOutputFileName() { diff --git a/client/MediaCubeClient/MediaCubeApi.cs b/client/MediaCubeClient/MediaCubeApi.cs index 92b005ec..f2614a45 100644 --- a/client/MediaCubeClient/MediaCubeApi.cs +++ b/client/MediaCubeClient/MediaCubeApi.cs @@ -69,7 +69,7 @@ namespace MediaCubeClient { return null; } - if (!response.Content.IsEmpty()) { + if (!response.Content.IsEmpty() && response.Content != "[]") { JObject media = JObject.Parse(response.Content); if (media != null) return media.Deserialize(); diff --git a/server/hu.user.mediacube.executors.tests/src/hu/user/mediacube/executors/tests/Support.java b/server/hu.user.mediacube.executors.tests/src/hu/user/mediacube/executors/tests/Support.java index 9d66e965..85c2b8e2 100644 --- a/server/hu.user.mediacube.executors.tests/src/hu/user/mediacube/executors/tests/Support.java +++ b/server/hu.user.mediacube.executors.tests/src/hu/user/mediacube/executors/tests/Support.java @@ -110,6 +110,8 @@ public class Support { private File check_rd_nonzero_fellow(String mediaid, List allFiles) { for (File mediaFile : allFiles) { + if (!mediaFile.getName().contains("-")) + continue; String houseid = mediaFile.getName().substring(0, mediaFile.getName().lastIndexOf("-")); if (houseid.equals(mediaid) && mediaFile.length() > 0) return mediaFile; diff --git a/server/user.jobengine.executors/src/user/jobengine/server/steps/BatchRetrieveForkStep.java b/server/user.jobengine.executors/src/user/jobengine/server/steps/BatchRetrieveForkStep.java index b605404f..547d019b 100644 --- a/server/user.jobengine.executors/src/user/jobengine/server/steps/BatchRetrieveForkStep.java +++ b/server/user.jobengine.executors/src/user/jobengine/server/steps/BatchRetrieveForkStep.java @@ -26,8 +26,13 @@ public class BatchRetrieveForkStep extends JobStep { public Object[] execute(List basket, String houseId, String recipient, String targetPathType, IJobEngine jobEngine, IJobRuntime jobRuntime) throws Exception { marker = (MediaCubeMarker) jobRuntime.getMarker(); + + //session szinten csak a finishMarker cimzettje az erdekes, es ezt a cimet pluszban hasznalja a konfigban megadott cimmel + //a finishMarker orokli a cim bellitast a sessionMarkertol marker.setTo(recipient); + ((MediaCubeMarker) jobRuntime.getFinishMarker()).setTo(recipient); + if (basket == null || basket.size() == 0) return null; setProgress(10); diff --git a/server/user.jobengine.executors/src/user/jobengine/server/steps/CopyForArchiveNEXIOMaterialsStep.java b/server/user.jobengine.executors/src/user/jobengine/server/steps/CopyForArchiveNEXIOMaterialsStep.java index d39c6ca1..75fedded 100644 --- a/server/user.jobengine.executors/src/user/jobengine/server/steps/CopyForArchiveNEXIOMaterialsStep.java +++ b/server/user.jobengine.executors/src/user/jobengine/server/steps/CopyForArchiveNEXIOMaterialsStep.java @@ -4,6 +4,7 @@ import java.io.OutputStream; import java.net.InetAddress; import java.net.URI; import java.util.ArrayList; +import java.util.Arrays; import java.util.Calendar; import java.util.Date; import java.util.List; @@ -101,27 +102,28 @@ public class CopyForArchiveNEXIOMaterialsStep extends JobStep { //A mar letezo mozikat nem archivaljuk le ujra, csak a metaadatot long existingMediaId = 0; + boolean transferDone = false; if (transferredFileNames.contains(origFileName)) { logger.info(systemMarker, "A '{}' file archiválásra felkészítése egy másik story kapcsán már megtörtént, ezért csak metaadat archiválás szükséges.", origFileName); if (!demo) - transferChunk(videoFileName); + transferDone = transferChunk(videoFileName); } else { existingMediaId = ItemManagerExtensions.getExistingRundownMedia(manager, origFileName); if (existingMediaId == 0) { transferredFileNames.add(origFileName); if (!demo) - transferFile(origFileName + MXFEXT, videoFileName); + transferDone = transferFile(origFileName + MXFEXT, videoFileName); logger.info(systemMarker, "A '{}' file archiválásra felkészítése sikeres volt.", origFileName); } else { logger.info(systemMarker, "A '{}' file archiválása már megtörtént, ezért csak metaadat archiválás szükséges.", origFileName); if (!demo) - transferChunk(videoFileName); + transferDone = transferChunk(videoFileName); } } - if (!demo) { + if (!demo && transferDone) { BasicDBObject metadata = createMetadata(rundownArchive, storyArchive, fileArchive, existingMediaId); transferMetadata(videoFileName, metadata); createSourceKillDateFile(rundownArchive, origFileName); @@ -408,14 +410,22 @@ public class CopyForArchiveNEXIOMaterialsStep extends JobStep { } - private void transferChunk(String fileName) throws Exception { + private boolean transferChunk(String fileName) throws Exception { logger.info("Transfer chunk {}", fileName); OutputStream outStream = null; try { targetFtp = ((FtpDirectoryLister) targetUri.getLister()).connect(); + + //ha ugyan abban a tukorben szerepel meg 1x a mozi, akkor a chunk felulirja a tenyleges file-t, es az sosem archivalodik + String[] names = targetFtp.listNames(); + if (names != null) { + if (Arrays.asList(names).contains(fileName)) + return false; + } + outStream = targetFtp.storeFileStream(fileName); if (outStream == null) { - throw new NullPointerException("Can not open: " + fileName + MXFEXT + " Reply:" + targetFtp.getReplyString()); + throw new NullPointerException("Can not open: " + fileName + " Reply:" + targetFtp.getReplyString()); } } catch (Exception e) { logger.catching(e); @@ -425,9 +435,11 @@ public class CopyForArchiveNEXIOMaterialsStep extends JobStep { outStream.close(); targetUri.cleanUp(); } + return true; } - private void transferFile(String sourceFileName, String targetFileName) throws Exception { + private boolean transferFile(String sourceFileName, String targetFileName) throws Exception { + boolean result = true; int reply = 0; logger.info("Transfer clip {}", sourceFileName); try { @@ -467,12 +479,13 @@ public class CopyForArchiveNEXIOMaterialsStep extends JobStep { } } catch (Exception e) { logger.catching(e); - throw e; + result = false; } finally { sourceUri.cleanUp(); targetUri.cleanUp(); } + return result; } private void transferMetadata(String fileName, BasicDBObject metadata) throws Exception { diff --git a/server/user.jobengine.executors/src/user/jobengine/server/steps/TSMRestoreStep.java b/server/user.jobengine.executors/src/user/jobengine/server/steps/TSMRestoreStep.java index 00582eee..446f9653 100644 --- a/server/user.jobengine.executors/src/user/jobengine/server/steps/TSMRestoreStep.java +++ b/server/user.jobengine.executors/src/user/jobengine/server/steps/TSMRestoreStep.java @@ -11,7 +11,6 @@ import org.apache.logging.log4j.Marker; import org.apache.logging.log4j.message.Message; import user.commons.LogUtils; -import user.commons.MediaCubeMarker; import user.commons.RemoteFile; import user.commons.StoreUri; import user.commons.remotestore.IProgressEventListener; @@ -79,12 +78,9 @@ public class TSMRestoreStep extends JobStep { String globalTargetPath = Paths.get(targetPath, targetFileName).getParent().toString().replace(Paths.get(localRetrievePath).toString(), globalRetrievePath); - Message msg = LogUtils.format( + logger.info(marker, "Az '{}' állomány visszatöltése sikeres volt '{}' néven. A célmappa a ide kattintva nyitható meg.", sourceFileName, targetFileName, globalTargetPath); - if (StringUtils.isNotBlank(successRecipient)) - logger.info(new MediaCubeMarker(successRecipient), msg); - logger.info(marker, msg); afterRestore(targetUri, targetPath, killDateDays, targetFileName); } catch (Exception e) { diff --git a/server/user.jobengine.osgi.server/src/user/jobengine/server/JobRuntime.java b/server/user.jobengine.osgi.server/src/user/jobengine/server/JobRuntime.java index 13072a48..5d47c4aa 100644 --- a/server/user.jobengine.osgi.server/src/user/jobengine/server/JobRuntime.java +++ b/server/user.jobengine.osgi.server/src/user/jobengine/server/JobRuntime.java @@ -51,6 +51,7 @@ public class JobRuntime extends Job implements IJobRuntime { private int runtimeWeight; private List children; private MediaCubeMarker sessionMarker; + private MediaCubeMarker finishMarker; private boolean isService; private Semaphore forkSempahore; private IJobChangedListener jobChangedListener; @@ -277,7 +278,9 @@ public class JobRuntime extends Job implements IJobRuntime { @Override public Marker getFinishMarker() { - return new MediaCubeFinishMarker((MediaCubeMarker) getMarker()); + if (finishMarker == null) + finishMarker = new MediaCubeFinishMarker((MediaCubeMarker) getMarker()); + return finishMarker; } @Override -- 2.54.0