From 2d1e5c0cd449f65172a632801fd5ad43c1ab0f5a Mon Sep 17 00:00:00 2001 From: =?utf8?q?V=C3=A1s=C3=A1ry=20D=C3=A1niel?= Date: Wed, 7 Mar 2018 19:12:38 +0000 Subject: [PATCH] git-tfs-id: [http://tfs.userrendszerhaz.hu:8080/tfs/DefaultCollection]$/MediaCube;C30958 --- .../Configuration/configuration-grafika.json | 15 ++- client/Maestro/Maestro.csproj | 4 + client/Maestro/MaestroForm.Target.cs | 16 +++ client/Maestro/Properties/AssemblyInfo.cs | 4 +- client/Maestro/Sources/FileSourceItem.cs | 11 +- client/Maestro/Sources/ISourceItem.cs | 2 +- client/Maestro/Sources/NexioRESTSource.cs | 4 +- client/Maestro/Sources/NexioSourceItem.cs | 9 ++ .../Targets/TargetProcessorParameter.cs | 1 + .../Targets/UNCTargetProcessor.cs | 3 +- ...aunch => debug-mediacube-bkup-9999.launch} | 0 .../debug-mediacube-main-9999.launch | 18 +++ server/-configuration/scheduledjobs.json | 2 +- .../test-deploy-main-mediacube-plugins.launch | 19 +++ .../META-INF/MANIFEST.MF | 3 +- .../jobtemplates/archive-recording.xml | 46 +++---- .../CopyForArchiveNEXIORecordingsStep.java | 21 +++ .../steps/DownloadRecordingFromNexioStep.java | 11 ++ .../user/jobengine/server/steps/FakeStep.java | 15 ++- .../server/steps/ImportStatisticsStep.java | 115 ++++++++++++++-- .../server/steps/MetadataTransformStep.java | 126 ++++++++++-------- .../RecordingsArchiveItemBuilderStep.java | 38 +++++- .../server/steps/TranscodeFFAStranStep.java | 47 ++++--- .../test-deploy-steps-to-bsh-main.bat | 30 +++++ .../test-deploy-steps-to-bsh-main.batx | 32 ----- .../META-INF/MANIFEST.MF | 3 +- .../{transcoder => }/FFAStransAPI.java | 2 +- .../{transcoder => }/IFFAStransAPI.java | 2 +- .../IProgressChangedListener.java | 2 +- .../src/user/jobengine/db/IItemManager.java | 5 + .../src/user/jobengine/db/ItemManager.java | 48 +++++++ .../src/user/jobengine/db/WorkflowAction.java | 1 + .../test/user/jobengine/db/Support_new.java | 85 ++++++++++++ .../pages/jobselector.zul | 35 +++-- .../pages/newshistory.zul | 54 ++++++-- .../src/user/jobengine/server/JobEngine.java | 17 ++- .../src/user/jobengine/server/JobRuntime.java | 6 +- .../server/scheduler/ScheduledJob.java | 10 +- .../jobengine/zk/model/NewsHistoryModel.java | 123 +++++++++++++++-- .../zk/model/NewsMaterialTypeChart.java | 56 ++++++++ 40 files changed, 830 insertions(+), 211 deletions(-) rename server/-configuration/{debug-mediacube-remote.launch => debug-mediacube-bkup-9999.launch} (100%) create mode 100644 server/-configuration/debug-mediacube-main-9999.launch create mode 100644 server/-configuration/test-deploy-main-mediacube-plugins.launch create mode 100644 server/user.jobengine.executors/test-deploy-steps-to-bsh-main.bat delete mode 100644 server/user.jobengine.executors/test-deploy-steps-to-bsh-main.batx rename server/user.jobengine.osgi.commons/src/user/commons/{transcoder => }/FFAStransAPI.java (96%) rename server/user.jobengine.osgi.commons/src/user/commons/{transcoder => }/IFFAStransAPI.java (87%) rename server/user.jobengine.osgi.commons/src/user/commons/{transcoder => }/IProgressChangedListener.java (69%) create mode 100644 server/user.jobengine.osgi.server/src/user/jobengine/zk/model/NewsMaterialTypeChart.java diff --git a/client/Maestro/Configuration/configuration-grafika.json b/client/Maestro/Configuration/configuration-grafika.json index 3e23f552..b09b6dd4 100644 --- a/client/Maestro/Configuration/configuration-grafika.json +++ b/client/Maestro/Configuration/configuration-grafika.json @@ -10,7 +10,7 @@ }, "source": { "$type": "UNCSource", - "filter": "png,tga,mov,mxf", + "filter": "png,tga,mov,mxf,wav", "local": { "address": "file://C:/x/", "userName": "mediacube", @@ -75,6 +75,19 @@ "password": "Dn8t4gfHcK98o8hyPgLDhr5SgSji4JCxsfpMJsODikUp3nXgrM0UNCi45lLAK8ZOnmEneO44P9qpJ4QDqhctN6MxZodjJgdZTyoZKmSa+ECzEzLr/wPYNgxVaXrVotEy", "timeout": 1000 } + }, + { + "label": "Archiválás", + "processor": "FTPTargetProcessor", + "outputFormat": "%SOURCENAME%-%ID%", + "tag": "Archiválás", + "saveArchiveMetadata": true, + "remote": { + "address": "ftp://10.10.1.100:21/ARCHIVE", + "userName": "mediacube", + "password": "Dn8t4gfHcK98o8hyPgLDhr5SgSji4JCxsfpMJsODikUp3nXgrM0UNCi45lLAK8ZOnmEneO44P9qpJ4QDqhctN6MxZodjJgdZTyoZKmSa+ECzEzLr/wPYNgxVaXrVotEy", + "timeout": 1000 + } } ] } diff --git a/client/Maestro/Maestro.csproj b/client/Maestro/Maestro.csproj index 76ed944f..2335d5e7 100644 --- a/client/Maestro/Maestro.csproj +++ b/client/Maestro/Maestro.csproj @@ -353,6 +353,10 @@ {cefd0348-ded4-453e-bf3a-0b52f8890ab3} MediaCubeClient + + {bf494ee9-1e70-44e8-8942-dd726510a766} + MXFFileParser + {8cc7c930-9dbf-487b-aed5-776937a649d5} OctopusClient diff --git a/client/Maestro/MaestroForm.Target.cs b/client/Maestro/MaestroForm.Target.cs index 4b74b05f..5824f70e 100644 --- a/client/Maestro/MaestroForm.Target.cs +++ b/client/Maestro/MaestroForm.Target.cs @@ -9,6 +9,7 @@ using MaestroShared.Targets; using MaestroShared.Target; using System.ComponentModel; using MaestroShared.Commons; +using Myriadbits.MXF; namespace Maestro { @@ -89,6 +90,20 @@ namespace Maestro { result = null; break; } + + if (sourceItem is FileSourceItem && sourceItem.Frames == 0) { + try { + string fullName = (sourceItem as FileSourceItem).FileInfo.FullName; + MXFFile mxf = new MXFFile(fullName); + mxf.Inspect(); + sourceItem.Frames = (long)mxf.TimecodeComponent.Duration; + } + catch (Exception e) { + string fullName = (sourceItem as FileSourceItem).FileInfo.FullName; + logger.Error("Can not detect length of '{0}'. Error message is: {1}", fullName, e.Message); + } + } + TargetProcessorParameter processorParameter = null; try { processorParameter = CreateProcessorParameter(target, sourceItem, createDate); @@ -178,6 +193,7 @@ namespace Maestro { result.UserName = TrayApplicationContext.UserName; result.MessageBus = MessageBus; result.CreateDate = createDate; + result.Frames = sourceItem.Frames; result.ArchiveMetadata.userName = result.UserName; if (SelectedMetadata.Kind == MetadataType.MediaCube && result.ArchiveMetadata != null) { diff --git a/client/Maestro/Properties/AssemblyInfo.cs b/client/Maestro/Properties/AssemblyInfo.cs index cf01775d..4124d82a 100644 --- a/client/Maestro/Properties/AssemblyInfo.cs +++ b/client/Maestro/Properties/AssemblyInfo.cs @@ -32,5 +32,5 @@ using System.Runtime.InteropServices; // You can specify all the values or you can default the Build and Revision Numbers // by using the '*' as shown below: // [assembly: AssemblyVersion("1.0.*")] -[assembly: AssemblyVersion("2.0.7.6")] -[assembly: AssemblyFileVersion("2.0.7.6")] +[assembly: AssemblyVersion("2.0.7.7")] +[assembly: AssemblyFileVersion("2.0.7.7")] diff --git a/client/Maestro/Sources/FileSourceItem.cs b/client/Maestro/Sources/FileSourceItem.cs index faa013cc..94ac7fbe 100644 --- a/client/Maestro/Sources/FileSourceItem.cs +++ b/client/Maestro/Sources/FileSourceItem.cs @@ -13,7 +13,7 @@ namespace Maestro.Sources { private bool isHighlighted; private bool isChecked; private static string[] sizes = { "", "KB", "MB", "GB", "TB" }; - + private long frames; public event PropertyChangedEventHandler PropertyChanged; protected void NotifyPropertyChanged([CallerMemberName] string propertyName = "") { @@ -73,6 +73,14 @@ namespace Maestro.Sources { } } + public long Frames { + get => frames; + set { + frames = value; + NotifyPropertyChanged(); + } + } + private string GetSizeString(long length) { double len = length; int order = 0; @@ -93,7 +101,6 @@ namespace Maestro.Sources { } } - public bool CanHandle(List extensions) { if (extensions == null || extensions.Count < 1) return true; diff --git a/client/Maestro/Sources/ISourceItem.cs b/client/Maestro/Sources/ISourceItem.cs index eb43f6cd..31f669ea 100644 --- a/client/Maestro/Sources/ISourceItem.cs +++ b/client/Maestro/Sources/ISourceItem.cs @@ -7,6 +7,6 @@ namespace Maestro.Sources { DateTime Created { get; } bool IsHighlighted { get; set; } bool IsChecked { get; set; } - + long Frames { get; set; } } } \ No newline at end of file diff --git a/client/Maestro/Sources/NexioRESTSource.cs b/client/Maestro/Sources/NexioRESTSource.cs index bca5dea7..6f0416b2 100644 --- a/client/Maestro/Sources/NexioRESTSource.cs +++ b/client/Maestro/Sources/NexioRESTSource.cs @@ -228,8 +228,8 @@ namespace Maestro.Sources { item.Created = created ?? DateTime.MinValue; DateTime? modified = token.Value(MODIFIED); item.Modified = modified ?? DateTime.MinValue; - int d = token.Value(DURATION); - item.Duration = new Timecode(d).ToString(); + item.Frames = token.Value(DURATION); + item.Duration = new Timecode((int)item.Frames).ToString(); } catch (Exception e) { logger.Error(e.Message); diff --git a/client/Maestro/Sources/NexioSourceItem.cs b/client/Maestro/Sources/NexioSourceItem.cs index cbc0800f..354293ed 100644 --- a/client/Maestro/Sources/NexioSourceItem.cs +++ b/client/Maestro/Sources/NexioSourceItem.cs @@ -12,6 +12,7 @@ namespace Maestro.Sources { private string agency; private string id; private string duration; + private long frames; public event PropertyChangedEventHandler PropertyChanged; @@ -93,6 +94,14 @@ namespace Maestro.Sources { } } + public long Frames { + get => frames; + set { + frames = value; + NotifyPropertyChanged(); + } + } + public bool CanHandle(List extensions) { if (extensions == null || extensions.Count < 1) return true; diff --git a/client/MaestroShared/Targets/TargetProcessorParameter.cs b/client/MaestroShared/Targets/TargetProcessorParameter.cs index 6f923a96..1520f2ee 100644 --- a/client/MaestroShared/Targets/TargetProcessorParameter.cs +++ b/client/MaestroShared/Targets/TargetProcessorParameter.cs @@ -21,5 +21,6 @@ namespace MaestroShared.Target { public string UserName { get; set; } public IMessageBus MessageBus { get; set; } public DateTime CreateDate { get; set; } + public long Frames { get; set; } } } diff --git a/client/MaestroShared/Targets/UNCTargetProcessor.cs b/client/MaestroShared/Targets/UNCTargetProcessor.cs index 6e8b517c..4fea4d0a 100644 --- a/client/MaestroShared/Targets/UNCTargetProcessor.cs +++ b/client/MaestroShared/Targets/UNCTargetProcessor.cs @@ -57,7 +57,8 @@ namespace MaestroShared.Targets { houseId = ID, tag = parameters.TargetConfig.Tag, touched = DateTime.Now, - userName = parameters.UserName + userName = parameters.UserName, + size = parameters.Frames }; Label = parameters.TargetConfig.Label; string popupMessage = parameters.TargetConfig?.PopupMessage; diff --git a/server/-configuration/debug-mediacube-remote.launch b/server/-configuration/debug-mediacube-bkup-9999.launch similarity index 100% rename from server/-configuration/debug-mediacube-remote.launch rename to server/-configuration/debug-mediacube-bkup-9999.launch diff --git a/server/-configuration/debug-mediacube-main-9999.launch b/server/-configuration/debug-mediacube-main-9999.launch new file mode 100644 index 00000000..f188461c --- /dev/null +++ b/server/-configuration/debug-mediacube-main-9999.launch @@ -0,0 +1,18 @@ + + + + + + + + + + + + + + + + + + diff --git a/server/-configuration/scheduledjobs.json b/server/-configuration/scheduledjobs.json index b2fc4826..beb98cbb 100644 --- a/server/-configuration/scheduledjobs.json +++ b/server/-configuration/scheduledjobs.json @@ -31,7 +31,7 @@ "template": "import-statistics.xml", "cronexpression": "0 0 0/1 1/1 * ? *", "parameters": [ - {"name": "daysBeforeNow", "value": 1, "type": "java.lang.Integer"} + {"name": "daysBeforeNow", "value": 3, "type": "java.lang.Integer"} ] }, { diff --git a/server/-configuration/test-deploy-main-mediacube-plugins.launch b/server/-configuration/test-deploy-main-mediacube-plugins.launch new file mode 100644 index 00000000..44754530 --- /dev/null +++ b/server/-configuration/test-deploy-main-mediacube-plugins.launch @@ -0,0 +1,19 @@ + + + + + + + + + + + + + + + + + + + diff --git a/server/user.jobengine.executors/META-INF/MANIFEST.MF b/server/user.jobengine.executors/META-INF/MANIFEST.MF index 121efda8..5c6c38de 100644 --- a/server/user.jobengine.executors/META-INF/MANIFEST.MF +++ b/server/user.jobengine.executors/META-INF/MANIFEST.MF @@ -28,4 +28,5 @@ Import-Package: com.fasterxml.jackson.annotation;version="2.4.5", org.jboss.resteasy.plugins.providers, org.jboss.resteasy.spi, org.joda.time;version="2.2.0", - org.joda.time.base;version="2.2.0" + org.joda.time.base;version="2.2.0", + sqlj.runtime.ref diff --git a/server/user.jobengine.executors/jobtemplates/archive-recording.xml b/server/user.jobengine.executors/jobtemplates/archive-recording.xml index b18b03ba..1b22d263 100644 --- a/server/user.jobengine.executors/jobtemplates/archive-recording.xml +++ b/server/user.jobengine.executors/jobtemplates/archive-recording.xml @@ -1,16 +1,16 @@ - + - - + + - - + + @@ -25,7 +25,7 @@ - + @@ -62,7 +62,7 @@ - + @@ -95,19 +95,19 @@ - - - - - - - - - - - - - + + + + + + + + + + + + + @@ -123,13 +123,13 @@ - + - + - + diff --git a/server/user.jobengine.executors/src/user/jobengine/server/steps/CopyForArchiveNEXIORecordingsStep.java b/server/user.jobengine.executors/src/user/jobengine/server/steps/CopyForArchiveNEXIORecordingsStep.java index 98cc14ed..03380a5d 100644 --- a/server/user.jobengine.executors/src/user/jobengine/server/steps/CopyForArchiveNEXIORecordingsStep.java +++ b/server/user.jobengine.executors/src/user/jobengine/server/steps/CopyForArchiveNEXIORecordingsStep.java @@ -249,14 +249,35 @@ public class CopyForArchiveNEXIORecordingsStep extends JobStep { result.setItemHouseId(String.valueOf(rundownID)); result.setItemTitle(String.format("%s %s %s", start, name, channel)); + // StringBuilder sb = new StringBuilder(); + // for (DBObject s : stories) { + // BasicDBObject story = (BasicDBObject) s; + // sb.append(story.getString(IOctopusAPI.PARENT_STORY_ID)); + // sb.append(" [" + story.getString(IOctopusAPI.FORMAT) + "] "); + // sb.append(story.getString(IOctopusAPI.NAME)); + // sb.append("\r\n"); + // } + StringBuilder sb = new StringBuilder(); for (DBObject s : stories) { BasicDBObject story = (BasicDBObject) s; + + sb.append("*** "); sb.append(story.getString(IOctopusAPI.PARENT_STORY_ID)); sb.append(" [" + story.getString(IOctopusAPI.FORMAT) + "] "); sb.append(story.getString(IOctopusAPI.NAME)); + sb.append(" ***"); sb.append("\r\n"); + String content = story.getString(IOctopusAPI.SCRIPT_CONTENT); + if (content != null) { + content = content.replace("\r\n\r\n\r\n\r\n", "\r\n"); + content = content.replace("\r\n\r\n\r\n", "\r\n"); + content = content.replace("\r\n\r\n", "\r\n"); + sb.append(content); + sb.append("\r\n"); + } } + StoryArchive storyArchive = new StoryArchive(); storyArchive.setMediaHouseId(result.getItemHouseId()); storyArchive.setMediaTitle(clipName); diff --git a/server/user.jobengine.executors/src/user/jobengine/server/steps/DownloadRecordingFromNexioStep.java b/server/user.jobengine.executors/src/user/jobengine/server/steps/DownloadRecordingFromNexioStep.java index 10caec80..c334f89c 100644 --- a/server/user.jobengine.executors/src/user/jobengine/server/steps/DownloadRecordingFromNexioStep.java +++ b/server/user.jobengine.executors/src/user/jobengine/server/steps/DownloadRecordingFromNexioStep.java @@ -65,6 +65,17 @@ public class DownloadRecordingFromNexioStep extends JobStep { String sourceFileName = targetFileName; try { final IJobRuntime runtime = jobRuntime; + + RemoteFile sourceFile = null; + while (sourceFile == null) { + try { + sourceFile = sourceUri.getRemoteFile(sourceFileName); + Thread.sleep(1000); + } catch (Exception e) { + logger.warn(e.getMessage()); + } + } + targetUri.addProgressListener(new IProgressEventListener() { @Override public void progressChanged(ProgressEvent evt) { diff --git a/server/user.jobengine.executors/src/user/jobengine/server/steps/FakeStep.java b/server/user.jobengine.executors/src/user/jobengine/server/steps/FakeStep.java index 431205b5..5e49068c 100644 --- a/server/user.jobengine.executors/src/user/jobengine/server/steps/FakeStep.java +++ b/server/user.jobengine.executors/src/user/jobengine/server/steps/FakeStep.java @@ -3,9 +3,6 @@ package user.jobengine.server.steps; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; -import com.ibm.nosql.json.api.DB; - -import user.commons.nosql.NoSQLUtils; import user.jobengine.server.IJobEngine; import user.jobengine.server.IJobRuntime; @@ -17,10 +14,10 @@ public class FakeStep extends JobStep { @StepEntry public Object[] execute(long itemID, IJobEngine jobEngine, IJobRuntime jobRuntime) throws Exception { Object[] result = { itemID + 1 }; - DB noSQLDB = NoSQLUtils.getNoSQLDB(); - //logger.info(jobRuntime.getMarker(), "Starting with {} #{}", itemID, jobRuntime.getId()); + + //logransger.info(jobRuntime.getMarker(), "Starting with {} #{}", itemID, jobRuntime.getId()); // Message message = logger.getMessageFactory().newMessage("Starting with {} #{}", itemID, jobRuntime.getId()); - // logger.error(jobRuntime.getMarker(), message, new Exception("TESZT")); + //logger.error(jobRuntime.getMarker(), message, new Exception("TESZT")); try { for (int i = 0; i < count; i++) { //sb.append("Sikeres iteráció : ") @@ -28,10 +25,14 @@ public class FakeStep extends JobStep { break; jobRuntime.incrementProgress((i + 1) * count); Thread.sleep(100); + + if (i == 2) + throw new Exception("TESZT"); //logger.info("Progress {}", jobRuntime.getProgress()); } } catch (Exception e) { - logger.error(e.getMessage()); + logger.error(jobRuntime.getMarker(), e.getMessage()); + throw e; } return result; } diff --git a/server/user.jobengine.executors/src/user/jobengine/server/steps/ImportStatisticsStep.java b/server/user.jobengine.executors/src/user/jobengine/server/steps/ImportStatisticsStep.java index 798b3fb9..5a73d619 100644 --- a/server/user.jobengine.executors/src/user/jobengine/server/steps/ImportStatisticsStep.java +++ b/server/user.jobengine.executors/src/user/jobengine/server/steps/ImportStatisticsStep.java @@ -1,5 +1,6 @@ package user.jobengine.server.steps; +import java.util.ArrayList; import java.util.Calendar; import java.util.Date; import java.util.HashMap; @@ -20,6 +21,7 @@ import user.commons.CalendarUtils; import user.commons.nosql.NoSQLUtils; import user.commons.octopus.IOctopusAPI; import user.commons.octopus.OctopusAPI; +import user.jobengine.db.IItemManager; import user.jobengine.server.IJobEngine; import user.jobengine.server.IJobRuntime; @@ -35,6 +37,7 @@ public class ImportStatisticsStep extends JobStep { private int current; private DB db; private DBCollection clipCollection; + private IItemManager manager; private void createAPI(IJobRuntime jobRuntime) throws Exception { try { @@ -63,6 +66,7 @@ public class ImportStatisticsStep extends JobStep { @StepEntry public Object[] execute(int daysBeforeNow, IJobEngine jobEngine, IJobRuntime jobRuntime) throws Exception { marker = jobRuntime.getMarker(); + manager = jobEngine.getItemManager(); db = NoSQLUtils.getNoSQLDB(); clipCollection = db.getCollection(NEXIOCLIPS); @@ -88,7 +92,77 @@ public class ImportStatisticsStep extends JobStep { stories.putAll(folderStories); logger.info(jobRuntime.getMarker(), "Gyűjtőkben megtalálható anyagok száma {}", stories.size()); - store(scheduledDate, stories); + BasicDBObject planStat = new BasicDBObject(); + + BasicDBObject typeStat = new BasicDBObject(); + BasicDBObject typeStatRawData = new BasicDBObject(); + typeStat.put("rawData", typeStatRawData); + long sumduration = 0; + + List rawData = new ArrayList<>(); + for (BasicDBObject story : stories.values()) { + List mosObjects = NoSQLUtils.asList(story, IOctopusAPI.MOS_OBJECTS); + for (BasicDBObject mos : mosObjects) { + String label = NoSQLUtils.asString(mos, IOctopusAPI.LABEL); + long duration = NoSQLUtils.asLong(mos, "duration"); + long rdCount = NoSQLUtils.asLong(story, IOctopusAPI.REF_RUNDOWN); + long sfCount = NoSQLUtils.asLong(story, IOctopusAPI.REF_STORYFOLDER); + long parentStoryId = NoSQLUtils.asLong(story, IOctopusAPI.PARENT_STORY_ID); + + BasicDBObject raw = new BasicDBObject(); + raw.put(IOctopusAPI.ID, NoSQLUtils.asLong(story, IOctopusAPI.ID)); + raw.put(IOctopusAPI.NAME, NoSQLUtils.asString(story, IOctopusAPI.NAME)); + raw.put(IOctopusAPI.PARENT_STORY_ID, parentStoryId); + raw.put(IOctopusAPI.REPORTERS, NoSQLUtils.asString(story, IOctopusAPI.REPORTERS)); + raw.put(IOctopusAPI.OBJ_ID, NoSQLUtils.asString(mos, IOctopusAPI.OBJ_ID)); + raw.put(IOctopusAPI.LABEL, label); + raw.put("duration", duration); + raw.put(IOctopusAPI.REF_RUNDOWN, rdCount); + raw.put(IOctopusAPI.REF_STORYFOLDER, sfCount); + + long ingestCount = 0; + long ingestDuration = 0; + BasicDBObject ingestInfo = manager.getIngestInfo(scheduledDate, String.valueOf(parentStoryId)); + if (ingestInfo != null) { + ingestCount = ingestInfo.getLong("count"); + ingestDuration = ingestInfo.getLong("duration"); + raw.put("ingest_count", ingestCount); + raw.put("ingest_duration", ingestDuration); + } + + rawData.add(raw); + + //planStat + if (sfCount > 0 && rdCount == 0) { + long planned = NoSQLUtils.asLong(planStat, "unreleased_count"); + planned++; + planStat.put("unreleased_count", planned); + long plannedDuration = NoSQLUtils.asLong(planStat, "unreleased_duration"); + planStat.put("unreleased_duration", plannedDuration + duration); + } + if (rdCount > 0) { + long released = NoSQLUtils.asLong(planStat, "released_count"); + released++; + planStat.put("released_count", released); + long releasedDuration = NoSQLUtils.asLong(planStat, "released_duration"); + planStat.put("released_duration", releasedDuration + duration); + + //typeStat + sumduration += duration; + if (typeStatRawData.containsKey(label)) { + BasicDBObject dbo = NoSQLUtils.asDBObject(typeStatRawData, label); + dbo.put("count", dbo.getInt("count") + 1); + dbo.put("duration", dbo.getInt("duration") + duration); + } else + typeStatRawData.put(label, new BasicDBObject("count", 1).append("duration", duration)); + } + } + } + + planStat.put("all_count", rawData.size()); + planStat.put("all_duration", sumduration); + + store(scheduledDate, rawData, typeStat, planStat); return null; } @@ -155,6 +229,8 @@ public class ImportStatisticsStep extends JobStep { long storyID = NoSQLUtils.asLong(story, IOctopusAPI.ID); if (storyID == 0) continue; + if (result.containsKey(storyID)) + continue; if (scheduledDate != null) { Date scheduleFrom = story.getDate(IOctopusAPI.SCHEDULEFROM); @@ -180,20 +256,43 @@ public class ImportStatisticsStep extends JobStep { } } - if (!result.containsKey(storyID)) - result.put(storyID, story); + List rdList = NoSQLUtils.asList(story, IOctopusAPI.REF_RUNDOWN); + story.put(IOctopusAPI.REF_RUNDOWN, rdList == null ? 0 : rdList.size()); + List sfList = NoSQLUtils.asList(story, IOctopusAPI.REF_STORYFOLDER); + story.put(IOctopusAPI.REF_STORYFOLDER, sfList == null ? 0 : sfList.size()); + + List reporters = NoSQLUtils.asList(story, IOctopusAPI.REPORTERS); + String reporterNames = ""; + if (reporters != null && reporters.size() > 0) { + for (BasicDBObject reporter : reporters) { + String userName = NoSQLUtils.asString(reporter, "userName"); + if (userName != null) { + if (reporterNames.length() > 0) + reporterNames += ", "; + reporterNames += userName; + } + } + } + story.put(IOctopusAPI.REPORTERS, reporterNames); + + result.put(storyID, story); } return result; } - private void store(Calendar scheduledDate, Map stories) { + private void store(Calendar scheduledDate, List stories, BasicDBObject typeStat, BasicDBObject planStat) { BasicDBObject dailyHistory = new BasicDBObject(); dailyHistory.put("dateTime", scheduledDate.getTime()); - dailyHistory.put("rawData", new BasicDBList(stories.values())); + BasicDBList list = new BasicDBList(); + dailyHistory.put("rawData", list); + for (BasicDBObject story : stories) { + list.add(story); + } + dailyHistory.put("typeStat", typeStat); + dailyHistory.put("planStat", planStat); + DBCollection collection = db.getCollection("daily_news_history"); - BasicDBObject existingHistory = (BasicDBObject) collection.findOne(new BasicDBObject("dateTime", scheduledDate.getTime())); - if (existingHistory != null) - dailyHistory.put(IOctopusAPI._ID, existingHistory.getID()); + collection.remove(new BasicDBObject("dateTime", scheduledDate.getTime())); collection.save(dailyHistory); } diff --git a/server/user.jobengine.executors/src/user/jobengine/server/steps/MetadataTransformStep.java b/server/user.jobengine.executors/src/user/jobengine/server/steps/MetadataTransformStep.java index 9b4631a7..9f40d80d 100644 --- a/server/user.jobengine.executors/src/user/jobengine/server/steps/MetadataTransformStep.java +++ b/server/user.jobengine.executors/src/user/jobengine/server/steps/MetadataTransformStep.java @@ -15,6 +15,7 @@ import org.apache.logging.log4j.Logger; import user.jobengine.db.Item; import user.jobengine.db.ItemManager; +import user.jobengine.db.ItemType; import user.jobengine.db.Media; import user.jobengine.server.IJobEngine; import user.jobengine.server.IJobRuntime; @@ -31,82 +32,89 @@ public class MetadataTransformStep extends JobStep { public static final String DEFAULT_MEDIATYPE = "Generic"; private ItemManager itemManager; - @StepEntry - public Object[] execute(ArchiveItem archiveItem, IJobEngine jobEngine, IJobRuntime jobRuntime) throws Exception { - Media mediaCubeMedia = null; - try { - itemManager = (ItemManager) jobEngine.getItemManager(); - if (itemManager == null) - throw new NullPointerException(ITEM_MANAGER_IS_NULL); - - File sourceMediaFile = new File(archiveItem.getMediaFile()); - String sourceFileName = sourceMediaFile.getName(); - - if (itemManager.isMediaFileExists(sourceFileName)) { - try { - Path sourcePath = Paths.get(archiveItem.getMediaFile()); - Path parent = sourcePath.getParent(); - Path conflictPath = Paths.get(parent.toString(), CONFLICT); - File folder = conflictPath.toFile(); - if (!folder.exists() || !folder.isDirectory()) { - Set perms = PosixFilePermissions.fromString("rwxrwxrwx"); - FileAttribute> attr = PosixFilePermissions.asFileAttribute(perms); + private void checkDuplicates(ArchiveItem archiveItem, String sourceFileName) throws Exception { + if (itemManager.isMediaFileExists(sourceFileName)) { + try { + Path sourcePath = Paths.get(archiveItem.getMediaFile()); + Path parent = sourcePath.getParent(); + Path conflictPath = Paths.get(parent.toString(), CONFLICT); + File folder = conflictPath.toFile(); + if (!folder.exists() || !folder.isDirectory()) { + Set perms = PosixFilePermissions.fromString("rwxrwxrwx"); + FileAttribute> attr = PosixFilePermissions.asFileAttribute(perms); + try { + Files.createDirectories(conflictPath, attr); + } catch (Exception e) { try { - Files.createDirectories(conflictPath, attr); - } catch (Exception e) { - try { - Files.createDirectory(conflictPath); - } catch (Exception e1) { - logger.catching(e); - throw e; - } + Files.createDirectory(conflictPath); + } catch (Exception e1) { + logger.catching(e); + throw e; } } - - Files.move(sourcePath, Paths.get(conflictPath.toString(), sourceFileName + (new Date()).getTime())); - } catch (Exception e1) { - logger.catching(e1); - logger.error(getMarker(), "Hiba az állomány {} mappába másolásakor. A rendszer üzenete: {}", CONFLICT, e1.getMessage()); } - throw new Exception("Az '" + sourceFileName + "' állomány már megtalálható az archívumban, archiválása nem lehetséges."); - } - - Item mediaCubeItem = itemManager.getItemByHouseID(archiveItem.getItemHouseId()); - jobRuntime.incrementProgress(50); - if (mediaCubeItem == null) - mediaCubeItem = itemManager.createItem(DEFAULT_MEDIATYPE, archiveItem.getItemTitle(), archiveItem.getItemDescription(), - archiveItem.getItemHouseId()); - - mediaCubeMedia = itemManager.createMedia(DEFAULT_MEDIATYPE, archiveItem.getMediaTitle(), archiveItem.getMediaDescription(), - archiveItem.getMediaHouseId()); - // String mediaType = archiveItem.getMediaType(); - // if (StringUtils.isBlank(mediaType)) { - // mediaCubeMedia = itemManager.createMedia(DEFAULT_MEDIATYPE, archiveItem.getMediaTitle(), archiveItem.getMediaDescription(), - // archiveItem.getMediaHouseId()); - // } else { - // ItemType mediaItemType = itemManager.getItemType(mediaType); - // if (mediaItemType == null) - // itemManager.createItemType(mediaType, mediaType).add(); - // mediaCubeMedia = itemManager.createMedia(mediaType, archiveItem.getMediaTitle(), archiveItem.getMediaDescription(), - // archiveItem.getMediaHouseId()); - // } + Files.move(sourcePath, Paths.get(conflictPath.toString(), sourceFileName + (new Date()).getTime())); + } catch (Exception e1) { + logger.catching(e1); + logger.error(getMarker(), "Hiba az állomány {} mappába másolásakor. A rendszer üzenete: {}", CONFLICT, e1.getMessage()); + } + throw new Exception("Az '" + sourceFileName + "' állomány már megtalálható az archívumban, archiválása nem lehetséges."); + } + } - mediaCubeMedia.setLength(archiveItem.getDuration()); - mediaCubeItem.appendMedia(mediaCubeMedia); + private Media createMedia(ArchiveItem archiveItem, Item mediaCubeItem, String mediaType) { + Media mediaCubeMedia; + mediaCubeMedia = itemManager.createMedia(mediaType, archiveItem.getMediaTitle(), archiveItem.getMediaDescription(), archiveItem.getMediaHouseId()); + mediaCubeMedia.setLength(archiveItem.getDuration()); + mediaCubeItem.appendMedia(mediaCubeMedia); + return mediaCubeMedia; + } + @StepEntry + public Object[] execute(ArchiveItem archiveItem, IJobEngine jobEngine, IJobRuntime jobRuntime) throws Exception { + Media mediaCubeMedia = null; + itemManager = (ItemManager) jobEngine.getItemManager(); + if (itemManager == null) + throw new NullPointerException(ITEM_MANAGER_IS_NULL); + try { + File sourceMediaFile = new File(archiveItem.getMediaFile()); + String sourceFileName = sourceMediaFile.getName(); + checkDuplicates(archiveItem, sourceFileName); + Item mediaCubeItem = getCreateItem(archiveItem); + jobRuntime.incrementProgress(50); + String mediaType = getCreateType(archiveItem); + mediaCubeMedia = createMedia(archiveItem, mediaCubeItem, mediaType); itemManager.mergeItemStructure(mediaCubeItem); } catch (Exception e) { logger.catching(e); if (!archiveItem.removeCatchedFile()) logger.error(getMarker(), "A {} állomány .catched jelző állománya nem törölhető.", new File(archiveItem.getMediaFile()).getName()); - throw e; - } finally { jobRuntime.incrementProgress(100); } return new Object[] { mediaCubeMedia }; } + private Item getCreateItem(ArchiveItem archiveItem) { + Item mediaCubeItem = itemManager.getItemByHouseID(archiveItem.getItemHouseId()); + if (mediaCubeItem == null) + mediaCubeItem = itemManager.createItem(DEFAULT_MEDIATYPE, archiveItem.getItemTitle(), archiveItem.getItemDescription(), + archiveItem.getItemHouseId()); + return mediaCubeItem; + } + + private String getCreateType(ArchiveItem archiveItem) { + String mediaType = archiveItem.getMediaType(); + if (mediaType == null || mediaType.length() == 0) + mediaType = DEFAULT_MEDIATYPE; + else { + ItemType mediaItemType = itemManager.getItemType(mediaType); + if (mediaItemType == null) + itemManager.createItemType(mediaType, mediaType).add(); + } + return mediaType; + } + } diff --git a/server/user.jobengine.executors/src/user/jobengine/server/steps/RecordingsArchiveItemBuilderStep.java b/server/user.jobengine.executors/src/user/jobengine/server/steps/RecordingsArchiveItemBuilderStep.java index f1aa4e9b..daf8c1e0 100644 --- a/server/user.jobengine.executors/src/user/jobengine/server/steps/RecordingsArchiveItemBuilderStep.java +++ b/server/user.jobengine.executors/src/user/jobengine/server/steps/RecordingsArchiveItemBuilderStep.java @@ -10,6 +10,7 @@ import java.nio.file.SimpleFileVisitor; import java.nio.file.attribute.BasicFileAttributes; import java.text.ParseException; import java.text.SimpleDateFormat; +import java.util.Calendar; import java.util.Date; import java.util.List; @@ -19,6 +20,8 @@ import org.apache.logging.log4j.Logger; import org.apache.logging.log4j.Marker; import com.ibm.nosql.json.api.BasicDBObject; +import com.ibm.nosql.json.api.DB; +import com.ibm.nosql.json.api.DBCollection; import com.ibm.nosql.json.api.DBObject; import user.commons.CalendarUtils; @@ -39,6 +42,7 @@ public class RecordingsArchiveItemBuilderStep extends JobStep { private static final String SCHEDULED_FORMAT = "yyyy.MM.dd HH:mm"; private Marker marker; + private DBCollection existingRecordings; private ArchiveItem createArchiveItem(Path mediaFilePath, Path catchedFilePath) { ArchiveItem result = null; @@ -48,6 +52,7 @@ public class RecordingsArchiveItemBuilderStep extends JobStep { String clipName = mediaFilePath.toFile().getName(); Date scheduledStart = getScheduledStart(clipName, recordDate); IOctopusAPI octopusAPI = new OctopusAPI(); + DBObject rundown = octopusAPI.getRundown(scheduledStart); if (rundown == null) { logger.error(marker, "A '{}' anyaghoz nem található tükör '{}' kezdéssel, ezért nem archiválható.", clipName, scheduledStart); @@ -57,6 +62,22 @@ public class RecordingsArchiveItemBuilderStep extends JobStep { if (result == null) return null; + if (clipName.startsWith("1900")) { + Calendar cal = CalendarUtils.createCalendar(scheduledStart); + cal.add(Calendar.MINUTE, 5); + rundown = octopusAPI.getRundown(cal.getTime()); + if (rundown == null) { + logger.error(marker, "A '{}' anyaghoz nem található tükör '{}' kezdéssel, ezért nem archiválható.", clipName, scheduledStart); + return null; + } + ArchiveItem item2 = processRundow(octopusAPI, rundown); + if (item2 == null) + return null; + + result.setItemTitle(result.getItemTitle() + " + NAPIAKT"); + result.setMediaDescription(result.getMediaDescription() + "\r\n\r\n****** NAPIAKT ******\r\n\r\n" + item2.getMediaDescription()); + } + result.setMediaTitle(clipName); result.setMediaType(MEDIATYPE); result.setMediaFile(mediaFilePath.toString()); @@ -84,6 +105,8 @@ public class RecordingsArchiveItemBuilderStep extends JobStep { @StepEntry public Object[] execute(String sourcePath, IJobEngine jobEngine, IJobRuntime jobRuntime) throws Exception { final ArchiveItem[] archiveItems = { null }; + DB db = NoSQLUtils.getNoSQLDB(); + existingRecordings = db.getCollection("tmp_existing_recordings"); marker = getJobRuntime().getMarker(); try { Files.walkFileTree(Paths.get(sourcePath), new SimpleFileVisitor() { @@ -150,6 +173,9 @@ public class RecordingsArchiveItemBuilderStep extends JobStep { String name = new File(mediaFile).getName(); int extPos = name.toLowerCase().lastIndexOf(LXFEXT); targetFileName = String.format("20%s-%s", Paths.get(mediaFile).getParent().getFileName(), name.substring(0, extPos)); + if (targetFileName.length() > 32) { + targetFileName = targetFileName.substring(0, 27) + "_PGM"; + } logger.info(marker, "Az archiváló folyamat az '{}' anyagot archiválja.", mediaFile); } @@ -195,7 +221,7 @@ public class RecordingsArchiveItemBuilderStep extends JobStep { } if (mediaFile.isDirectory() || !mediaFile.getName().toLowerCase().endsWith(LXFEXT.toLowerCase()) - || mediaFilePath.getParent().toFile().getName().length() != 6) { + || mediaFilePath.getParent().toFile().getName().length() != 6 || !mediaFile.getName().toLowerCase().contains("_pgm_")) { logger.info("Skipping '{}'", mediaFilePath); return null; } @@ -207,6 +233,13 @@ public class RecordingsArchiveItemBuilderStep extends JobStep { return null; } + DBObject existingRecording = existingRecordings.findOne(new BasicDBObject("title", archiveItem.getItemTitle().substring(0, 16))); + + if (existingRecording != null) { + logger.warn("'{}' already archived, skipping.", archiveItem.getItemTitle()); + return null; + } + if (StringUtils.isBlank(archiveItem.getItemHouseId())) { logger.warn("'{}' has no Item HouseID specified in metadata.", mediaFilePath); return null; @@ -272,9 +305,6 @@ public class RecordingsArchiveItemBuilderStep extends JobStep { } result.setMediaHouseId(result.getItemHouseId()); result.setMediaDescription(sb.toString()); - //TODO - if (result.getMediaDescription() != null && result.getMediaDescription().length() > 5000) - result.setMediaDescription(result.getMediaDescription().substring(0, 5000)); return result; } } diff --git a/server/user.jobengine.executors/src/user/jobengine/server/steps/TranscodeFFAStranStep.java b/server/user.jobengine.executors/src/user/jobengine/server/steps/TranscodeFFAStranStep.java index c09674f2..ae3802d8 100644 --- a/server/user.jobengine.executors/src/user/jobengine/server/steps/TranscodeFFAStranStep.java +++ b/server/user.jobengine.executors/src/user/jobengine/server/steps/TranscodeFFAStranStep.java @@ -12,8 +12,8 @@ import org.apache.logging.log4j.Marker; import org.apache.logging.log4j.message.Message; import org.apache.logging.log4j.message.ParameterizedMessage; -import user.commons.transcoder.FFAStransAPI; -import user.commons.transcoder.IFFAStransAPI; +import user.commons.FFAStransAPI; +import user.commons.IFFAStransAPI; import user.jobengine.db.FileType; import user.jobengine.db.IItemManager; import user.jobengine.db.Media; @@ -36,11 +36,11 @@ public class TranscodeFFAStranStep extends JobStep { public Object[] execute(ArchiveItem archiveItem, Media mediaCubeMedia, String transcoderAddress, String transcoderTemplateName, String localHiresTargetPath, String globalHiresSourcePath, String localLowresTargetPath, String webPath, boolean deleteSource, IJobEngine jobEngine, IJobRuntime jobRuntime) throws Exception { - marker = jobRuntime.getMarker(); - manager = jobEngine.getItemManager(); - store = check(manager.getSystemStore(true), "lowres Store"); - fileType = check(manager.getFileType(LOWRES_FILETYPE), "lowres FileType"); - mediaCubeMedia = check(mediaCubeMedia, "mediaCubeMedia"); + this.marker = jobRuntime.getMarker(); + this.manager = jobEngine.getItemManager(); + this.store = check(manager.getSystemStore(true), "lowres Store"); + this.fileType = check(manager.getFileType(LOWRES_FILETYPE), "lowres FileType"); + this.mediaCubeMedia = check(mediaCubeMedia, "mediaCubeMedia"); check(archiveItem, "archiveItem"); check(transcoderAddress, "transcoderAddress"); check(transcoderTemplateName, "transcoderTemplateName"); @@ -52,42 +52,59 @@ public class TranscodeFFAStranStep extends JobStep { File sourceMediaFile = new File(archiveItem.getMediaFile()); String sourceFileName = sourceMediaFile.getName(); String details = String.format("%s (%d bytes)", sourceFileName, sourceMediaFile.length()); + Path targetPath = null; try { - String sourceFile = sourceMediaFile.toString().replace(localHiresTargetPath, globalHiresSourcePath); + String sourceFile = Paths.get(globalHiresSourcePath, sourceFileName).toString(); IFFAStransAPI api = new FFAStransAPI(transcoderAddress, p -> { - jobRuntime.incrementProgress(p); + if (p <= 100) + jobRuntime.incrementProgress(p); }); api.submit(transcoderTemplateName, sourceFile); jobRuntime.setDescription(String.format("%s: %s", jobRuntime.getDescription(), details)); api.monitor(1000); - Path targetPath = Paths.get(localLowresTargetPath, sourceFileName.replace(MXFEXT, MP4EXT)); + targetPath = Paths.get(localLowresTargetPath, sourceFileName.replace(MXFEXT, MP4EXT)); postprocess(targetPath, webPath); - if (deleteSource) - sourceMediaFile.delete(); } catch (Exception e) { logger.catching(e); Message m = new ParameterizedMessage("Az '{}' állomány átkódolása sikertelen. A rendszer hibaüzenete: {}", details, e.getMessage()); logger.error(marker, m); throw new Exception(m.getFormattedMessage()); + } finally { + try { + if (deleteSource && sourceMediaFile != null && sourceMediaFile.exists()) + sourceMediaFile.delete(); + } catch (Exception e) { + logger.catching(e); + } + try { + if (deleteSource && targetPath != null && targetPath.toFile().exists()) + Files.delete(targetPath); + } catch (Exception e) { + logger.catching(e); + } } return null; } private void postprocess(Path transcodedFilePath, String webPath) { + Path lowresPath = null; try { String transcodedFileName = transcodedFilePath.getFileName().toString(); if (transcodedFileName.indexOf(".") > 2) { Path subdir = Paths.get(transcodedFileName.substring(0, 1), transcodedFileName.substring(1, 2), transcodedFileName.substring(2, 3)); manager.createMediaFile(Paths.get(subdir.toString(), transcodedFileName).toString(), fileType, store, mediaCubeMedia).add(); EscortFiles.ensureUNCFolder(webPath, subdir.toString()); - Files.move(transcodedFilePath, Paths.get(webPath, subdir.toString(), transcodedFileName)); + lowresPath = Paths.get(webPath, subdir.toString(), transcodedFileName); + Files.move(transcodedFilePath, lowresPath); } else { manager.createMediaFile(transcodedFileName, fileType, store, mediaCubeMedia).add(); - Files.move(transcodedFilePath, Paths.get(webPath, transcodedFileName)); + lowresPath = Paths.get(webPath, transcodedFileName); + Files.move(transcodedFilePath, lowresPath); } } catch (IOException e) { + lowresPath = transcodedFilePath; logger.catching(e); - logger.error(marker, "A '{}' állomány mozgatása a '{}' mappába nem sikerült.", transcodedFilePath, webPath); + logger.error(marker, "A '{}' állomány mozgatása a '{}' helyre nem sikerült.", transcodedFilePath, transcodedFilePath); } } diff --git a/server/user.jobengine.executors/test-deploy-steps-to-bsh-main.bat b/server/user.jobengine.executors/test-deploy-steps-to-bsh-main.bat new file mode 100644 index 00000000..4ad3d7ef --- /dev/null +++ b/server/user.jobengine.executors/test-deploy-steps-to-bsh-main.bat @@ -0,0 +1,30 @@ +@ECHO OFF + +SET REMOTE_HOST=10.10.1.28 +SET REMOTE_SERVER_HOSTKEY=ssh-ed25519 256 ea:ab:67:70:79:63:2f:6a:34:81:48:e2:b9:dd:ca:d4 +SET REMOTE_SERVER_ADDRESS=scp://root:password@%REMOTE_HOST% + +SET LOCAL_STEPS_LOCATION=bin/user +SET REMOTE_STEPS_LOCATION=/opt/test-mediacube/configuration/executors/user + +SET LOCAL_TEMPLATES_LOCATION=jobtemplates +SET REMOTE_TEMPLATES_LOCATION=/opt/test-mediacube/configuration/jobtemplates + + ECHO *** Deploy steps to server %REMOTE_HOST% *** + WinSCP.com /command ^ + "open %REMOTE_SERVER_ADDRESS% -hostkey=""%REMOTE_SERVER_HOSTKEY%"" -timeout=60" ^ + "lcd %LOCAL_STEPS_LOCATION%" ^ + "cd %REMOTE_STEPS_LOCATION%" ^ + "synchronize remote" ^ + "exit" + + ECHO *** Deploy templates to server %REMOTE_HOST% *** + WinSCP.com /command ^ + "open %REMOTE_SERVER_ADDRESS% -hostkey=""%REMOTE_SERVER_HOSTKEY%"" -timeout=60" ^ + "lcd %LOCAL_TEMPLATES_LOCATION%" ^ + "cd %REMOTE_TEMPLATES_LOCATION%" ^ + "synchronize remote" ^ + "exit" + +@ECHO ON + \ No newline at end of file diff --git a/server/user.jobengine.executors/test-deploy-steps-to-bsh-main.batx b/server/user.jobengine.executors/test-deploy-steps-to-bsh-main.batx deleted file mode 100644 index 8a83ec7d..00000000 --- a/server/user.jobengine.executors/test-deploy-steps-to-bsh-main.batx +++ /dev/null @@ -1,32 +0,0 @@ -@ECHO OFF - -SET REMOTE_HOST=10.10.1.28 -SET REMOTE_LOCATION=/opt/test-mediacube/configuration/executors -SET REMOTE_SERVER_HOSTKEY=ssh-ed25519 256 ea:ab:67:70:79:63:2f:6a:34:81:48:e2:b9:dd:ca:d4 -SET REMOTE_SERVER_ADDRESS=scp://root:password@%REMOTE_HOST% -SET LOCAL_STEPS_LOCATION=bin/user/jobengine/server/steps -SET LOCAL_CONFIG_LOCATION=config - - ECHO *** Begin deploy steps to server %REMOTE_HOST% *** - ECHO --- Deploying - - WinSCP.com /command ^ - "open %REMOTE_SERVER_ADDRESS% -hostkey=""%REMOTE_SERVER_HOSTKEY%"" -timeout=60" ^ - "cd %REMOTE_LOCATION%" ^ - "lcd %LOCAL_STEPS_LOCATION%" ^ - "synchronize remote" ^ - "lcd ..\" ^ - "lcd ..\" ^ - "lcd ..\" ^ - "lcd ..\" ^ - "lcd ..\" ^ - "lcd %LOCAL_CONFIG_LOCATION%" ^ - "synchronize remote -filemask=config.xml" ^ - "cd .." ^ - "synchronize remote -filemask=scheduledjobs.json" ^ - "exit" - - ECHO *** Completed *** -@ECHO ON - - diff --git a/server/user.jobengine.osgi.commons/META-INF/MANIFEST.MF b/server/user.jobengine.osgi.commons/META-INF/MANIFEST.MF index f5903f80..51f1af6f 100644 --- a/server/user.jobengine.osgi.commons/META-INF/MANIFEST.MF +++ b/server/user.jobengine.osgi.commons/META-INF/MANIFEST.MF @@ -31,8 +31,7 @@ Export-Package: user.commons, user.commons.selenio.wsclient, user.commons.server.protocol, user.commons.server.tcp, - user.commons.strings, - user.commons.transcoder + user.commons.strings Require-Bundle: org.apache.commons.lang;bundle-version="2.4.0", user.tsm.client;bundle-version="1.2.0", org.apache.commons.digester;bundle-version="3.2.0";visibility:=reexport, diff --git a/server/user.jobengine.osgi.commons/src/user/commons/transcoder/FFAStransAPI.java b/server/user.jobengine.osgi.commons/src/user/commons/FFAStransAPI.java similarity index 96% rename from server/user.jobengine.osgi.commons/src/user/commons/transcoder/FFAStransAPI.java rename to server/user.jobengine.osgi.commons/src/user/commons/FFAStransAPI.java index e4688763..37ec202b 100644 --- a/server/user.jobengine.osgi.commons/src/user/commons/transcoder/FFAStransAPI.java +++ b/server/user.jobengine.osgi.commons/src/user/commons/FFAStransAPI.java @@ -1,4 +1,4 @@ -package user.commons.transcoder; +package user.commons; import java.util.List; diff --git a/server/user.jobengine.osgi.commons/src/user/commons/transcoder/IFFAStransAPI.java b/server/user.jobengine.osgi.commons/src/user/commons/IFFAStransAPI.java similarity index 87% rename from server/user.jobengine.osgi.commons/src/user/commons/transcoder/IFFAStransAPI.java rename to server/user.jobengine.osgi.commons/src/user/commons/IFFAStransAPI.java index a42b1de1..d14ba3e6 100644 --- a/server/user.jobengine.osgi.commons/src/user/commons/transcoder/IFFAStransAPI.java +++ b/server/user.jobengine.osgi.commons/src/user/commons/IFFAStransAPI.java @@ -1,4 +1,4 @@ -package user.commons.transcoder; +package user.commons; import java.util.List; diff --git a/server/user.jobengine.osgi.commons/src/user/commons/transcoder/IProgressChangedListener.java b/server/user.jobengine.osgi.commons/src/user/commons/IProgressChangedListener.java similarity index 69% rename from server/user.jobengine.osgi.commons/src/user/commons/transcoder/IProgressChangedListener.java rename to server/user.jobengine.osgi.commons/src/user/commons/IProgressChangedListener.java index 395435fb..ee5155e0 100644 --- a/server/user.jobengine.osgi.commons/src/user/commons/transcoder/IProgressChangedListener.java +++ b/server/user.jobengine.osgi.commons/src/user/commons/IProgressChangedListener.java @@ -1,4 +1,4 @@ -package user.commons.transcoder; +package user.commons; public interface IProgressChangedListener { void onProgressChanged(int progress); diff --git a/server/user.jobengine.osgi.db/src/user/jobengine/db/IItemManager.java b/server/user.jobengine.osgi.db/src/user/jobengine/db/IItemManager.java index 90d09852..4278d8a6 100644 --- a/server/user.jobengine.osgi.db/src/user/jobengine/db/IItemManager.java +++ b/server/user.jobengine.osgi.db/src/user/jobengine/db/IItemManager.java @@ -2,9 +2,12 @@ package user.jobengine.db; import java.net.URI; import java.sql.Timestamp; +import java.util.Calendar; import java.util.Date; import java.util.List; +import com.ibm.nosql.json.api.BasicDBObject; + import user.commons.BaseType; import user.commons.IEntityBase; import user.commons.IEntityPersister; @@ -179,6 +182,8 @@ public interface IItemManager extends IEntityPersister { */ List getFolders(long id); + BasicDBObject getIngestInfo(Calendar scheduleDate, String houseId); + Item getItemByHouseID(String houseID); ItemType getItemType(String itemTypeName); diff --git a/server/user.jobengine.osgi.db/src/user/jobengine/db/ItemManager.java b/server/user.jobengine.osgi.db/src/user/jobengine/db/ItemManager.java index 6cde5259..cc67e642 100644 --- a/server/user.jobengine.osgi.db/src/user/jobengine/db/ItemManager.java +++ b/server/user.jobengine.osgi.db/src/user/jobengine/db/ItemManager.java @@ -7,6 +7,7 @@ import java.sql.ResultSet; import java.sql.SQLException; import java.sql.Statement; import java.sql.Timestamp; +import java.text.SimpleDateFormat; import java.util.ArrayList; import java.util.Arrays; import java.util.Calendar; @@ -18,6 +19,8 @@ import java.util.Map; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; +import com.ibm.nosql.json.api.BasicDBObject; + import sqlj.runtime.ref.DefaultContext; import user.commons.BaseType; import user.commons.CalendarUtils; @@ -512,6 +515,51 @@ public class ItemManager extends MemoryCache implements IItemManager { return result; } + @Override + public BasicDBObject getIngestInfo(Calendar scheduleDate, String houseid) { + BasicDBObject result = null; + ResultSet rs = null; + PreparedStatement st = null; + DefaultContext context = getDbContext(); + Connection connection = context.getConnection(); + SimpleDateFormat df = new SimpleDateFormat("yyyy-MM-dd"); + try { + String date = df.format(scheduleDate.getTime()); + String query = String.format( + "SELECT COUNT(*) as count, SUM(size) as duration FROM VW_RD_INGEST WHERE HOUSEID='%s' AND FINISHED LIKE '%s%%' GROUP BY HOUSEID", houseid, + date); + st = connection.prepareStatement(query); + rs = st.executeQuery(); + if (rs.next()) { + result = new BasicDBObject(); + result.put("count", rs.getLong("count")); + result.put("duration", rs.getLong("duration")); + } + + connection.commit(); + } catch (Exception e) { + try { + connection.rollback(); + } catch (Exception e1) { + } + throwError(e); + } finally { + try { + if (rs != null) + rs.close(); + } catch (Exception e1) { + } + try { + if (st != null) + st.close(); + } catch (Exception e1) { + } + putDbContext(context); + } + + return result; + } + @Override public Item getItemByHouseID(String houseID) { traceIn(); diff --git a/server/user.jobengine.osgi.db/src/user/jobengine/db/WorkflowAction.java b/server/user.jobengine.osgi.db/src/user/jobengine/db/WorkflowAction.java index 8d89a664..a86ace16 100644 --- a/server/user.jobengine.osgi.db/src/user/jobengine/db/WorkflowAction.java +++ b/server/user.jobengine.osgi.db/src/user/jobengine/db/WorkflowAction.java @@ -8,6 +8,7 @@ public class WorkflowAction extends JSONBase implements Serializable { private Timestamp touched; private Timestamp started; private Timestamp finished; + private Timestamp fileCreated; private boolean successful; private String houseId; private String description; diff --git a/server/user.jobengine.osgi.db/test/user/jobengine/db/Support_new.java b/server/user.jobengine.osgi.db/test/user/jobengine/db/Support_new.java index b02ed534..2fc515ff 100644 --- a/server/user.jobengine.osgi.db/test/user/jobengine/db/Support_new.java +++ b/server/user.jobengine.osgi.db/test/user/jobengine/db/Support_new.java @@ -11,8 +11,12 @@ import java.sql.PreparedStatement; import java.sql.ResultSet; import java.text.SimpleDateFormat; import java.util.ArrayList; +import java.util.Calendar; +import java.util.Collections; import java.util.Date; +import java.util.LinkedHashMap; import java.util.List; +import java.util.Map; import java.util.function.Consumer; import java.util.stream.Stream; @@ -20,8 +24,15 @@ import org.junit.AfterClass; import org.junit.BeforeClass; import org.junit.Test; +import com.ibm.nosql.json.api.BasicDBObject; +import com.ibm.nosql.json.api.DB; +import com.ibm.nosql.json.api.DBCollection; + +import sqlj.runtime.ref.DefaultContext; +import user.commons.CalendarUtils; import user.commons.IEntityBase; import user.commons.logging.LogUtils; +import user.commons.nosql.NoSQLUtils; public class Support_new { public class BD { @@ -33,6 +44,9 @@ public class Support_new { @BeforeClass static public void setUpConnection() { + System.setProperty("jobengine.nosql.db.url", "jdbc:db2://10.10.1.27:50000/mc:retrieveMessagesFromServerOnGetMessage=true;"); + System.setProperty("jobengine.nosql.db.user", "db2admin"); + System.setProperty("jobengine.nosql.db.password", "password"); System.setProperty(ItemManager.DBURL, "jdbc:db2://10.10.1.27:50000/mc:retrieveMessagesFromServerOnGetMessage=true;"); System.setProperty(ItemManager.DBUSERNAME, "db2admin"); System.setProperty(ItemManager.DBPASSWORD, "password"); @@ -48,6 +62,77 @@ public class Support_new { manager.disconnect(); } + public void buildRecordings(Map recordings) { + ResultSet rs = null; + PreparedStatement st = null; + DefaultContext context = ((ItemManager) manager).getDbContext(); + Connection connection = context.getConnection(); + try { + DB db = NoSQLUtils.getNoSQLDB(); + DBCollection collection = db.getCollection("tmp_existing_recordings"); + collection.drop(); + String query = "select itemtitle, mediafilehouseid from vw_items where itemtitle like '%Echo TV%' and itemtitle like '2%' and mediatitle like '%PGM%' order by itemtitle, mediafilehouseid"; + st = connection.prepareStatement(query); + rs = st.executeQuery(); + while (rs.next()) { + String itemtitle = rs.getString("itemtitle").substring(0, 16); + String mediafilehosueid = rs.getString("mediafilehouseid"); + recordings.put(itemtitle, mediafilehosueid); + collection.save(new BasicDBObject("title", itemtitle)); + } + + connection.commit(); + } catch (Exception e) { + try { + connection.rollback(); + } catch (Exception e1) { + } + } finally { + try { + if (rs != null) + rs.close(); + } catch (Exception e1) { + } + try { + if (st != null) + st.close(); + } catch (Exception e1) { + } + ((ItemManager) manager).putDbContext(context); + } + + } + + @Test + public void generateMissingRecordingsList() throws Exception { + // select itemtitle, mediafilehouseid from vw_items where itemtitle like '%Echo TV%' and itemtitle like '2%' and mediatitle like '%PGM%' order by itemtitle, mediafilehouseid + String[] hours = { "07:00", "08:00", "09:00", "10:00", "11:00", "12:00", "13:00", "14:00", "15:00", "16:00", "17:00", "18:00", "19:00", "20:00", + "22:00" }; + + Map recordings = new LinkedHashMap<>(); + Calendar cal = CalendarUtils.createZeroCalendar(new Date(2018 - 1900, 0, 1)); + SimpleDateFormat df = new SimpleDateFormat("yyyy.MM.dd"); + for (int i = 0; i < 70; i++) { + for (String hour : hours) { + String key = String.format("%s %s", df.format(cal.getTime()), hour); + recordings.put(key, ""); + // System.out.println(key); + } + cal.add(Calendar.DAY_OF_MONTH, 1); + } + buildRecordings(recordings); + + List lines = new ArrayList<>(); + List keys = new ArrayList<>(recordings.keySet()); + Collections.sort(keys); + for (String key : keys) { + lines.add(key + ";" + recordings.get(key) + ";"); + System.out.println(key + "\t" + recordings.get(key)); + } + Files.write(Paths.get("c:\\temp\\recordings.csv"), lines, Charset.forName("UTF-8"), StandardOpenOption.CREATE_NEW); + + } + @Test public void generateMorpheusArchivedList() throws Exception { Path sourcePath = Paths.get("\\\\10.10.1.100\\BRAAVOS\\PLAYOUT\\Video"); diff --git a/server/user.jobengine.osgi.server/pages/jobselector.zul b/server/user.jobengine.osgi.server/pages/jobselector.zul index f18033d0..27605194 100644 --- a/server/user.jobengine.osgi.server/pages/jobselector.zul +++ b/server/user.jobengine.osgi.server/pages/jobselector.zul @@ -1,5 +1,5 @@ -
@@ -15,24 +15,23 @@
- - + - - - - - - + + + + + + + + + + + + + + diff --git a/server/user.jobengine.osgi.server/pages/newshistory.zul b/server/user.jobengine.osgi.server/pages/newshistory.zul index cb3cdb2f..3fd4ecaf 100644 --- a/server/user.jobengine.osgi.server/pages/newshistory.zul +++ b/server/user.jobengine.osgi.server/pages/newshistory.zul @@ -4,30 +4,62 @@ - -