From 84b9fe6cfc29078fee2e9fba8d454f1d027e13c2 Mon Sep 17 00:00:00 2001 From: =?utf8?q?V=C3=A1s=C3=A1ry=20D=C3=A1niel?= Date: Thu, 21 Dec 2017 16:44:59 +0000 Subject: [PATCH] git-tfs-id: [http://tfs.userrendszerhaz.hu:8080/tfs/DefaultCollection]$/MediaCube;C30819 --- client/DxPlay/DxPlayer.cs | 61 +++++++++--- ...son => -configuration-playout-ingest.json} | 0 .../configuration-tqc-check.json | 98 +++++++++++++++++++ client/Maestro/Maestro.csproj | 5 +- server/-product/log4j2.xml | 13 +-- server/-product/mediacube.product | 56 +++++------ .../config/scheduledjobs.json | 36 +++---- .../server/steps/ArchiveListBuilderStep.java | 6 +- .../CopyForArchiveNEXIOMaterialsStep.java | 14 +-- .../server/steps/CreateMissingLowresStep.java | 41 +++++++- .../server/steps/OctopusDataMiner.java | 17 +++- .../server/steps/TranscodeSELENIOStep.java | 17 +++- .../octopus/test/OctopusDataMinerTest.java | 2 +- server/user.jobengine.osgi.db/.classpath | 2 +- .../.settings/org.eclipse.jdt.core.prefs | 7 +- .../org.hibernate.eclipse.console.prefs | 3 + .../src/user/jobengine/db/ItemDAOEx.java | 42 ++++++++ .../user/jobengine/zk/model/SearchModel.java | 10 +- .../user/jobengine/zk/util/AuthInitiator.java | 29 +++--- .../user/jobengine/server/steps/General.java | 26 +++++ 20 files changed, 367 insertions(+), 118 deletions(-) rename client/Maestro/Configuration/{configuration-playout-ingest.json => -configuration-playout-ingest.json} (100%) create mode 100644 client/Maestro/Configuration/configuration-tqc-check.json create mode 100644 server/user.jobengine.osgi.db/.settings/org.hibernate.eclipse.console.prefs create mode 100644 server/user.jobengine.osgi.server/test/user/jobengine/server/steps/General.java diff --git a/client/DxPlay/DxPlayer.cs b/client/DxPlay/DxPlayer.cs index 3d8d3247..9cbe9d8e 100644 --- a/client/DxPlay/DxPlayer.cs +++ b/client/DxPlay/DxPlayer.cs @@ -14,10 +14,12 @@ using System.Drawing.Drawing2D; using System.Collections.Generic; using Myriadbits.MXF; using MaestroShared.Metadata; +using NLog; namespace DxPlay { internal class DxPlayer : ISampleGrabberCB, IDisposable { + private static readonly Logger logger = LogManager.GetCurrentClassLogger(); [DllImport("Kernel32.dll", EntryPoint = "RtlMoveMemory")] private static extern void CopyMemory(IntPtr Destination, IntPtr Source, [MarshalAs(UnmanagedType.U4)] uint Length); @@ -86,7 +88,6 @@ namespace DxPlay { int hr; IntPtr hEvent; MediaDescription = mediaDesc; - Debug.WriteLine("SetupGraph"); // Set up the graph playerWindow = hWin; @@ -145,7 +146,7 @@ namespace DxPlay { public void Play() { // If we aren't already playing (or shutting down) if (State == GraphState.Completed) - Stop(); + Stop(); if (State == GraphState.Stopped || State == GraphState.Paused) { int hr = m_mediaCtrl.Run(); DsError.ThrowExceptionForHR(hr); @@ -243,6 +244,7 @@ namespace DxPlay { #if DEBUG m_DsRot = new DsROTEntry(m_FilterGraph); #endif + logger.Debug("Add SourceFilter to graph"); IBaseFilter sourceFilter = null; hr = m_FilterGraph.AddSourceFilter(MediaDescription.FileName, MediaDescription.FileName, out sourceFilter); DsError.ThrowExceptionForHR(hr); @@ -252,10 +254,12 @@ namespace DxPlay { //m_FilterGraph.AddFilter(splitter, "MXF Splitter"); //splitter = FilterGraphTools.AddFilterByName(graphBuilder, FilterCategory.LegacyAmFilterCategory, "Sony MXF Splitter"); + logger.Debug("Add LAVSplitter to graph"); IBaseFilter splitter = LoadSplitter(graphBuilder); if (splitter == null) throw new Exception("No splitter!"); + logger.Debug("Connect SourceFilter -> LAVSplitter"); FilterGraphTools.ConnectFilters(graphBuilder, sourceFilter, "Output", splitter, "Input", true); //IAMStreamSelect amStreamSelect = (IAMStreamSelect)splitter; @@ -286,44 +290,69 @@ namespace DxPlay { // Debug.WriteLine("Audio count: " + audioCount); //} + logger.Debug("Add LAVVideo to graph"); IBaseFilter videoDecoder = LoadVideoDecoder(graphBuilder); if (videoDecoder == null) throw new Exception("No video decoder!"); + logger.Debug("Connect LAVSplitter -> LAVVideo"); FilterGraphTools.ConnectFilters(graphBuilder, splitter, "Video", videoDecoder, "Input", true); + logger.Debug("Add SampleGrabber to graph"); IBaseFilter sampGrabber = (IBaseFilter)new SampleGrabber(); ConfigureSampleGrabber((ISampleGrabber)sampGrabber); hr = m_FilterGraph.AddFilter(sampGrabber, "Sample Grabber"); DsError.ThrowExceptionForHR(hr); + logger.Debug("Connect LAVVideo -> SampleGrabber"); FilterGraphTools.ConnectFilters(graphBuilder, videoDecoder, "Output", sampGrabber, "Input", true); + logger.Debug("Add VideoMixingRenderer9 to graph"); m_videoRenderer = (IBaseFilter)new VideoMixingRenderer9(); hr = m_FilterGraph.AddFilter(m_videoRenderer, "Video Mixing Renderer 9"); DsError.ThrowExceptionForHR(hr); - IVMRDeinterlaceControl9 deinterlace = (IVMRDeinterlaceControl9)m_videoRenderer; - Guid interlaceMode; - deinterlace.GetActualDeinterlaceMode(0, out interlaceMode); + //logger.Debug("Add VideoMixingRenderer9 to graph"); + //IVMRDeinterlaceControl9 deinterlace = (IVMRDeinterlaceControl9)m_videoRenderer; + //Guid interlaceMode; + //deinterlace.GetActualDeinterlaceMode(0, out interlaceMode); - FilterGraphTools.ConnectFilters(graphBuilder, sampGrabber, "Output", m_videoRenderer, "VMR Input0", true); - - if (DsFindPin.ByName(splitter, "Audio") != null) { - IBaseFilter audioDecoder = null; - audioDecoder = LoadAudioDecoder(graphBuilder); - if (audioDecoder == null) - throw new Exception("No audio decoder!"); + try { + logger.Debug("Connect SampleGrabber -> VideoMixingRenderer9"); + FilterGraphTools.ConnectFilters(graphBuilder, sampGrabber, "Output", m_videoRenderer, "VMR Input0", true); + } + catch (Exception e) { + logger.Error(e); + } - FilterGraphTools.ConnectFilters(graphBuilder, splitter, "Audio", audioDecoder, "Input", true); - FilterGraphTools.RenderPin(graphBuilder, audioDecoder, "Output"); + try { + if (DsFindPin.ByName(splitter, "Audio") != null) { + logger.Debug("Add LAVAudio to graph"); + IBaseFilter audioDecoder = null; + audioDecoder = LoadAudioDecoder(graphBuilder); + if (audioDecoder == null) + throw new Exception("No audio decoder!"); + + logger.Debug("Connect LAVSplitter -> LAVAudio"); + FilterGraphTools.ConnectFilters(graphBuilder, splitter, "Audio", audioDecoder, "Input", true); + FilterGraphTools.RenderPin(graphBuilder, audioDecoder, "Output"); + } else { + logger.Warn("Audio pin not available"); + } + } + catch (Exception ex) { + logger.Warn("Audio pin not available"); } + logger.Debug("SaveSizeInfo"); SaveSizeInfo(sampGrabber as ISampleGrabber); + logger.Debug("SetTimeCodes"); SetTimeCodes(); + logger.Debug("ConfigureVideoWindow"); ConfigureVideoWindow(); + logger.Debug("Enable YADIF deinterlace"); ILAVVideoSettings settings = (ILAVVideoSettings)videoDecoder; //settings.SetSWDeintMode(LAVSWDeintModes.SWDeintMode_None); settings.SetSWDeintMode(LAVSWDeintModes.SWDeintMode_YADIF); @@ -389,11 +418,11 @@ namespace DxPlay { MXFFile mxf = new MXFFile(MediaDescription.FileName); mxf.Inspect(); //MediaDescription.firstFrame = new Timecode(mxf.FirstSystemItem?.UserDateFullFrameNb, MediaDescription.FrameRate); - MediaDescription.firstFrame = new Timecode((int) mxf.TimecodeComponent.StartTimecode,(float) mxf.TimecodeComponent.RoundedTimecodeBase); + MediaDescription.firstFrame = new Timecode((int)mxf.TimecodeComponent.StartTimecode, (float)mxf.TimecodeComponent.RoundedTimecodeBase); } catch (Exception ex) { MediaDescription.firstFrame = new Timecode(); - } + } } //MediaDescription.firstFrame = new Timecode(); CurrentTC = new Timecode(MediaDescription.FirstFrame); diff --git a/client/Maestro/Configuration/configuration-playout-ingest.json b/client/Maestro/Configuration/-configuration-playout-ingest.json similarity index 100% rename from client/Maestro/Configuration/configuration-playout-ingest.json rename to client/Maestro/Configuration/-configuration-playout-ingest.json diff --git a/client/Maestro/Configuration/configuration-tqc-check.json b/client/Maestro/Configuration/configuration-tqc-check.json new file mode 100644 index 00000000..6989f703 --- /dev/null +++ b/client/Maestro/Configuration/configuration-tqc-check.json @@ -0,0 +1,98 @@ +{ + "title": "TQC check", + "active": true, + "startInTray": false, + "enableCustomMetadataId": true, + "player": { + "enabled": true, + "autoStart": false, + "segmentEditor": true + }, + "source": { + "$type": "UNCSource", + "filter": "avi,wav,mxf", + "local": { + "address": "file://10.10.1.100/BRAAVOS/TQC/CHECK", + "userName": "mediacube", + "password": "Dn8t4gfHcK98o8hyPgLDhr5SgSji4JCxsfpMJsODikUp3nXgrM0UNCi45lLAK8ZOnmEneO44P9qpJ4QDqhctN6MxZodjJgdZTyoZKmSa+ECzEzLr/wPYNgxVaXrVotEy", + "timeout": 1000 + }, + "remote": { + "address": "ftp://10.10.1.100/TQC/CHECK", + "userName": "mediacube", + "password": "Dn8t4gfHcK98o8hyPgLDhr5SgSji4JCxsfpMJsODikUp3nXgrM0UNCi45lLAK8ZOnmEneO44P9qpJ4QDqhctN6MxZodjJgdZTyoZKmSa+ECzEzLr/wPYNgxVaXrVotEy", + "timeout": 1000 + } + }, + "metadatas": [ + { + "$type": "TrafficMetadata", + "server": { + "address": "Data Source=10.10.1.45\\sql16;Initial Catalog=PA_Echo;Persist Security Info=True;", + "userName": "MAM", + "password": "7RKZYBzumKjL40SJwuwiFCvX57xuCN8zay6OttUm2wbrgImyYZBHyZTUUYrXX31Ge2Uwew07HYsqh2uzdJeDBDwcVntxaHg3nIpv9Dyq/odVoiC4tUF/K+lgvKWANcrZ", + "timeout": 1000 + } + }, + { + "$type": "MediaCubeMetadata", + "server": { + "address": "http://10.10.1.27/services/rest/jobengine/", + "timeout": 1000 + } + } + ], + "targets": [ + { + "label": "Adáskész", + "processor": "FXPTargetProcessor", + "outputFormat": "%ID%", + "saveSegments": true, + "tag": "Adáskész", + "disableFileVersioning": true, + "enableOverride": true, + "saveMorpheusMetadata": true, + "pathMorpheusMetadata": "/PLAYOUT/MorpheusGateway/Input", + "deviceIdMorpheus": "ISILON", + "remote": { + "address": "ftp://10.10.1.100/PLAYOUT/Video", + "userName": "mediacube", + "password": "Dn8t4gfHcK98o8hyPgLDhr5SgSji4JCxsfpMJsODikUp3nXgrM0UNCi45lLAK8ZOnmEneO44P9qpJ4QDqhctN6MxZodjJgdZTyoZKmSa+ECzEzLr/wPYNgxVaXrVotEy", + "timeout": 1000 + } + }, + { + "label": "Újravágás", + "processor": "FXPTargetProcessor", + "subFolderFormat": "%IDROOT%-%TEXT%", + "outputFormat": "%ID%-%TEXT%", + "tag": "Újravágás", + "disableFileVersioning": true, + "enableOverride": true, + "remote": { + "address": "ftp://10.10.1.100/PLAYOUT_NLE", + "userName": "mediacube", + "password": "Dn8t4gfHcK98o8hyPgLDhr5SgSji4JCxsfpMJsODikUp3nXgrM0UNCi45lLAK8ZOnmEneO44P9qpJ4QDqhctN6MxZodjJgdZTyoZKmSa+ECzEzLr/wPYNgxVaXrVotEy", + "timeout": 1000 + }, + "sendEmailOnSuccess": true, + "successEmailRecipient": "vasary@elgekko.net", + "successEmailPattern": "A %TARGETNAME% újravágása szükséges." + }, + { + "label": "Archiválás", + "processor": "FXPTargetProcessor", + "outputFormat": "%ID%", + "tag": "Archiválás", + "saveArchiveMetadata": true, + "disableFileVersioning": true, + "enableOverride": true, + "remote": { + "address": "ftp://10.10.1.100/ARCHIVE", + "userName": "mediacube", + "password": "Dn8t4gfHcK98o8hyPgLDhr5SgSji4JCxsfpMJsODikUp3nXgrM0UNCi45lLAK8ZOnmEneO44P9qpJ4QDqhctN6MxZodjJgdZTyoZKmSa+ECzEzLr/wPYNgxVaXrVotEy", + "timeout": 1000 + } + } + ] +} diff --git a/client/Maestro/Maestro.csproj b/client/Maestro/Maestro.csproj index ee0d9d0b..69d690b4 100644 --- a/client/Maestro/Maestro.csproj +++ b/client/Maestro/Maestro.csproj @@ -270,6 +270,9 @@ Always + + Always + Always @@ -282,7 +285,7 @@ Always - + Always diff --git a/server/-product/log4j2.xml b/server/-product/log4j2.xml index 08bb456e..6080b766 100644 --- a/server/-product/log4j2.xml +++ b/server/-product/log4j2.xml @@ -1,12 +1,13 @@ - /opt/mediacube/log/mediacube.log - /opt/mediacube/log/$${date:yyyy-MM}/mediacube-%d{MM-dd-yyyy}-%i.log.gz - /opt/mediacube/log/markered-mediacube.log - /opt/mediacube/log/$${date:yyyy-MM}/markered-mediacube-%d{MM-dd-yyyy}-%i.log.gz - /opt/mediacube/log/mediacube-err.log - /opt/mediacube/log/$${date:yyyy-MM}/mediacube-err-%d{MM-dd-yyyy}-%i.log.gz + /opt/mediacube/log + ${logPath}/mediacube.log + ${logPath}/$${date:yyyy-MM}/mediacube-%d{MM-dd-yyyy}-%i.log.gz + ${logPath}/markered-mediacube.log + ${logPath}/$${date:yyyy-MM}/markered-mediacube-%d{MM-dd-yyyy}-%i.log.gz + ${logPath}/mediacube-err.log + ${logPath}/$${date:yyyy-MM}/mediacube-err-%d{MM-dd-yyyy}-%i.log.gz diff --git a/server/-product/mediacube.product b/server/-product/mediacube.product index 02f30950..8aff6869 100644 --- a/server/-product/mediacube.product +++ b/server/-product/mediacube.product @@ -28,34 +28,34 @@ -Djobengine.loglevel=INFO -Djetty.etc.config.urls=etc/user-jetty.xml,etc/user-jetty-ssl.xml,etc/user-jetty-ssl-context.xml,etc/user-jetty-http.xml,etc/user-jetty-https.xml - #Database --Djobengine.db.url=jdbc:db2://10.10.1.27:50000/mc:retrieveMessagesFromServerOnGetMessage=true; --Djobengine.db.user=db2admin --Djobengine.db.password=password --Djobengine.nosql.db.url=jdbc:db2://10.10.1.27:50000/mc:retrieveMessagesFromServerOnGetMessage=true; --Djobengine.nosql.db.user=db2admin --Djobengine.nosql.db.password=password --Djobengine.nosql.db.schema=test - -#Octopus --Djobengine.octopus.api.address=http://10.10.1.11/api/v1 --Djobengine.octopus.api.user=mam --Djobengine.octopus.api.password=napocska --Djobengine.octopus.rundowns.name=test_rundowns --Djobengine.octopus.storyfolders.name=test_storyfolders --Djobengine.octopus.stories.name=test_stories - -#Selenio --Djobengine.selenio.address=http://10.10.1.71:44000/TranscodeMgrWS?wsdl --Djobengine.selenio.projectfilepath=\\10.10.1.71\Data\Blueprints\MP4_H264_AAC.zenium - -#Nexio --Djobengine.nexio.db.url=jdbc:sqlserver://10.10.1.59:1433;databaseName=NXDB; --Djobengine.nexio.db.user=sa --Djobengine.nexio.db.password=resolve --Dnexio.useMOSGateway=true --Dnexio.host=10.10.1.55 --Dnexio.disable=true + #Database +-Djobengine.db.url=jdbc:db2://10.10.1.27:50000/mc:retrieveMessagesFromServerOnGetMessage=true; +-Djobengine.db.user=db2admin +-Djobengine.db.password=password +-Djobengine.nosql.db.url=jdbc:db2://10.10.1.27:50000/mc:retrieveMessagesFromServerOnGetMessage=true; +-Djobengine.nosql.db.user=db2admin +-Djobengine.nosql.db.password=password +-Djobengine.nosql.db.schema=test + +#Octopus +-Djobengine.octopus.api.address=http://10.10.1.11/api/v1 +-Djobengine.octopus.api.user=mam +-Djobengine.octopus.api.password=napocska +-Djobengine.octopus.rundowns.name=rundowns +-Djobengine.octopus.storyfolders.name=storyfolders +-Djobengine.octopus.stories.name=stories + +#Selenio +-Djobengine.selenio.address=http://10.10.1.71:44000/TranscodeMgrWS?wsdl +-Djobengine.selenio.projectfilepath=\\10.10.1.71\Data\Blueprints\MP4_H264_AAC.zenium + +#Nexio +-Djobengine.nexio.db.url=jdbc:sqlserver://10.10.1.59:1433;databaseName=NXDB; +-Djobengine.nexio.db.user=sa +-Djobengine.nexio.db.password=resolve +-Dnexio.useMOSGateway=true +-Dnexio.host=10.10.1.55 +-Dnexio.disable=false -XstartOnFirstThread -Dorg.eclipse.swt.internal.carbon.smallFonts diff --git a/server/user.jobengine.executors/config/scheduledjobs.json b/server/user.jobengine.executors/config/scheduledjobs.json index 044007dd..87fb7660 100644 --- a/server/user.jobengine.executors/config/scheduledjobs.json +++ b/server/user.jobengine.executors/config/scheduledjobs.json @@ -1,26 +1,16 @@ {"joblist":[ { - "active": false, + "active": true, + "executeimmediate": true, "name" : "OCTOPUS adatok szinkronizálása", "template": "sync-octopus.xml", - "executeimmediate": false, "cronexpression": "0/30 * * * * ?", "parameters": [ {"name": "includeArchived", "value": false, "type": "java.lang.Boolean"} ] }, { - "active": false, - "name" : "OCTOPUS adatok szinkronizálása (archivált tükrök is)", - "template": "sync-octopus.xml", - "executeimmediate": true, - "cronexpression": "0/30 * * * * ?", - "parameters": [ - {"name": "includeArchived", "value": true, "type": "java.lang.Boolean"} - ] - }, - { - "active": false, + "active": true, "executeimmediate": false, "name" : "Archiválás az ISILON/ARCHIVE mappából", "template": "archive-ondemand.xml", @@ -34,10 +24,10 @@ ] }, { - "active": false, + "active": true, + "executeimmediate": false, "name" : "Lejárt ISISLON/ARCHIVE anyagok törlése", "template": "delete-materials.xml", - "executeimmediate": false, "cronexpression": "0 0 5 * * ?", "parameters": [ {"name": "sourcePath", "value": "/mnt/ISILON/ARCHIVE", "type": "java.lang.String"} @@ -45,9 +35,9 @@ }, { "active": false, + "executeimmediate": true, "name" : "NEXIO anyagok másolása az ISILON/ARCHIVE mappába", "template": "copyforarchive-nexio-materials.xml", - "executeimmediate": true, "cronexpression": "0 0 0/1 1/1 * ? *", "parameters": [ {"name": "nexioPort", "value": 2098, "type": "java.lang.Integer"}, @@ -63,9 +53,9 @@ }, { "active": false, + "executeimmediate": true, "name" : "Archiválás az ISILON/ARCHIVE/ONE mappából", "template": "archive-ondemand.xml", - "executeimmediate": true, "cronexpression": "0 0 0/1 1/1 * ? *", "parameters": [ {"name": "sourcePath", "value": "/mnt/ISILON/ARCHIVE/ONE", "type": "java.lang.String"}, @@ -77,9 +67,9 @@ }, { "active": false, + "executeimmediate": true, "name" : "TRAFFIC anyagok visszatöltése", "template": "retrieve-traffic-missing-materials.xml", - "executeimmediate": true, "cronexpression": "0 0 0/1 1/1 * ? *", "parameters": [ {"name": "dbUrl", "value": "jdbc:sqlserver://10.10.1.45\\sql16;databaseName=PA_Echo;", "type": "java.lang.String"}, @@ -92,9 +82,9 @@ }, { "active": false, + "executeimmediate": true, "name" : "MORPHEUS anyagok visszatöltése", "template": "retrieve-morpheus-missing-materials.xml", - "executeimmediate": true, "cronexpression": "0 0 0/1 1/1 * ? *", "parameters": [ {"name": "sourcePath", "value": "/mnt/MORPHEUS", "type": "java.lang.String"}, @@ -104,9 +94,9 @@ }, { "active": false, + "executeimmediate": true, "name" : "Lejárt ISISLON/ARCHIVE_TEMP anyagok törlése", "template": "delete-materials.xml", - "executeimmediate": true, "cronexpression": "0 0 0/1 1/1 * ? *", "parameters": [ {"name": "sourcePath", "value": "/mnt/ISILON/ARCHIVE_TEMP", "type": "java.lang.String"} @@ -114,9 +104,9 @@ }, { "active": false, + "executeimmediate": true, "name" : "NEXIO anyagok törlése", "template": "delete-nexio-materials.xml", - "executeimmediate": true, "cronexpression": "0 0 0/1 1/1 * ? *", "parameters": [ {"name": "port", "value": 2098, "type": "java.lang.Integer"}, @@ -127,17 +117,17 @@ }, { "active": false, + "executeimmediate": false, "name" : "Párhuzamosított teszt folyamat", "template": "fake-concurrent.xml", - "executeimmediate": false, "cronexpression": "0/10 * * ? * *", "parameters": [ {"name": "itemID", "value": 1, "type": "java.lang.Long"} ] }, { "active": false, + "executeimmediate": false, "name" : "Teszt folyamat", "template": "fake.xml", - "executeimmediate": false, "cronexpression": "0/10 * * ? * *", "parameters": [ {"name": "itemID", "value": 1, "type": "java.lang.Long"} ] } diff --git a/server/user.jobengine.executors/src/user/jobengine/server/steps/ArchiveListBuilderStep.java b/server/user.jobengine.executors/src/user/jobengine/server/steps/ArchiveListBuilderStep.java index 8f68e6fa..01aba298 100644 --- a/server/user.jobengine.executors/src/user/jobengine/server/steps/ArchiveListBuilderStep.java +++ b/server/user.jobengine.executors/src/user/jobengine/server/steps/ArchiveListBuilderStep.java @@ -30,7 +30,8 @@ public class ArchiveListBuilderStep extends JobStep { private static final Logger logger = LogManager.getLogger(); // private static final String UTF8 = "utf-8"; private static final String STATUSFOLDER = ".STATUS"; - private static final String MEDIAEXT = ".mxf"; + private static final String MXFEXT = ".mxf"; + private static final String WAVEXT = ".wav"; private static final String JSONEXT = ".json"; private static final String CATCHEDEXT = ".catched"; @@ -109,7 +110,8 @@ public class ArchiveListBuilderStep extends JobStep { private void processPathItem(Path mediaFilePath, final List archiveList) { File mediaFile = mediaFilePath.toFile(); - if (mediaFile.isDirectory() || !mediaFile.getName().toLowerCase().endsWith(MEDIAEXT.toLowerCase())) { + if (mediaFile.isDirectory() || !mediaFile.getName().toLowerCase().endsWith(MXFEXT.toLowerCase()) + || mediaFile.getName().toLowerCase().endsWith(WAVEXT.toLowerCase())) { return; } diff --git a/server/user.jobengine.executors/src/user/jobengine/server/steps/CopyForArchiveNEXIOMaterialsStep.java b/server/user.jobengine.executors/src/user/jobengine/server/steps/CopyForArchiveNEXIOMaterialsStep.java index 655e8cf8..ad38685f 100644 --- a/server/user.jobengine.executors/src/user/jobengine/server/steps/CopyForArchiveNEXIOMaterialsStep.java +++ b/server/user.jobengine.executors/src/user/jobengine/server/steps/CopyForArchiveNEXIOMaterialsStep.java @@ -107,7 +107,7 @@ public class CopyForArchiveNEXIOMaterialsStep extends JobStep { // "userName": "echotest" // } BasicDBObject result = new BasicDBObject(); - result.put("itemHouseID", rundownArchive.getItemHouseId()); + result.put("itemHouseId", rundownArchive.getItemHouseId()); result.put("itemTitle", rundownArchive.getItemTitle()); result.put("itemDescription", rundownArchive.getItemDesc()); result.put("userName", "mediacube"); @@ -193,7 +193,8 @@ public class CopyForArchiveNEXIOMaterialsStep extends JobStep { BasicDBObject clip = (BasicDBObject) clips.findOne(new BasicDBObject(LONGNAMEID, mosID)); if (clip == null) { logger.debug("File NOT exists {}", mosID); - throw new Exception(String.format("File NOT exists %s", mosID)); + return null; + //throw new Exception(String.format("File NOT exists %s", mosID)); } else { logger.debug("File exists {}", mosID); } @@ -245,10 +246,10 @@ public class CopyForArchiveNEXIOMaterialsStep extends JobStep { for (DBObject r : rundowns) { BasicDBObject rundown = (BasicDBObject) r; setProgress(index * 100 / rundowns.size()); + long rundownID = NoSQLUtils.asLong(rundown, IOctopusAPI.ID); + String rundownName = NoSQLUtils.asString(rundown, IOctopusAPI.NAME); try { - long rundownID = NoSQLUtils.asLong(rundown, IOctopusAPI.ID); BasicDBObject currentRundownID = new BasicDBObject(IOctopusAPI.ID, rundownID); - String rundownName = rundown.getString(IOctopusAPI.NAME); if (archivedRundowns != null && archivedRundowns.contains(currentRundownID)) { logger.info("Skipping archived rundown {} {}", rundownID, rundownName); continue; @@ -256,7 +257,7 @@ public class CopyForArchiveNEXIOMaterialsStep extends JobStep { RundownArchive rundownArchive = processRundow(r); if (rundownArchive == null || rundownArchive.isEmpty()) { - logger.info("Skipping rundown {} {}", NoSQLUtils.asLong(rundown, IOctopusAPI.ID), rundown.getString(IOctopusAPI.NAME)); + logger.info("Skipping rundown {} {}", rundownID, rundownName); continue; } @@ -267,7 +268,8 @@ public class CopyForArchiveNEXIOMaterialsStep extends JobStep { } catch (Exception e) { logger.catching(e); logger.error(getMarker(), - "A {} tükör archiválása nem lehetséges, mert a annak ellenőrzése hibát jelzett. A rendszer üzenete: " + e.getMessage()); + String.format("A %s %s tükör archiválása nem lehetséges, mert a annak ellenőrzése hibát jelzett. A rendszer üzenete: %s", rundownID, + rundownName, e.getMessage())); } index++; } diff --git a/server/user.jobengine.executors/src/user/jobengine/server/steps/CreateMissingLowresStep.java b/server/user.jobengine.executors/src/user/jobengine/server/steps/CreateMissingLowresStep.java index cc38abec..ca0d2f83 100644 --- a/server/user.jobengine.executors/src/user/jobengine/server/steps/CreateMissingLowresStep.java +++ b/server/user.jobengine.executors/src/user/jobengine/server/steps/CreateMissingLowresStep.java @@ -1,18 +1,55 @@ package user.jobengine.server.steps; +import java.nio.file.Files; +import java.nio.file.Paths; +import java.util.List; + import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; -import user.jobengine.db.IItemManager; +import user.jobengine.db.ItemDAOEx; +import user.jobengine.db.ItemManager; +import user.jobengine.db.Media; import user.jobengine.server.IJobEngine; import user.jobengine.server.IJobRuntime; public class CreateMissingLowresStep extends JobStep { private static final Logger logger = LogManager.getLogger(); - private IItemManager manager; + private static final String targetNamePattern = "%s"; + private String workPath = "/mnt/ISILON/ARCHIVE_RESTORE"; + private String transcoderPath = "/mnt/SELENIO"; + private String webPath = "/mediacube/data/lowres/www/video"; @StepEntry public Object[] execute(IJobEngine jobEngine, IJobRuntime jobRuntime) throws Exception { + ItemDAOEx dao = new ItemDAOEx((ItemManager) jobEngine.getItemManager()); + List medias = dao.getUntranscodedMedias(); + if (medias == null) + return null; + + int i = 1; + for (Media media : medias) { + String name = media.getMediaFilesName(); + logger.info("Processing {}", name); + restore(jobEngine, jobRuntime, media); + transcode(jobEngine, jobRuntime, media); + Files.delete(Paths.get(workPath, name)); + setProgress(i * 100 / medias.size()); + i++; + } + return null; } + + private void restore(IJobEngine jobEngine, IJobRuntime jobRuntime, Media media) throws Exception { + TSMRestoreStep restore = new TSMRestoreStep(); + restore.execute(media, workPath, targetNamePattern, null, 0, jobEngine, jobRuntime); + } + + private void transcode(IJobEngine jobEngine, IJobRuntime jobRuntime, Media media) throws Exception { + TranscodeSELENIOStep selenio = new TranscodeSELENIOStep(); + ArchiveItem archiveItem = new ArchiveItem(); + archiveItem.setMediaFile(media.getMediaFilesName()); + selenio.execute(workPath, archiveItem, media, transcoderPath, webPath, jobEngine, jobRuntime); + } } diff --git a/server/user.jobengine.executors/src/user/jobengine/server/steps/OctopusDataMiner.java b/server/user.jobengine.executors/src/user/jobengine/server/steps/OctopusDataMiner.java index 307b8dbb..f710e641 100644 --- a/server/user.jobengine.executors/src/user/jobengine/server/steps/OctopusDataMiner.java +++ b/server/user.jobengine.executors/src/user/jobengine/server/steps/OctopusDataMiner.java @@ -73,6 +73,8 @@ public class OctopusDataMiner { private String apiUser; private String apiPwd; private HashSet storyIDs = new HashSet<>(); + private HashSet folderIDs = new HashSet<>(); + private HashSet rundownIDs = new HashSet<>(); private EventListenerList progressListenerList; private ProgressEvent progressEvent = new ProgressEvent(this, 0); private Map storyRundowns; @@ -115,9 +117,12 @@ public class OctopusDataMiner { List stories = NoSQLUtils.asList(storyFolderWithStoryIds, IOctopusAPI.STORIES); if (stories == null) continue; + if (!folderIDs.contains(storyFolderId)) + folderIDs.add(storyFolderId); long position = 1; for (BasicDBObject story : stories) { long storyId = story.getLong(IOctopusAPI.ID); + BasicDBList references = result.get(storyId); if (references == null) { references = new BasicDBList(); @@ -140,6 +145,10 @@ public class OctopusDataMiner { List slugs = NoSQLUtils.asList(rundownWithStoryids, IOctopusAPI.SLUGS); if (slugs == null) continue; + + if (!rundownIDs.contains(rundownId)) + rundownIDs.add(rundownId); + for (BasicDBObject slug : slugs) { if (!slug.containsKey(IOctopusAPI.STORYID)) continue; @@ -283,9 +292,9 @@ public class OctopusDataMiner { processRundowns(rundowns); processStoryFolders(storyFolders); - deleteDiff(currentRundowns.keySet(), storyRundowns.keySet(), RUNDOWN_COLLECTION); - deleteDiff(currentFolder.keySet(), storyStoryFolders.keySet(), RUNDOWN_COLLECTION); - deleteDiff(currentStories.keySet(), storyIDs, RUNDOWN_COLLECTION); + deleteDiff(currentRundowns.keySet(), rundownIDs, RUNDOWN_COLLECTION); + deleteDiff(currentFolder.keySet(), folderIDs, FOLDER_COLLECTION); + deleteDiff(currentStories.keySet(), storyIDs, STORY_COLLECTION); // deleteOrphanRundowns(); // deleteOrphanStoryFolders(); // deleteOrphanStories(); @@ -321,7 +330,7 @@ public class OctopusDataMiner { // } logger.trace(FINISHED); - throw new Exception("Mérés"); + //throw new Exception("Mérés"); } private String extractContent(BasicDBObject content) { diff --git a/server/user.jobengine.executors/src/user/jobengine/server/steps/TranscodeSELENIOStep.java b/server/user.jobengine.executors/src/user/jobengine/server/steps/TranscodeSELENIOStep.java index 5a9dce91..b1cb4701 100644 --- a/server/user.jobengine.executors/src/user/jobengine/server/steps/TranscodeSELENIOStep.java +++ b/server/user.jobengine.executors/src/user/jobengine/server/steps/TranscodeSELENIOStep.java @@ -179,14 +179,21 @@ public class TranscodeSELENIOStep extends JobStep { List outputs = of.getOutputFiles(); if (outputs.size() != 1) throw new IndexOutOfBoundsException("There are more than 1 file in the response: " + transcodeTask.getId()); - user.commons.selenio.wsclient.MediaFile selenioMediaFile = outputs.get(0); String outFile = selenioMediaFile.getFile(); - outFile = outFile.substring(outFile.lastIndexOf("\\") + 1); - user.jobengine.db.MediaFile mediaFile = manager.createMediaFile(outFile, fileType, store, mediaCubeMedia); - mediaFile.add(); try { - Files.move(Paths.get(transcoderTargetPath, outFile), Paths.get(webPath, outFile), StandardCopyOption.REPLACE_EXISTING); + + outFile = outFile.substring(outFile.lastIndexOf("\\") + 1); + if (outFile.indexOf(".") > 2) { + Path subdir = Paths.get(outFile.substring(0, 1), outFile.substring(1, 2), outFile.substring(2, 3)); + manager.createMediaFile(Paths.get(subdir.toString(), outFile).toString(), fileType, store, mediaCubeMedia).add(); + subdir = Paths.get(webPath, subdir.toString()); + subdir.toFile().mkdirs(); + Files.move(Paths.get(transcoderTargetPath, outFile), Paths.get(subdir.toString(), outFile), StandardCopyOption.REPLACE_EXISTING); + } else { + manager.createMediaFile(outFile, fileType, store, mediaCubeMedia).add(); + Files.move(Paths.get(transcoderTargetPath, outFile), Paths.get(webPath, outFile), StandardCopyOption.REPLACE_EXISTING); + } } catch (IOException e) { logger.catching(e); logger.error(getMarker(), "A {} állomány mozgatása a {} mappába nem sikerült.", outFile, webPath); diff --git a/server/user.jobengine.osgi.commons/test/user/common/octopus/test/OctopusDataMinerTest.java b/server/user.jobengine.osgi.commons/test/user/common/octopus/test/OctopusDataMinerTest.java index b2957a28..9818a7ad 100644 --- a/server/user.jobengine.osgi.commons/test/user/common/octopus/test/OctopusDataMinerTest.java +++ b/server/user.jobengine.osgi.commons/test/user/common/octopus/test/OctopusDataMinerTest.java @@ -101,7 +101,7 @@ public class OctopusDataMinerTest { @Test public void testClient() { - ResteasyWebTarget webTarget = new ResteasyClientBuilder().build().target("http://10.10.1.29/services/rest/octopus/"); + ResteasyWebTarget webTarget = new ResteasyClientBuilder().build().target("http://10.10.1.28/services/rest/octopus/"); Response response = null; for (int i = 0; i < 10000; i++) { response = webTarget.path("rundowns").request().get(); diff --git a/server/user.jobengine.osgi.db/.classpath b/server/user.jobengine.osgi.db/.classpath index cf38f8c6..ad20a72a 100644 --- a/server/user.jobengine.osgi.db/.classpath +++ b/server/user.jobengine.osgi.db/.classpath @@ -1,9 +1,9 @@ - + diff --git a/server/user.jobengine.osgi.db/.settings/org.eclipse.jdt.core.prefs b/server/user.jobengine.osgi.db/.settings/org.eclipse.jdt.core.prefs index 66737877..295926d9 100644 --- a/server/user.jobengine.osgi.db/.settings/org.eclipse.jdt.core.prefs +++ b/server/user.jobengine.osgi.db/.settings/org.eclipse.jdt.core.prefs @@ -1,8 +1,7 @@ -#Mon Feb 06 13:26:25 CET 2012 eclipse.preferences.version=1 org.eclipse.jdt.core.compiler.codegen.inlineJsrBytecode=enabled -org.eclipse.jdt.core.compiler.codegen.targetPlatform=1.7 -org.eclipse.jdt.core.compiler.compliance=1.7 +org.eclipse.jdt.core.compiler.codegen.targetPlatform=1.8 +org.eclipse.jdt.core.compiler.compliance=1.8 org.eclipse.jdt.core.compiler.problem.assertIdentifier=error org.eclipse.jdt.core.compiler.problem.enumIdentifier=error -org.eclipse.jdt.core.compiler.source=1.7 +org.eclipse.jdt.core.compiler.source=1.8 diff --git a/server/user.jobengine.osgi.db/.settings/org.hibernate.eclipse.console.prefs b/server/user.jobengine.osgi.db/.settings/org.hibernate.eclipse.console.prefs new file mode 100644 index 00000000..8fdae10c --- /dev/null +++ b/server/user.jobengine.osgi.db/.settings/org.hibernate.eclipse.console.prefs @@ -0,0 +1,3 @@ +default.configuration= +eclipse.preferences.version=1 +hibernate3.enabled=false diff --git a/server/user.jobengine.osgi.db/src/user/jobengine/db/ItemDAOEx.java b/server/user.jobengine.osgi.db/src/user/jobengine/db/ItemDAOEx.java index b60264e1..aa13dd6f 100644 --- a/server/user.jobengine.osgi.db/src/user/jobengine/db/ItemDAOEx.java +++ b/server/user.jobengine.osgi.db/src/user/jobengine/db/ItemDAOEx.java @@ -12,6 +12,7 @@ import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import sqlj.runtime.ref.DefaultContext; +import user.commons.ListUtils; public class ItemDAOEx { private static final Logger logger = LogManager.getLogger(); @@ -103,6 +104,47 @@ public class ItemDAOEx { return results; } + public List getUntranscodedMedias() { + List result = null; + + ResultSet rs = null; + PreparedStatement st = null; + DefaultContext context = manager.getDbContext(); + Connection connection = context.getConnection(); + try { + + String query = "select m.ID, m.TITLE, m.DESCRIPTION, m.ITEMID, m.ITEMTYPEID, m.HOUSEID, m.LENGTH, m.CREATED, m.ARCHIVED, m.POSTER from MEDIA m right outer join VW_MEDIAFILES f on (f.MEDIAID = m.ID) where m.CREATED > '2017-12-14' and f.MEDIAFILECOUNT = 1"; + st = connection.prepareStatement(query); + rs = st.executeQuery(); + MediaDAO dao = (MediaDAO) manager.getBaseDAO(Media.class); + result = ListUtils.cast(dao.entities(context, rs)); + if (result != null) + result.forEach(m -> m.setPersister(manager)); + connection.commit(); + } catch (Exception e) { + try { + connection.rollback(); + } catch (Exception e1) { + e1.printStackTrace(); + } + manager.throwError(e); + } finally { + try { + if (rs != null) + rs.close(); + } catch (Exception e1) { + } + try { + if (st != null) + st.close(); + } catch (Exception e1) { + } + manager.putDbContext(context); + } + + return result; + } + @SuppressWarnings("unchecked") public List search(QueryObject queryObject) { manager.traceIn(); diff --git a/server/user.jobengine.osgi.server/src/user/jobengine/zk/model/SearchModel.java b/server/user.jobengine.osgi.server/src/user/jobengine/zk/model/SearchModel.java index f1b6618d..2c4bf4d6 100644 --- a/server/user.jobengine.osgi.server/src/user/jobengine/zk/model/SearchModel.java +++ b/server/user.jobengine.osgi.server/src/user/jobengine/zk/model/SearchModel.java @@ -1,6 +1,6 @@ package user.jobengine.zk.model; -import java.io.File; +import java.net.URL; import java.nio.file.Paths; import java.util.HashMap; import java.util.List; @@ -28,6 +28,7 @@ public class SearchModel extends BaseModel { private static final Logger logger = LogManager.getLogger(); private static final String MEDIACUBEMEDIA = "mediaCubeMedia"; private static final String PAGES_RETRIEVESELECTOR_ZUL = "/pages/retrieveselector.zul"; + private static final String PAGES_MEDIAPLAYER = "/pages/mediaplayer.jsp?mediaurl="; private String searchValue = null; protected CachedListModel searchResult = null; @@ -100,8 +101,11 @@ public class SearchModel extends BaseModel { Store store = mf.getStore(); if (store.isLowres()) { StoreUri sourceStoreUri = store.getSourceStoreUri(RemoteStoreProtocol.HTTP); - File f = new File("file://" + mf.getRelativePath().replace("\\", "/")); - ret = String.format("%s%s%s", "/pages/mediaplayer.jsp?mediaurl=", sourceStoreUri.toString(), f.getName()).toString(); + URL base = new URL(sourceStoreUri.toString()); + String relative = mf.getRelativePath().replace("\\", "/"); + ret = PAGES_MEDIAPLAYER + new URL(base, relative).toString(); + // File f = new File("file://" + mf.getRelativePath().replace("\\", "/")); + // ret = String.format("%s%s%s", "/pages/mediaplayer.jsp?mediaurl=", sourceStoreUri.toString(), f.getName()).toString(); break; } } diff --git a/server/user.jobengine.osgi.server/src/user/jobengine/zk/util/AuthInitiator.java b/server/user.jobengine.osgi.server/src/user/jobengine/zk/util/AuthInitiator.java index fe803dfe..218466b0 100644 --- a/server/user.jobengine.osgi.server/src/user/jobengine/zk/util/AuthInitiator.java +++ b/server/user.jobengine.osgi.server/src/user/jobengine/zk/util/AuthInitiator.java @@ -1,25 +1,22 @@ package user.jobengine.zk.util; - import java.util.Map; -import org.zkoss.zk.ui.*; -import org.zkoss.zk.ui.util.Initiator; +import org.zkoss.zk.ui.Page; +import org.zkoss.zk.ui.util.Initiator; /** - Ellenorzi, hogy van e beloginelt user, ha nincs, - akkor -> login.zul - Minden oldal belinkeli ezt az ellenorzest. -*/ + * Ellenorzi, hogy van e beloginelt user, ha nincs, akkor -> login.zul Minden oldal belinkeli ezt az ellenorzest. + */ public class AuthInitiator implements Initiator { - - public void doInit(Page page, Map args) throws Exception{ - //nincs beloginelt felhasznalo -// if(SessionUtil.getAttribute(SessionUtil.PRINCIPALNAME) == null){ -// Executions.sendRedirect("/index.jsp"); -// return; -// } - } - + + @Override + public void doInit(Page page, Map args) throws Exception { + //nincs beloginelt felhasznalo + // if(SessionUtil.getAttribute(SessionUtil.PRINCIPALNAME) == null){ + // Executions.sendRedirect("/index.jsp"); + // return; + // } + } } \ No newline at end of file diff --git a/server/user.jobengine.osgi.server/test/user/jobengine/server/steps/General.java b/server/user.jobengine.osgi.server/test/user/jobengine/server/steps/General.java new file mode 100644 index 00000000..8ca2ccfc --- /dev/null +++ b/server/user.jobengine.osgi.server/test/user/jobengine/server/steps/General.java @@ -0,0 +1,26 @@ +package user.jobengine.server.steps; + +import java.net.URL; +import java.nio.file.Path; +import java.nio.file.Paths; + +import org.junit.Test; + +import user.jobengine.db.TestBase; + +public class General extends TestBase { + + @Test + public void test1() throws Exception { + String webPath = "c:\\xxxx"; + String outFile = "M012345.MXF"; + Path subdir = Paths.get(outFile.substring(0, 1), outFile.substring(1, 2), outFile.substring(2, 3)); + String rel = Paths.get(subdir.toString(), outFile).toString(); + System.out.println(rel); + subdir = Paths.get(webPath, subdir.toString()); + System.out.println(subdir.toString()); + //subdir.toFile().mkdirs(); + System.out.println(new URL(new URL("http://host/"), rel.replace("\\", "/")).toString()); + } + +} -- 2.54.0