From e3c02748293369330449d93e4226a413c6d33484 Mon Sep 17 00:00:00 2001 From: =?utf8?q?V=C3=A1s=C3=A1ry=20D=C3=A1niel?= Date: Tue, 19 Dec 2017 14:44:14 +0000 Subject: [PATCH] git-tfs-id: [http://tfs.userrendszerhaz.hu:8080/tfs/DefaultCollection]$/MediaCube;C30813 --- .../AudioRecorder/Properties/AssemblyInfo.cs | 4 +- client/DxPlay/DxPlayer.cs | 3 +- client/DxPlay/Properties/AssemblyInfo.cs | 4 +- client/MXFFileParser/MXFFile.cs | 271 +++++++++- .../Metadata/B17 MXFTimecodeComponent.cs | 2 +- .../MXFFileParser/Properties/AssemblyInfo.cs | 4 +- .../installforge-installer-project.ifp | Bin 4093 -> 4093 bytes client/Maestro/Properties/AssemblyInfo.cs | 4 +- client/MaestroShared/Metadata/Timecode.cs | 5 + ...h => deploy-bkup-mediacube-modules.launch} | 2 +- .../deploy-cluster-mediacube-modules.launch | 19 + .../run-mediacube-server-bsh.launch | 2 +- server/-configuration/scheduledjobs.json | 6 +- server/-modules/pom.xml | 5 +- server/-product/mediacube.product | 4 + server/-product/pom.xml | 5 +- .../config/scheduledjobs.json | 34 +- .../deploy-steps-to-bsh-bkup.bat | 32 ++ .../deploy-steps-to-bsh-bkup.launch | 8 + .../jobtemplates/sync-octopus.xml | 4 +- .../server/steps/OctopusDataMiner.java | 486 +++++++----------- .../server/steps/SyncOCTOPUSDataStep.java | 8 +- .../src/user/commons/octopus/IOctopusAPI.java | 6 +- ...sDataMiner.java => OctopusDataMiner1.java} | 4 +- .../octopus/test/OctopusDataMinerTest.java | 47 +- server/user.jobengine.osgi.db/sql/cleanup.sql | 12 +- .../server/actions/DoneSuspendAction.java | 3 + 27 files changed, 625 insertions(+), 359 deletions(-) rename server/-configuration/{deploy-mediacube-modules.launch => deploy-bkup-mediacube-modules.launch} (88%) create mode 100644 server/-configuration/deploy-cluster-mediacube-modules.launch create mode 100644 server/user.jobengine.executors/deploy-steps-to-bsh-bkup.bat create mode 100644 server/user.jobengine.executors/deploy-steps-to-bsh-bkup.launch rename server/user.jobengine.osgi.commons/src/user/commons/octopus/{OctopusDataMiner.java => OctopusDataMiner1.java} (97%) diff --git a/client/AudioRecorder/Properties/AssemblyInfo.cs b/client/AudioRecorder/Properties/AssemblyInfo.cs index 91267681..5516b1be 100644 --- a/client/AudioRecorder/Properties/AssemblyInfo.cs +++ b/client/AudioRecorder/Properties/AssemblyInfo.cs @@ -32,5 +32,5 @@ using System.Runtime.InteropServices; // You can specify all the values or you can default the Build and Revision Numbers // by using the '*' as shown below: // [assembly: AssemblyVersion("1.0.*")] -[assembly: AssemblyVersion("2.0.6.8")] -[assembly: AssemblyFileVersion("2.0.6.8")] +[assembly: AssemblyVersion("2.0.6.9")] +[assembly: AssemblyFileVersion("2.0.6.9")] diff --git a/client/DxPlay/DxPlayer.cs b/client/DxPlay/DxPlayer.cs index 200bb5e9..3d8d3247 100644 --- a/client/DxPlay/DxPlayer.cs +++ b/client/DxPlay/DxPlayer.cs @@ -388,7 +388,8 @@ namespace DxPlay { try { MXFFile mxf = new MXFFile(MediaDescription.FileName); mxf.Inspect(); - MediaDescription.firstFrame = new Timecode(mxf.FirstSystemItem?.UserDateFullFrameNb, MediaDescription.FrameRate); + //MediaDescription.firstFrame = new Timecode(mxf.FirstSystemItem?.UserDateFullFrameNb, MediaDescription.FrameRate); + MediaDescription.firstFrame = new Timecode((int) mxf.TimecodeComponent.StartTimecode,(float) mxf.TimecodeComponent.RoundedTimecodeBase); } catch (Exception ex) { MediaDescription.firstFrame = new Timecode(); diff --git a/client/DxPlay/Properties/AssemblyInfo.cs b/client/DxPlay/Properties/AssemblyInfo.cs index 177a4f0e..83b07a22 100644 --- a/client/DxPlay/Properties/AssemblyInfo.cs +++ b/client/DxPlay/Properties/AssemblyInfo.cs @@ -26,7 +26,7 @@ using System.Runtime.CompilerServices; // You can specify all the values or you can default the Revision and Build Numbers // by using the '*' as shown below: -[assembly: AssemblyVersion("1.0.*")] +[assembly: AssemblyVersion("2.0.6.9")] // // In order to sign your assembly you must specify a key to use. Refer to the @@ -56,3 +56,5 @@ using System.Runtime.CompilerServices; [assembly: AssemblyDelaySign(false)] [assembly: AssemblyKeyFile("")] [assembly: AssemblyKeyName("")] +[assembly: AssemblyFileVersion("2.0.6.9")] + diff --git a/client/MXFFileParser/MXFFile.cs b/client/MXFFileParser/MXFFile.cs index b4c146cb..c1f103db 100644 --- a/client/MXFFileParser/MXFFile.cs +++ b/client/MXFFileParser/MXFFile.cs @@ -47,7 +47,12 @@ namespace Myriadbits.MXF { /// Name object, represents a complete MXF file /// public class MXFFile : MXFObject { + public MXFTimecodeComponent TimecodeComponent { + get; internal set; + } + private MXFReader m_reader; + private List m_results; public string Filename { get; set; } public long Filesize { get; set; } @@ -55,9 +60,12 @@ namespace Myriadbits.MXF { public List Partitions { get; set; } public MXFRIP RIP { get; set; } + public List Results { get { return m_results; } } public MXFSystemItem FirstSystemItem { get; set; } public MXFSystemItem LastSystemItem { get; set; } + public List FlatList { get; set; } + public MXFLogicalObject LogicalBase { get; set; } @@ -68,19 +76,96 @@ namespace Myriadbits.MXF { public MXFFile(string fileName) { this.Filename = fileName; this.Partitions = new List(); + this.m_results = new List(); + } + + public void Inspect(FileParseOptions options = FileParseOptions.FastStartTC) { + switch (options) { + case FileParseOptions.Normal: + ParseFull(); + break; + case FileParseOptions.Fast: + ParsePartial(); + break; + case FileParseOptions.FastStartTC: + ParsePartial(true); + break; + } } - public void Inspect() { - ParsePartial(true); + /// + /// Fully Parse an MXF file + /// + protected void ParseFull() { + Stopwatch sw = Stopwatch.StartNew(); + + MXFKLVFactory klvFactory = new MXFKLVFactory(); + + MXFPartition currentPartition = null; + Dictionary allPrimerKeys = null; + int[] counters = new int[Enum.GetNames(typeof(KeyType)).Length]; + using (m_reader = new MXFReader(this.Filename)) { + this.Filesize = m_reader.Size; + MXFObject partitions = new MXFNamedObject("Partitions", 0); + this.AddChild(partitions); + + int partitionNumber = 0; // For easy partition identification + while (!m_reader.EOF) { + MXFKLV klv = klvFactory.CreateObject(m_reader, currentPartition); + + // Update overall counters + if (klv.Key.Type == KeyType.None) + counters[(int)klv.Key.Type]++; + + // Process the new KLV + ProcessKLVObject(klv, partitions, ref currentPartition, ref partitionNumber, ref allPrimerKeys); + + // Next KLV please + m_reader.Seek(klv.DataOffset + klv.Length); + } + } + // Progress should now be 90% + + // Update all type descriptions + klvFactory.UpdateAllTypeDescriptions(allPrimerKeys); + + Debug.WriteLine("Finished parsing file '{0}' in {1} ms", this.Filename, sw.ElapsedMilliseconds); + sw.Restart(); + + // Create a list with all UID keys + Dictionary allKeys = new Dictionary(); + CreateKeyList(allKeys, this); + + // Resolve the references + ResolveReferences(allKeys, this); + Debug.WriteLine("Finished resolving references in {0} ms", sw.ElapsedMilliseconds); + sw.Restart(); + + this.FlatList = new List(); + this.AddToList(this.FlatList); + Debug.WriteLine("Flatlist created in {0} ms", sw.ElapsedMilliseconds); + sw.Restart(); + + + // Create the logical tree + CreateLogicalTree(); + Debug.WriteLine("Logical tree created in {0} ms", sw.ElapsedMilliseconds); + sw.Restart(); + + // And Execute ALL test + this.ExecuteValidationTest(true); } + /// /// Partially Parse an MXF file, skip all data /// protected void ParsePartial(bool stopOnStartTC = false) { MXFKLVFactory klvFactory = new MXFKLVFactory(); + MXFPartition currentPartition = null; Dictionary allPrimerKeys = null; + int[] counters = new int[Enum.GetNames(typeof(KeyType)).Length]; PartialSeekMode seekMode = PartialSeekMode.Unknown; using (m_reader = new MXFReader(this.Filename)) { this.Filesize = m_reader.Size; @@ -98,6 +183,11 @@ namespace Myriadbits.MXF { while (!m_reader.EOF && seekMode != PartialSeekMode.Backwards) // Eof and NOT searching backwards { MXFKLV klv = klvFactory.CreateObject(m_reader, currentPartition); + if (klv is MXFTimecodeComponent && stopOnStartTC) { + TimecodeComponent = klv as MXFTimecodeComponent; + break; + } + // Update overall counters //if (klv.Key.Type == KeyType.None) @@ -113,9 +203,9 @@ namespace Myriadbits.MXF { // Process the new KLV - if (!ProcessKLVObject(klv, partitions, ref currentPartition, ref partitionNumber, ref allPrimerKeys, true)) + if (!ProcessKLVObjectOrig(klv, partitions, ref currentPartition, ref partitionNumber, ref allPrimerKeys)) break; - Debug.WriteLine("Processed partition {0}/{1} ", currentPartition, partitionNumber); + Debug.WriteLine("Processed partition {0}/{1} ", klv.Key.Name, partitionNumber); // If we found the second partition @@ -190,7 +280,7 @@ namespace Myriadbits.MXF { return false; } currentPartition.AddChild(klv); - } + } if (this.FirstSystemItem == null) { this.FirstSystemItem = klv as MXFSystemItem; @@ -200,6 +290,92 @@ namespace Myriadbits.MXF { return true; } + private bool ProcessKLVObjectOrig(MXFKLV klv, MXFObject partitions, ref MXFPartition currentPartition, ref int partitionNumber, ref Dictionary allPrimerKeys) { + // Is this a header, add to the partitions + switch (klv.Key.Type) { + case KeyType.Partition: + currentPartition = klv as MXFPartition; + currentPartition.File = this; + currentPartition.PartitionNumber = partitionNumber; + this.Partitions.Add(currentPartition); + partitions.AddChild(currentPartition); + partitionNumber++; + break; + + case KeyType.PrimerPack: + if (currentPartition != null) { + MXFPrimerPack primer = klv as MXFPrimerPack; + if (primer != null) // Just to be sure + { + // Let the partition know all primer keys + allPrimerKeys = primer.AllKeys; + currentPartition.PrimerKeys = primer.AllKeys; + } + currentPartition.AddChild(klv); // Add the primer + } + break; + + case KeyType.RIP: + // Only add the RIP when not yet present + if (this.RIP == null) { + this.AddChild(klv); + this.RIP = klv as MXFRIP; + } + break; + + case KeyType.SystemItem: + if (currentPartition != null) { + // Store the first system item for every partition + // (required to calculate essence positions) + if (currentPartition.FirstSystemItem == null) { + currentPartition.FirstSystemItem = klv as MXFSystemItem; + } + currentPartition.AddChild(klv); + } else + this.AddChild(klv); + + // Store the first and the last system item + if (this.FirstSystemItem == null) { + this.FirstSystemItem = klv as MXFSystemItem; + } + this.LastSystemItem = klv as MXFSystemItem; + break; + + + case KeyType.Essence: + if (currentPartition != null) { + // Store the first system item for every partition + // (required to calculate essence positions) + MXFEssenceElement ee = klv as MXFEssenceElement; + if (ee.IsPicture && currentPartition.FirstPictureEssenceElement == null) + currentPartition.FirstPictureEssenceElement = ee; + currentPartition.AddChild(klv); + } else + this.AddChild(klv); + break; + + case KeyType.Preface: + this.LogicalBase = new MXFLogicalObject(klv, klv.ToString()); + // Normal + if (currentPartition != null) + currentPartition.AddChild(klv); + else + this.AddChild(klv); + break; + + default: + // Normal + if (currentPartition != null) + currentPartition.AddChild(klv); + else + this.AddChild(klv); + break; + } + + return true; + } + + /// /// Try to locate the RIP /// @@ -224,6 +400,91 @@ namespace Myriadbits.MXF { } + /// + /// Execute all validation tests + /// + public void ExecuteValidationTest(bool extendedTest) { + // Reset results + this.m_results.Clear(); + + // Execute validation tests + List allTest = new List(); + allTest.Add(new MXFValidatorInfo()); + allTest.Add(new MXFValidatorPartitions()); + allTest.Add(new MXFValidatorRIP()); + if (extendedTest) { + allTest.Add(new MXFValidatorIndex()); + } + foreach (MXFValidator mxfTest in allTest) { + mxfTest.Initialize(this); + mxfTest.ExecuteTest(ref m_results); + } + + if (!extendedTest) { + MXFValidationResult valResult = new MXFValidationResult("Index Table"); + this.m_results.Add(valResult); + valResult.SetWarning("Index table test not executed in partial loading mode (to execute test press the execute all test button)."); + //MXFValidationResult valResult = new MXFValidationResult("Index Tables"); + //this.Results.Add(valResult); // And directly add the results + } + } + + /// + /// Create the logical view (starting with the preface) + /// + protected void CreateLogicalTree() { + if (this.LogicalBase == null) + return; + + LogicalAddChilds(this.LogicalBase); + } + + /// + /// Add all children (recusive) + /// + /// + protected MXFLogicalObject LogicalAddChilds(MXFLogicalObject parent) { + // Check properties for reference + MXFObject reference = parent.Object; + if (reference != null) { + foreach (PropertyInfo propertyInfo in reference.GetType().GetProperties()) { + if (propertyInfo.CanRead) { + if (propertyInfo.PropertyType == typeof(MXFRefKey)) { + // Found one! + MXFRefKey refKey = (MXFRefKey)propertyInfo.GetValue(reference, null); + if (refKey != null && refKey.Reference != null) { + // Add the child + MXFLogicalObject lo = new MXFLogicalObject(refKey.Reference, refKey.Reference.ToString()); + parent.AddChild(lo); + + // Add all sub stuff + LogicalAddChilds(lo); + } + } + } + } + + if (reference.HasChildren) { + foreach (MXFObject child in reference.Children) { + if (child.HasChildren) { + foreach (MXFObject grandchild in child.Children) { + MXFRefKey refKey = grandchild as MXFRefKey; + if (refKey != null && refKey.Reference != null) { + MXFLogicalObject lo = new MXFLogicalObject(refKey.Reference, refKey.Reference.ToString()); + + // Add the child + parent.AddChild(lo); + + // Add all sub stuff + LogicalAddChilds(lo); + } + } + } + } + } + } + return parent; + } /// /// Loop through all items, when an item with a reference is found, diff --git a/client/MXFFileParser/Metadata/B17 MXFTimecodeComponent.cs b/client/MXFFileParser/Metadata/B17 MXFTimecodeComponent.cs index dadf6527..7a23b67b 100644 --- a/client/MXFFileParser/Metadata/B17 MXFTimecodeComponent.cs +++ b/client/MXFFileParser/Metadata/B17 MXFTimecodeComponent.cs @@ -24,7 +24,7 @@ using System.ComponentModel; namespace Myriadbits.MXF { - class MXFTimecodeComponent : MXFStructuralComponent + public class MXFTimecodeComponent : MXFStructuralComponent { [CategoryAttribute("TimecodeComponent"), Description("1501")] public UInt64? StartTimecode { get; set; } diff --git a/client/MXFFileParser/Properties/AssemblyInfo.cs b/client/MXFFileParser/Properties/AssemblyInfo.cs index e5f4f80c..95a916f9 100644 --- a/client/MXFFileParser/Properties/AssemblyInfo.cs +++ b/client/MXFFileParser/Properties/AssemblyInfo.cs @@ -32,5 +32,5 @@ using System.Runtime.InteropServices; // You can specify all the values or you can default the Build and Revision Numbers // by using the '*' as shown below: // [assembly: AssemblyVersion("1.0.*")] -[assembly: AssemblyVersion("2.2.0.4")] -[assembly: AssemblyFileVersion("2.2.0.4")] +[assembly: AssemblyVersion("2.2.0.5")] +[assembly: AssemblyFileVersion("2.2.0.5")] diff --git a/client/Maestro/Installer/installforge-installer-project.ifp b/client/Maestro/Installer/installforge-installer-project.ifp index 02cc44e2481889d3187072bf28654d92e6685e10..a59de308abf132099858b570da9dccd62ebb5560 100644 GIT binary patch delta 19 bcmew>|5tuOHlyXnoEt2RmYZE!cW?m!Q_Kfo delta 19 bcmew>|5tuOHlxMHoEt2R7MopJcW?m!Q@RIU diff --git a/client/Maestro/Properties/AssemblyInfo.cs b/client/Maestro/Properties/AssemblyInfo.cs index d54db5b5..571031c3 100644 --- a/client/Maestro/Properties/AssemblyInfo.cs +++ b/client/Maestro/Properties/AssemblyInfo.cs @@ -32,5 +32,5 @@ using System.Runtime.InteropServices; // You can specify all the values or you can default the Build and Revision Numbers // by using the '*' as shown below: // [assembly: AssemblyVersion("1.0.*")] -[assembly: AssemblyVersion("2.0.6.8")] -[assembly: AssemblyFileVersion("2.0.6.8")] +[assembly: AssemblyVersion("2.0.6.9")] +[assembly: AssemblyFileVersion("2.0.6.9")] diff --git a/client/MaestroShared/Metadata/Timecode.cs b/client/MaestroShared/Metadata/Timecode.cs index bf29d379..254c1517 100644 --- a/client/MaestroShared/Metadata/Timecode.cs +++ b/client/MaestroShared/Metadata/Timecode.cs @@ -15,6 +15,11 @@ namespace MaestroShared.Metadata { Set(frames); } + public Timecode(int frames, float frameRate) { + this.frameRate = frameRate; + Set(frames); + } + public Timecode(Timecode tc) { if (tc == null) return; diff --git a/server/-configuration/deploy-mediacube-modules.launch b/server/-configuration/deploy-bkup-mediacube-modules.launch similarity index 88% rename from server/-configuration/deploy-mediacube-modules.launch rename to server/-configuration/deploy-bkup-mediacube-modules.launch index b74846aa..1388508f 100644 --- a/server/-configuration/deploy-mediacube-modules.launch +++ b/server/-configuration/deploy-bkup-mediacube-modules.launch @@ -1,7 +1,7 @@ - + diff --git a/server/-configuration/deploy-cluster-mediacube-modules.launch b/server/-configuration/deploy-cluster-mediacube-modules.launch new file mode 100644 index 00000000..22037459 --- /dev/null +++ b/server/-configuration/deploy-cluster-mediacube-modules.launch @@ -0,0 +1,19 @@ + + + + + + + + + + + + + + + + + + + diff --git a/server/-configuration/run-mediacube-server-bsh.launch b/server/-configuration/run-mediacube-server-bsh.launch index 5b9bf249..9a02b737 100644 --- a/server/-configuration/run-mediacube-server-bsh.launch +++ b/server/-configuration/run-mediacube-server-bsh.launch @@ -19,7 +19,7 @@ - + diff --git a/server/-configuration/scheduledjobs.json b/server/-configuration/scheduledjobs.json index c18a56ad..47183e21 100644 --- a/server/-configuration/scheduledjobs.json +++ b/server/-configuration/scheduledjobs.json @@ -102,17 +102,17 @@ "executeimmediate": true, "cronexpression": "0/20 * * * * ?", "parameters": [ - {"name": "forceFull", "value": false, "type": "java.lang.Boolean"} + {"name": "includeArchived", "value": false, "type": "java.lang.Boolean"} ] }, { "active": false, - "name" : "OCTOPUS adatok teljes szinkronizálása", + "name" : "OCTOPUS adatok szinkronizálása archivált tükrökkel együtt", "template": "sync-octopus.xml", "executeimmediate": true, "cronexpression": "0/30 * * * * ?", "parameters": [ - {"name": "forceFull", "value": true, "type": "java.lang.Boolean"} + {"name": "includeArchived", "value": true, "type": "java.lang.Boolean"} ] }, { diff --git a/server/-modules/pom.xml b/server/-modules/pom.xml index a7da8e11..1c58fc6c 100644 --- a/server/-modules/pom.xml +++ b/server/-modules/pom.xml @@ -18,17 +18,14 @@ - scp://root:password@10.10.1.27 - ssh-ed25519 256 ea:ab:67:70:79:63:2f:6a:34:81:48:e2:b9:dd:ca:d4 + ssh-ed25519 256 ea:ab:67:70:79:63:2f:6a:34:81:48:e2:b9:dd:ca:d4 - 1.0.0 1.0.0 1.8 1.8 UTF-8 UTF-8 - diff --git a/server/-product/mediacube.product b/server/-product/mediacube.product index 02a52595..3762d6c7 100644 --- a/server/-product/mediacube.product +++ b/server/-product/mediacube.product @@ -42,6 +42,9 @@ -Djobengine.octopus.api.address=http://10.10.1.11/api/v1 -Djobengine.octopus.api.user=mam -Djobengine.octopus.api.password=napocska +-Djobengine.octopus.rundowns.name=test-rundowns +-Djobengine.octopus.storyfolders.name=test-storyfolders +-Djobengine.octopus.stories.name=test-stories #Selenio -Djobengine.selenio.address=http://10.10.1.71:44000/TranscodeMgrWS?wsdl @@ -53,6 +56,7 @@ -Djobengine.nexio.db.password=resolve -Dnexio.useMOSGateway=true -Dnexio.host=10.10.1.55 +-Dnexio.disable=true -XstartOnFirstThread -Dorg.eclipse.swt.internal.carbon.smallFonts diff --git a/server/-product/pom.xml b/server/-product/pom.xml index 1343c4a8..3ab00a53 100644 --- a/server/-product/pom.xml +++ b/server/-product/pom.xml @@ -289,6 +289,7 @@ winscp.com /command + "echo STARTING DEPLOY TO: ${remote.address}" "open ${remote.address} -hostkey=""${remote.hostkey}""" @@ -299,6 +300,8 @@ "synchronize remote" "mkdir log" "call chmod +x mediacube" + "call chmod +x start-mediacube.sh" + "call chmod +x stop-mediacube.sh" "exit" @@ -311,6 +314,6 @@ ${project.build.directory}/products/MediaCube/linux/gtk/x86_64 - /opt/mediacube + /opt/mediacube-test diff --git a/server/user.jobengine.executors/config/scheduledjobs.json b/server/user.jobengine.executors/config/scheduledjobs.json index c0c9dff6..8e3af0f4 100644 --- a/server/user.jobengine.executors/config/scheduledjobs.json +++ b/server/user.jobengine.executors/config/scheduledjobs.json @@ -1,16 +1,26 @@ {"joblist":[ { - "active": true, + "active": false, "name" : "OCTOPUS adatok szinkronizálása", "template": "sync-octopus.xml", "executeimmediate": false, "cronexpression": "0/30 * * * * ?", "parameters": [ - {"name": "forceFull", "value": true, "type": "java.lang.Boolean"} + {"name": "includeArchived", "value": false, "type": "java.lang.Boolean"} ] }, { "active": true, + "name" : "OCTOPUS adatok szinkronizálása (archivált tükrök is)", + "template": "sync-octopus.xml", + "executeimmediate": true, + "cronexpression": "0/30 * * * * ?", + "parameters": [ + {"name": "includeArchived", "value": true, "type": "java.lang.Boolean"} + ] + }, + { + "active": false, "executeimmediate": false, "name" : "Archiválás az ISILON/ARCHIVE mappából", "template": "archive-ondemand.xml", @@ -24,6 +34,16 @@ ] }, { + "active": false, + "name" : "Lejárt ISISLON/ARCHIVE anyagok törlése", + "template": "delete-materials.xml", + "executeimmediate": false, + "cronexpression": "0 0 5 * * ?", + "parameters": [ + {"name": "sourcePath", "value": "/mnt/ISILON/ARCHIVE", "type": "java.lang.String"} + ] + }, + { "active": false, "name" : "NEXIO anyagok másolása az ISILON/ARCHIVE mappába", "template": "copyforarchive-nexio-materials.xml", @@ -42,16 +62,6 @@ ] }, { - "active": true, - "name" : "Lejárt ISISLON/ARCHIVE anyagok törlése", - "template": "delete-materials.xml", - "executeimmediate": false, - "cronexpression": "0 0 5 * * ?", - "parameters": [ - {"name": "sourcePath", "value": "/mnt/ISILON/ARCHIVE", "type": "java.lang.String"} - ] - }, - { "active": false, "name" : "Archiválás az ISILON/ARCHIVE/ONE mappából", "template": "archive-ondemand.xml", diff --git a/server/user.jobengine.executors/deploy-steps-to-bsh-bkup.bat b/server/user.jobengine.executors/deploy-steps-to-bsh-bkup.bat new file mode 100644 index 00000000..8235a562 --- /dev/null +++ b/server/user.jobengine.executors/deploy-steps-to-bsh-bkup.bat @@ -0,0 +1,32 @@ +@ECHO OFF + +SET REMOTE_HOST=10.10.1.29 +SET REMOTE_LOCATION=/opt/mediacube/configuration/executors +SET REMOTE_SERVER_HOSTKEY=ssh-ed25519 256 ea:ab:67:70:79:63:2f:6a:34:81:48:e2:b9:dd:ca:d4 +SET REMOTE_SERVER_ADDRESS=scp://root:password@%REMOTE_HOST% +SET LOCAL_STEPS_LOCATION=bin/user/jobengine/server/steps +SET LOCAL_CONFIG_LOCATION=config + + ECHO *** Begin deploy steps to server %REMOTE_HOST% *** + ECHO --- Deploying + + WinSCP.com /command ^ + "open %REMOTE_SERVER_ADDRESS% -hostkey=""%REMOTE_SERVER_HOSTKEY%"" -timeout=60" ^ + "cd %REMOTE_LOCATION%" ^ + "lcd %LOCAL_STEPS_LOCATION%" ^ + "synchronize remote" ^ + "lcd ..\" ^ + "lcd ..\" ^ + "lcd ..\" ^ + "lcd ..\" ^ + "lcd ..\" ^ + "lcd %LOCAL_CONFIG_LOCATION%" ^ + "synchronize remote -filemask=config.xml" ^ + "cd .." ^ + "synchronize remote -filemask=scheduledjobs.json" ^ + "exit" + + ECHO *** Completed *** +@ECHO ON + + diff --git a/server/user.jobengine.executors/deploy-steps-to-bsh-bkup.launch b/server/user.jobengine.executors/deploy-steps-to-bsh-bkup.launch new file mode 100644 index 00000000..7e5b62eb --- /dev/null +++ b/server/user.jobengine.executors/deploy-steps-to-bsh-bkup.launch @@ -0,0 +1,8 @@ + + + + + + + + diff --git a/server/user.jobengine.executors/jobtemplates/sync-octopus.xml b/server/user.jobengine.executors/jobtemplates/sync-octopus.xml index 63c50a3b..58aa09f7 100644 --- a/server/user.jobengine.executors/jobtemplates/sync-octopus.xml +++ b/server/user.jobengine.executors/jobtemplates/sync-octopus.xml @@ -2,14 +2,14 @@ - + - + diff --git a/server/user.jobengine.executors/src/user/jobengine/server/steps/OctopusDataMiner.java b/server/user.jobengine.executors/src/user/jobengine/server/steps/OctopusDataMiner.java index 7bc8223e..cc5aa35e 100644 --- a/server/user.jobengine.executors/src/user/jobengine/server/steps/OctopusDataMiner.java +++ b/server/user.jobengine.executors/src/user/jobengine/server/steps/OctopusDataMiner.java @@ -1,5 +1,6 @@ package user.jobengine.server.steps; +import java.util.Calendar; import java.util.Date; import java.util.HashMap; import java.util.HashSet; @@ -27,18 +28,19 @@ import com.ibm.nosql.json.api.BasicDBObject; import com.ibm.nosql.json.api.DB; import com.ibm.nosql.json.api.DBCollection; import com.ibm.nosql.json.api.DBCursor; -import com.ibm.nosql.json.api.DBObject; import com.ibm.nosql.json.api.QueryBuilder; import com.ibm.nosql.json.api.WriteResult; +import user.commons.CalendarUtils; import user.commons.ListUtils; import user.commons.nosql.NoSQLUtils; import user.commons.octopus.IOctopusAPI; -import user.commons.octopus.OctopusAPI; import user.commons.remotestore.IProgressEventListener; import user.commons.remotestore.ProgressEvent; -public class OctopusDataMiner implements Runnable { +public class OctopusDataMiner { + private static final String ARCHIVED = "archived"; + private static final String FILTER = "filter"; private static final String _TMP = "_tmp"; private static final Logger logger = LogManager.getLogger(); private static final String LINEFEED = "\r\n"; @@ -48,8 +50,10 @@ public class OctopusDataMiner implements Runnable { // private static final String CHECKING_RUNDOWN = "Checking Rundown {} ({}/{})"; private static final String FIELDS_STORYFOLDER_STORIES = "stories,Story.modified,Story.name,Story.id,Story.mosObjects,Story.script,Story.type,Story.format,Story.customColumns,CustomColumn.label,CustomColumn.value"; private static final String FIELDS_RUNDOWN_STORIES = "slugs,Slug.story,Slug.position,Story.name,Story.id,Story.modified,Story.mosObjects,Story.script,Story.type,Story.format,Story.customColumns,CustomColumn.label,CustomColumn.value"; - private static final String FIELDS_RUNDOWN_STORYIDS = "id,name,modified,scheduledStart,channel,Channel.name,rundownType,RundownType.name,slugs,Slug.storyId,Slug.position"; - private static final String FIELDS_STORY_FOLDER_LIST = "id,name,modified,stories,Story.id"; + private static final String FIELDS_RUNDOWN = "id,name,modified,scheduledStart,channel,Channel.name,rundownType,RundownType.name"; + private static final String FIELDS_RUNDOWN_STORYIDS = "id,slugs,Slug.storyId,Slug.position"; + private static final String FIELDS_STORYFOLDER = "id,name,modified"; + private static final String FIELDS_STORYFOLDER_STORYIDS = "id,stories,Story.id"; private static final String RUNDOWN = "Rundown"; private static final String OCTOPUS_DEVICE_NAME = "Octopus-Device-Name"; private static final String OCTOPUS_DEVICE_ID = "Octopus-Device-Id"; @@ -72,16 +76,11 @@ public class OctopusDataMiner implements Runnable { private ProgressEvent progressEvent = new ProgressEvent(this, 0); private Map storyRundowns; private Map storyStoryFolders; - private Map storedStoryRundowns; - private Map storedStoryStoryFolders; - private Map storedStoryMosObjects; - private String RUNDOWN_COLLECTION = IOctopusAPI.RUNDOWN_COLLECTION; - private String FOLDER_COLLECTION = IOctopusAPI.FOLDER_COLLECTION; - private String STORY_COLLECTION = IOctopusAPI.STORY_COLLECTION; - - private Map newRundowns = new HashMap<>(); - private Map newStoryFolders = new HashMap<>(); - private Map newStories = new HashMap<>(); + private String RUNDOWN_COLLECTION; + private String FOLDER_COLLECTION; + private String STORY_COLLECTION; + private boolean includeArchived; + private Calendar zeroDate = CalendarUtils.createCalendar(2017, 11, 4); public OctopusDataMiner() { db = NoSQLUtils.getNoSQLDB(); @@ -105,8 +104,11 @@ public class OctopusDataMiner implements Runnable { Map result = new HashMap<>(); List storyFolderList = NoSQLUtils.asList(storyFolders); for (BasicDBObject storyFolder : storyFolderList) { - long storyFolderId = storyFolder.getLong(IOctopusAPI.ID); - List stories = NoSQLUtils.asList(storyFolder, IOctopusAPI.STORIES); + if (storyFolder == null || !storyFolder.containsKey(IOctopusAPI.ID)) + continue; + BasicDBObject storyFolderWithStoryIds = queryStoryFolder(storyFolder, FIELDS_STORYFOLDER_STORYIDS); + long storyFolderId = storyFolderWithStoryIds.getLong(IOctopusAPI.ID); + List stories = NoSQLUtils.asList(storyFolderWithStoryIds, IOctopusAPI.STORIES); if (stories == null) continue; long position = 1; @@ -127,10 +129,11 @@ public class OctopusDataMiner implements Runnable { Map result = new HashMap<>(); List rundownsList = NoSQLUtils.asList(rundowns); for (BasicDBObject rundown : rundownsList) { - if (!rundown.containsKey(IOctopusAPI.ID)) + if (rundown == null || !rundown.containsKey(IOctopusAPI.ID)) continue; - long rundownId = rundown.getLong(IOctopusAPI.ID); - List slugs = NoSQLUtils.asList(rundown, IOctopusAPI.SLUGS); + BasicDBObject rundownWithStoryids = queryRundown(rundown, FIELDS_RUNDOWN_STORYIDS); + long rundownId = rundownWithStoryids.getLong(IOctopusAPI.ID); + List slugs = NoSQLUtils.asList(rundownWithStoryids, IOctopusAPI.SLUGS); if (slugs == null) continue; for (BasicDBObject slug : slugs) { @@ -151,44 +154,6 @@ public class OctopusDataMiner implements Runnable { return result; } - private void buildStoriesReferences() { - DBCollection collection = db.getCollection(STORY_COLLECTION); - DBCursor cursor = collection.find(null, new BasicDBObject(IOctopusAPI.ID, 1).append(IOctopusAPI.REF_RUNDOWN, 1).append(IOctopusAPI.REF_STORYFOLDER, 1) - .append(IOctopusAPI.MOS_OBJECTS, 1)); - //DBCursor find = collection.find(QueryBuilder.start(ID).greaterThan(0).get()); - try { - - while (cursor.hasNext()) { - BasicDBObject story = (BasicDBObject) cursor.next(); - long storyId = story.getLong(IOctopusAPI.ID); - BasicDBList rundownRef = NoSQLUtils.asDBList(story, IOctopusAPI.REF_RUNDOWN); - if (rundownRef != null) { - if (storedStoryRundowns == null) - storedStoryRundowns = new HashMap<>(); - storedStoryRundowns.put(storyId, rundownRef); - } - BasicDBList storyFolderRef = NoSQLUtils.asDBList(story, IOctopusAPI.REF_STORYFOLDER); - if (storyFolderRef != null) { - if (storedStoryStoryFolders == null) - storedStoryStoryFolders = new HashMap<>(); - storedStoryStoryFolders.put(storyId, storyFolderRef); - } - - BasicDBList storyMosObjects = NoSQLUtils.asDBList(story, IOctopusAPI.MOS_OBJECTS); - if (storyMosObjects != null) { - if (storedStoryMosObjects == null) - storedStoryMosObjects = new HashMap<>(); - storedStoryMosObjects.put(storyId, storyMosObjects); - } - } - } catch (Exception e) { - logger.catching(e); - throw e; - } finally { - - } - } - public void clear() { db.getCollection(RUNDOWN_COLLECTION).remove(); db.getCollection(STORY_COLLECTION).remove(); @@ -212,41 +177,14 @@ public class OctopusDataMiner implements Runnable { return concatParentsToStoryFolder(parent, newName); } - private void deleteDiff(String oldCollectionName, String newCollectionName, String idFieldName) { - DBCollection oldCollection = db.getCollection(oldCollectionName); - DBCollection newCollection = db.getCollection(newCollectionName); - DBCursor oldCollectionCursor = oldCollection.find(new BasicDBObject(), new BasicDBObject(idFieldName, 1)); - if (!oldCollectionCursor.hasNext()) { - logger.error("{} collection is empty", newCollectionName); - return; - } - List oldItems = ListUtils.cast(oldCollectionCursor.toArray()); - - DBCursor newCollectionCursor = newCollection.find(); - ConcurrentHashMap newItems = null; - if (newCollectionCursor.hasNext()) { - List newList = ListUtils.cast(newCollectionCursor.toArray()); - newItems = ListUtils.map(newList, item -> item.getLong(IOctopusAPI.ID)); - } - if (newItems == null) - newItems = new ConcurrentHashMap<>(); - - for (BasicDBObject oldItem : oldItems) { - if (oldItem == null) { - logger.error("Item is null"); - continue; - } - if (!oldItem.containsKey(idFieldName)) { - logger.error("{} is null", idFieldName); - continue; - } - long id = oldItem.getLong(idFieldName); - BasicDBObject newItem = newItems.get(id); - if (newItem == null) { - //remove - logger.info("Deleting {}", oldItem.toPrettyString(null)); - oldCollection.remove(new BasicDBObject(idFieldName, id)); - } + private void deleteOrphanRundowns() { + try { + DBCollection collection = db.getCollection(RUNDOWN_COLLECTION); + BasicDBObject query = (BasicDBObject) QueryBuilder.start().put(IOctopusAPI.ID).notIn(storyRundowns.keySet().toArray()).get(); + WriteResult res = collection.remove(query); + logger.trace(String.format("Deleted orphan rundowns: %d", res.getN())); + } catch (Exception e) { + logger.error(e); } } @@ -261,6 +199,17 @@ public class OctopusDataMiner implements Runnable { } } + private void deleteOrphanStoryFolders() { + try { + DBCollection collection = db.getCollection(RUNDOWN_COLLECTION); + BasicDBObject query = (BasicDBObject) QueryBuilder.start().put(IOctopusAPI.ID).notIn(storyStoryFolders.keySet().toArray()).get(); + WriteResult res = collection.remove(query); + logger.trace(String.format("Deleted orphan rundowns: %d", res.getN())); + } catch (Exception e) { + logger.error(e); + } + } + private void ensureIndexes() { DBCollection collection = db.getCollection(FOLDER_COLLECTION); if (collection.count() == 0) @@ -275,26 +224,30 @@ public class OctopusDataMiner implements Runnable { collection.ensureIndex(IOctopusAPI.ID); } - public void execute() throws Exception { + public void execute(boolean includeArchived) throws Exception { + this.includeArchived = includeArchived; logger.trace(STARTING); - //{"filter" :{ "archived" : true }} - Response response = query(RUNDOWN, "id,name,modified,scheduledStart,channel,Channel.name,rundownType,RundownType.name") - .post(Entity.entity(new BasicDBObject("filter", new BasicDBObject("archived", true)).toPrettyString(null), MediaType.APPLICATION_JSON)); - String json = response.readEntity(String.class); - RUNDOWN_COLLECTION = IOctopusAPI.RUNDOWN_COLLECTION + _TMP; - FOLDER_COLLECTION = IOctopusAPI.FOLDER_COLLECTION + _TMP; - STORY_COLLECTION = IOctopusAPI.STORY_COLLECTION + _TMP; + // String MAIN_RUNDOWN_COLLECTION = IOctopusAPI.RUNDOWN_COLLECTION; + // String MAIN_FOLDER_COLLECTION = IOctopusAPI.FOLDER_COLLECTION; + // String MAIN_STORY_COLLECTION = IOctopusAPI.STORY_COLLECTION; - try { - db.getCollection(RUNDOWN_COLLECTION).drop(); - db.getCollection(FOLDER_COLLECTION).drop(); - db.getCollection(STORY_COLLECTION).drop(); - } catch (Exception e) { - logger.catching(e); - throw e; - } + // RUNDOWN_COLLECTION = MAIN_RUNDOWN_COLLECTION + _TMP; + // FOLDER_COLLECTION = MAIN_FOLDER_COLLECTION + _TMP; + // STORY_COLLECTION = MAIN_STORY_COLLECTION + _TMP; + // + // try { + // db.getCollection(RUNDOWN_COLLECTION).drop(); + // db.getCollection(FOLDER_COLLECTION).drop(); + // db.getCollection(STORY_COLLECTION).drop(); + // } catch (Exception e) { + // logger.catching(e); + // throw e; + // } + RUNDOWN_COLLECTION = IOctopusAPI.RUNDOWN_COLLECTION; + FOLDER_COLLECTION = IOctopusAPI.FOLDER_COLLECTION; + STORY_COLLECTION = IOctopusAPI.STORY_COLLECTION; BasicDBList rundowns = null; BasicDBList storyFolders = null; @@ -309,31 +262,42 @@ public class OctopusDataMiner implements Runnable { processRundowns(rundowns); processStoryFolders(storyFolders); + deleteOrphanRundowns(); + deleteOrphanStoryFolders(); + deleteOrphanStories(); + //a sorrend fontos ! - updateDiff(IOctopusAPI.STORY_COLLECTION, STORY_COLLECTION, IOctopusAPI.ID); - updateDiff(IOctopusAPI.RUNDOWN_COLLECTION, RUNDOWN_COLLECTION, IOctopusAPI.ID); - updateDiff(IOctopusAPI.FOLDER_COLLECTION, FOLDER_COLLECTION, IOctopusAPI.ID); - deleteDiff(IOctopusAPI.RUNDOWN_COLLECTION, RUNDOWN_COLLECTION, IOctopusAPI.ID); - //deleteDiff(IOctopusAPI.FOLDER_COLLECTION, FOLDER_COLLECTION, IOctopusAPI.ID); - deleteDiff(IOctopusAPI.STORY_COLLECTION, STORY_COLLECTION, IOctopusAPI.ID); - - //setLastUpdateTime(new Date()); - logger.info("Activate"); - - // db.getCollection(RUNDOWN_COLLECTION).rename(IOctopusAPI.RUNDOWN_COLLECTION, true); - // db.getCollection(FOLDER_COLLECTION).rename(IOctopusAPI.FOLDER_COLLECTION, true); - // db.getCollection(STORY_COLLECTION).rename(IOctopusAPI.STORY_COLLECTION, true); - logger.trace(FINISHED); + // updateDiff(MAIN_STORY_COLLECTION, STORY_COLLECTION, IOctopusAPI.ID); + // updateDiff(MAIN_RUNDOWN_COLLECTION, RUNDOWN_COLLECTION, IOctopusAPI.ID); + // updateDiff(MAIN_FOLDER_COLLECTION, FOLDER_COLLECTION, IOctopusAPI.ID); + // + // updateDeleteDiff(MAIN_RUNDOWN_COLLECTION, RUNDOWN_COLLECTION, IOctopusAPI.ID); + // updateDeleteDiff(MAIN_FOLDER_COLLECTION, FOLDER_COLLECTION, IOctopusAPI.ID); + // updateDeleteDiff(MAIN_STORY_COLLECTION, STORY_COLLECTION, IOctopusAPI.ID); - } + // try { + // db.getCollection(RUNDOWN_COLLECTION).rename(MAIN_RUNDOWN_COLLECTION, true); + // db.getCollection(FOLDER_COLLECTION).rename(MAIN_FOLDER_COLLECTION, true); + // db.getCollection(STORY_COLLECTION).rename(MAIN_STORY_COLLECTION, true); + // } catch (Exception e) { + // logger.catching(e); + // throw e; + // } + + // logger.info("Activate"); - public void executetest() { - // ResteasyWebTarget target = webTarget.path(RUNDOWN); - // Builder result = target.request().header(OCTOPUS_DEVICE_ID, apiUser).header(OCTOPUS_DEVICE_NAME, apiPwd); - // Response r = result.get(); - // String x = r.readEntity(String.class); - // logger.info(x); - // return; + // try { + // long ts = new Date().getTime(); + // db.getCollection(RUNDOWN_COLLECTION).exportFile(String.format("/opt/mediacube-test/log/%s-%s.js", RUNDOWN_COLLECTION, ts)); + // db.getCollection(FOLDER_COLLECTION).exportFile(String.format("/opt/mediacube-test/log/%s-%s.js", FOLDER_COLLECTION, ts)); + // db.getCollection(STORY_COLLECTION).exportFile(String.format("/opt/mediacube-test/log/%s-%s.js", STORY_COLLECTION, ts)); + // } catch (Exception e) { + // logger.catching(e); + // throw e; + // } + + logger.trace(FINISHED); + throw new Exception("Mérés"); } private String extractContent(BasicDBObject content) { @@ -452,31 +416,6 @@ public class OctopusDataMiner implements Runnable { } } - private Date getLastUpdateTime() { - Date result = null; - DBCollection collection = db.getCollection(IOctopusAPI.TIME_COLLECTION_NAME); - DBObject timeObject = collection.findOne(); - if (timeObject != null) - result = (Date) timeObject.get(IOctopusAPI.LASTUPDATE_TIME); - return result; - } - - // private boolean isModified(Date date, BasicDBObject object, String name) { - // Date actualModifiedString = toDate(object, name); - // if (actualModifiedString == null) - // logger.trace(ACTUAL_MODIFIED_STRING_IS_NULL); - // int result = date.compareTo(actualModifiedString); - // return result <= 0; - // } - - private boolean isModified(Date date, BasicDBObject object) { - if (date == null) - return true; - Date modified = (Date) object.get(IOctopusAPI.MODIFIED); - int result = date.compareTo(modified); - return result <= 0; - } - private void processRundowns(BasicDBList rundowns) throws Exception { if (rundowns == null || rundowns.size() == 0) { progressEvent.setProgress(50); @@ -508,14 +447,6 @@ public class OctopusDataMiner implements Runnable { } private Builder query(String path, String fields) { - //logger.info("Class loader {}", getClass().getClassLoader()); - // try { - // //TODO kell e? - // ResteasyDeployment deployment = new ResteasyDeployment(); - // deployment.start(); - // } catch (Exception e) { - // logger.catching(e); - // } ResteasyWebTarget target = webTarget.path(path).queryParam(FIELDS, fields); Builder result = target.request().header(OCTOPUS_DEVICE_ID, apiUser).header(OCTOPUS_DEVICE_NAME, apiPwd); return result; @@ -539,11 +470,11 @@ public class OctopusDataMiner implements Runnable { return rundowns; } - private BasicDBObject queryRundown(BasicDBObject rundown) { + private BasicDBObject queryRundown(BasicDBObject rundown, String fields) { logger.trace(ENTER); BasicDBObject result = null; long id = NoSQLUtils.asLong(rundown, IOctopusAPI.ID); - Response response = query(String.format("%s/", RUNDOWN) + id, FIELDS_RUNDOWN_STORIES).get(); + Response response = query(String.format("%s/%d", RUNDOWN, id), fields).get(); String json = response.readEntity(String.class); BasicDBObject resultObject = (BasicDBObject) JSONUtil.jsonToDbObject(json); if (resultObject == null) @@ -557,7 +488,13 @@ public class OctopusDataMiner implements Runnable { private BasicDBList queryRundowns() { logger.trace(ENTER); BasicDBList result = null; - Response response = query(RUNDOWN, FIELDS_RUNDOWN_STORYIDS).get(); + Builder query = query(RUNDOWN, FIELDS_RUNDOWN); + Response response = null; + if (includeArchived) { + response = query.post(Entity.entity(new BasicDBObject(FILTER, new BasicDBObject(ARCHIVED, true)).toString(), MediaType.APPLICATION_JSON)); + } else + response = query.get(); + String json = response.readEntity(String.class); BasicDBObject resultObject = (BasicDBObject) JSONUtil.jsonToDbObject(json); if (resultObject != null) @@ -566,11 +503,11 @@ public class OctopusDataMiner implements Runnable { return result; } - private BasicDBObject queryStoryFolder(BasicDBObject storyFolder) { + private BasicDBObject queryStoryFolder(BasicDBObject storyFolder, String fields) { logger.trace(ENTER); BasicDBObject result = null; long id = NoSQLUtils.asLong(storyFolder, IOctopusAPI.ID); - Response response = query(String.format("%s/", STORY_FOLDER) + id, FIELDS_STORYFOLDER_STORIES).get(); + Response response = query(String.format("%s/%d", STORY_FOLDER, id), fields).get(); String json = response.readEntity(String.class); BasicDBObject resultObject = (BasicDBObject) JSONUtil.jsonToDbObject(json); if (resultObject == null) @@ -584,21 +521,21 @@ public class OctopusDataMiner implements Runnable { private BasicDBList queryStoryFolders() { logger.trace(ENTER); BasicDBList result = null; - Response response = query(STORY_FOLDER, FIELDS_STORY_FOLDER_LIST).get(); + Response response = query(STORY_FOLDER, FIELDS_STORYFOLDER).get(); String json = response.readEntity(String.class); BasicDBObject resultObject = (BasicDBObject) JSONUtil.jsonToDbObject(json); if (resultObject != null) result = NoSQLUtils.asDBList(resultObject, RESULT); - /* teszt */ - List list = NoSQLUtils.asList(result); - for (BasicDBObject actual : list) { - String fullName = concatParentsToStoryFolder(actual, actual.getString(IOctopusAPI.NAME)); - //logger.info("Checking StoryFolder {}", fullName); - actual.remove(IOctopusAPI.NAME); - actual.append(IOctopusAPI.NAME, fullName); - } - + // /* teszt */ + // List list = NoSQLUtils.asList(result); + // for (BasicDBObject actual : list) { + // String fullName = concatParentsToStoryFolder(actual, actual.getString(IOctopusAPI.NAME)); + // //logger.info("Checking StoryFolder {}", fullName); + // actual.remove(IOctopusAPI.NAME); + // actual.append(IOctopusAPI.NAME, fullName); + // } + // logger.trace(EXIT); return result; } @@ -607,86 +544,31 @@ public class OctopusDataMiner implements Runnable { progressListenerList.remove(IProgressEventListener.class, listener); } - @Override - public void run() { - logger.trace(STARTING); - try { - ensureIndexes(); - Date lastUpdateTime = getLastUpdateTime(); - - buildStoriesReferences(); - - BasicDBList rundowns = queryRundowns(); - storyRundowns = buildRundownReferences(rundowns); - BasicDBList storyFolders = queryStoryFolders(); - storyStoryFolders = buildFolderReferences(storyFolders); - - if (rundowns == null || rundowns.size() == 0) { - progressEvent.setProgress(50); - fireProgressEvent(progressEvent); - } else { - storeRundowns(rundowns, lastUpdateTime); - } - - if (storyFolders == null || storyFolders.size() == 0) { - progressEvent.setProgress(100); - fireProgressEvent(progressEvent); - } else { - storeStoryFolders(storyFolders, lastUpdateTime); - } - deleteOrphanStories(); - setLastUpdateTime(new Date()); - } catch (Exception e) { - logger.catching(e); - throw e; - } - logger.trace(FINISHED); - } - - public void run(boolean forceFull) { - if (forceFull) { - clear(); - } - run(); - // if (forceFull) { - // RUNDOWN_COLLECTION_NAME = "rundowns"; - // STORY_COLLECTION_NAME = "stories"; - // STORY_FOLDER_COLLECTION_NAME = "storyfolders"; - // TIME_COLLECTION_NAME = "octopusSyncTime"; - // db.getCollection("rundowns_tmp").rename(RUNDOWN_COLLECTION_NAME, true); - // db.getCollection("stories_tmp").rename(STORY_COLLECTION_NAME, true); - // db.getCollection("storyfolders_tmp").rename(STORY_FOLDER_COLLECTION_NAME, true); - // db.getCollection("octopusSyncTime_tmp").rename(TIME_COLLECTION_NAME, true); - // //clear(); - // } - - //TODO reset collection names - } - - public void setLastUpdateTime(Date lastUpdateTime) { - DBCollection collection = db.getCollection(OctopusAPI.TIME_COLLECTION_NAME); - DBObject timeObject = collection.findOne(); - if (timeObject == null) - timeObject = new BasicDBObject(); - timeObject.put(IOctopusAPI.LASTUPDATE_TIME, lastUpdateTime); - - collection.save(timeObject); + void setObjectID(DBCollection collection, BasicDBObject objectToSave) { + BasicDBObject obj = (BasicDBObject) collection.findOne(new BasicDBObject(IOctopusAPI.ID, NoSQLUtils.asLong(objectToSave, IOctopusAPI.ID))); + if (obj == null) + return; + Object id = obj.getID(); + if (id == null) + return; + objectToSave.put(IOctopusAPI._ID, id); } - private void storeRundown(BasicDBObject rundown, Date lastUpdateTime) { + private void storeRundown(BasicDBObject rundown) { logger.trace(ENTER); String name = rundown.containsKey(IOctopusAPI.NAME) ? rundown.getString(IOctopusAPI.NAME) : null; logger.debug("Storing rundown {} {}", name, rundown.get(IOctopusAPI.SCHEDULED_START)); - BasicDBObject rundownWithStories = queryRundown(rundown); + BasicDBObject rundownWithStories = queryRundown(rundown, FIELDS_RUNDOWN_STORIES); if (rundownWithStories != null) { - BasicDBList stories = NoSQLUtils.asDBList(rundownWithStories, IOctopusAPI.SLUGS); - if (stories != null) - storeRundownStories(stories, lastUpdateTime); - rundown.put(IOctopusAPI.SCHEDULED_START, toDate(rundown, IOctopusAPI.SCHEDULED_START)); - rundown.put(IOctopusAPI.MODIFIED, toDate(rundown, IOctopusAPI.MODIFIED)); - DBCollection collection = db.getCollection(RUNDOWN_COLLECTION); - if (lastUpdateTime == null || (lastUpdateTime != null && isModified(lastUpdateTime, rundown))) { - //logger.debug(SAVING_RUNDOWN, rundownID, name); + Date scheduledStart = toDate(rundown, IOctopusAPI.SCHEDULED_START); + if (scheduledStart != null && scheduledStart.after(zeroDate.getTime())) { + BasicDBList stories = NoSQLUtils.asDBList(rundownWithStories, IOctopusAPI.SLUGS); + if (stories != null) + storeRundownStories(stories); + rundown.put(IOctopusAPI.SCHEDULED_START, toDate(rundown, IOctopusAPI.SCHEDULED_START)); + rundown.put(IOctopusAPI.MODIFIED, toDate(rundown, IOctopusAPI.MODIFIED)); + DBCollection collection = db.getCollection(RUNDOWN_COLLECTION); + setObjectID(collection, rundown); collection.save(rundown); } } @@ -699,7 +581,7 @@ public class OctopusDataMiner implements Runnable { int idx = 1; for (BasicDBObject rundown : rundownsList) { //logger.info(CHECKING_RUNDOWN, rundown.getLong(IOctopusAPI.ID), rundownsList.size(), idx); - storeRundown(rundown, lastUpdateTime); + storeRundown(rundown); int progress = idx * 50 / rundownsList.size(); if (progress - progressEvent.getProgress() > 0) { progressEvent.setProgress(progress); @@ -711,17 +593,17 @@ public class OctopusDataMiner implements Runnable { logger.trace(EXIT); } - private void storeRundownStories(BasicDBList slugs, Date lastUpdateTime) { + private void storeRundownStories(BasicDBList slugs) { logger.trace(ENTER); List slugsList = NoSQLUtils.asList(slugs); for (BasicDBObject slug : slugsList) { if (slug.containsKey(IOctopusAPI.STORY)) - storeStory((BasicDBObject) slug.get(IOctopusAPI.STORY), lastUpdateTime); + storeStory((BasicDBObject) slug.get(IOctopusAPI.STORY)); } logger.trace(EXIT); } - private void storeStory(BasicDBObject story, Date lastUpdateTime) { + private void storeStory(BasicDBObject story) { logger.trace(ENTER); if (!story.containsKey(IOctopusAPI.ID)) { logger.error("Missing id in story {}", story.toPrettyString(null)); @@ -737,40 +619,8 @@ public class OctopusDataMiner implements Runnable { BasicDBList storyFolderRef = storyStoryFolders.get(storyID); BasicDBList modifiedMOS = extractRelevantMOSObjects(story); - if (lastUpdateTime != null) { - rundownRef = (rundownRef == null) ? new BasicDBList() : rundownRef; - storyFolderRef = (storyFolderRef == null) ? new BasicDBList() : storyFolderRef; - modifiedMOS = (modifiedMOS == null) ? new BasicDBList() : modifiedMOS; - - boolean uptodate = true; - if (!isModified(lastUpdateTime, story)) { - BasicDBList storedRundownRef = storedStoryRundowns.get(storyID); - storedRundownRef = (storedRundownRef == null) ? new BasicDBList() : storedRundownRef; - uptodate = storedRundownRef.equals(rundownRef); - - if (uptodate) { - BasicDBList storedStoryFolderRef = storedStoryStoryFolders.get(storyID); - storedStoryFolderRef = (storedStoryFolderRef == null) ? new BasicDBList() : storedStoryFolderRef; - uptodate = storedStoryFolderRef.equals(storyFolderRef); - } - - if (uptodate) { - BasicDBList storedMOS = storedStoryMosObjects.get(storyID); - storedMOS = (storedMOS == null) ? new BasicDBList() : storedMOS; - uptodate = storedMOS.equals(modifiedMOS); - } - - if (uptodate) - return; - } - } - DBCollection collection = db.getCollection(STORY_COLLECTION); - if (lastUpdateTime != null) { - BasicDBObject orig = (BasicDBObject) collection.findOne(new BasicDBObject(IOctopusAPI.ID, storyID), new BasicDBObject(IOctopusAPI.ID, 1)); - if (orig != null) - story.put("_id", orig.getID()); - } + if (rundownRef != null) story.put(IOctopusAPI.REF_RUNDOWN, rundownRef); if (storyFolderRef != null) @@ -790,24 +640,24 @@ public class OctopusDataMiner implements Runnable { } else story.append(IOctopusAPI.PARENT_STORY_ID, parentStoryId); logger.debug(SAVING_STORY_ID, storyID); + setObjectID(collection, story); collection.save(story); logger.trace(EXIT); } private void storeStoryFolder(BasicDBObject storyFolder, Date lastUpdateTime) { logger.trace(ENTER); - BasicDBObject storyFoldersWithStories = queryStoryFolder(storyFolder); + BasicDBObject storyFoldersWithStories = queryStoryFolder(storyFolder, FIELDS_STORYFOLDER_STORIES); if (storyFoldersWithStories != null) { BasicDBList stories = NoSQLUtils.asDBList(storyFoldersWithStories, IOctopusAPI.STORIES); if (stories != null) - storeStoryFolderStories(stories, lastUpdateTime); + storeStoryFolderStories(stories); storyFolder.put(IOctopusAPI.MODIFIED, toDate(storyFolder, IOctopusAPI.MODIFIED)); DBCollection collection = db.getCollection(FOLDER_COLLECTION); - if (lastUpdateTime == null || (lastUpdateTime != null && isModified(lastUpdateTime, storyFolder))) { - String name = storyFolder.getString(IOctopusAPI.NAME); - logger.debug("Storing story folder {}", name); - collection.save(storyFolder); - } + String name = storyFolder.getString(IOctopusAPI.NAME); + logger.debug("Storing story folder {}", name); + setObjectID(collection, storyFolder); + collection.save(storyFolder); } logger.trace(EXIT); } @@ -829,11 +679,11 @@ public class OctopusDataMiner implements Runnable { logger.trace(EXIT); } - private void storeStoryFolderStories(BasicDBList stories, Date lastUpdateTime) { + private void storeStoryFolderStories(BasicDBList stories) { logger.trace(ENTER); List list = NoSQLUtils.asList(stories); for (BasicDBObject story : list) - storeStory(story, lastUpdateTime); + storeStory(story); logger.trace(EXIT); } @@ -850,6 +700,44 @@ public class OctopusDataMiner implements Runnable { return result; } + private void updateDeleteDiff(String oldCollectionName, String newCollectionName, String idFieldName) { + DBCollection oldCollection = db.getCollection(oldCollectionName); + DBCollection newCollection = db.getCollection(newCollectionName); + DBCursor oldCollectionCursor = oldCollection.find(new BasicDBObject(), new BasicDBObject(idFieldName, 1)); + if (!oldCollectionCursor.hasNext()) { + logger.error("{} collection is empty", newCollectionName); + return; + } + List oldItems = ListUtils.cast(oldCollectionCursor.toArray()); + + DBCursor newCollectionCursor = newCollection.find(); + ConcurrentHashMap newItems = null; + if (newCollectionCursor.hasNext()) { + List newList = ListUtils.cast(newCollectionCursor.toArray()); + newItems = ListUtils.map(newList, item -> item.getLong(IOctopusAPI.ID)); + } + if (newItems == null) + newItems = new ConcurrentHashMap<>(); + + for (BasicDBObject oldItem : oldItems) { + if (oldItem == null) { + logger.error("Item is null"); + continue; + } + if (!oldItem.containsKey(idFieldName)) { + logger.error("{} is null", idFieldName); + continue; + } + long id = oldItem.getLong(idFieldName); + BasicDBObject newItem = newItems.get(id); + if (newItem == null) { + //remove + logger.info("Deleting {}", oldItem.toPrettyString(null)); + oldCollection.remove(new BasicDBObject(idFieldName, id)); + } + } + } + private void updateDiff(String oldCollectionName, String newCollectionName, String idFieldName) { DBCollection oldCollection = db.getCollection(oldCollectionName); DBCollection newCollection = db.getCollection(newCollectionName); diff --git a/server/user.jobengine.executors/src/user/jobengine/server/steps/SyncOCTOPUSDataStep.java b/server/user.jobengine.executors/src/user/jobengine/server/steps/SyncOCTOPUSDataStep.java index 07510c89..64622951 100644 --- a/server/user.jobengine.executors/src/user/jobengine/server/steps/SyncOCTOPUSDataStep.java +++ b/server/user.jobengine.executors/src/user/jobengine/server/steps/SyncOCTOPUSDataStep.java @@ -3,7 +3,6 @@ package user.jobengine.server.steps; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; -import user.commons.octopus.OctopusDataMiner; import user.commons.remotestore.IProgressEventListener; import user.commons.remotestore.ProgressEvent; import user.jobengine.server.IJobEngine; @@ -24,17 +23,14 @@ public class SyncOCTOPUSDataStep extends JobStep { } @StepEntry - public Object[] execute(boolean forceFull, IJobEngine jobEngine, final IJobRuntime jobRuntime) throws Exception { + public Object[] execute(boolean includeArchived, IJobEngine jobEngine, final IJobRuntime jobRuntime) throws Exception { OctopusDataMiner dataMiner = null; try { //jobRuntime.incrementProgress(10); progressListener = createListener(jobRuntime); dataMiner = new OctopusDataMiner(); dataMiner.addProgressListener(progressListener); - if (forceFull) - dataMiner.execute(); - else - dataMiner.run(); + dataMiner.execute(includeArchived); } catch (Exception e) { logger.error(getMarker(), "Általános folyamat hiba. A rendszer hibaüzenete: {}", e.getMessage()); throw e; diff --git a/server/user.jobengine.osgi.commons/src/user/commons/octopus/IOctopusAPI.java b/server/user.jobengine.osgi.commons/src/user/commons/octopus/IOctopusAPI.java index 1dc7b900..824d4c05 100644 --- a/server/user.jobengine.osgi.commons/src/user/commons/octopus/IOctopusAPI.java +++ b/server/user.jobengine.osgi.commons/src/user/commons/octopus/IOctopusAPI.java @@ -6,9 +6,9 @@ import java.util.List; import com.ibm.nosql.json.api.DBObject; public interface IOctopusAPI { - static final String RUNDOWN_COLLECTION = "rundowns"; - static final String STORY_COLLECTION = "stories"; - static final String FOLDER_COLLECTION = "story_folders"; + static final String RUNDOWN_COLLECTION = System.getProperty("jobengine.octopus.rundowns.name", "rundowns"); + static final String STORY_COLLECTION = System.getProperty("jobengine.octopus.stories.name", "stories"); + static final String FOLDER_COLLECTION = System.getProperty("jobengine.octopus.storyfolders.name", "storyfolders"); static final String TIME_COLLECTION_NAME = "time"; static final String SCRIPT_CONTENT = "script_content"; static final String RUNDOWN_TYPE = "rundownType"; diff --git a/server/user.jobengine.osgi.commons/src/user/commons/octopus/OctopusDataMiner.java b/server/user.jobengine.osgi.commons/src/user/commons/octopus/OctopusDataMiner1.java similarity index 97% rename from server/user.jobengine.osgi.commons/src/user/commons/octopus/OctopusDataMiner.java rename to server/user.jobengine.osgi.commons/src/user/commons/octopus/OctopusDataMiner1.java index 73dac323..62ba7b56 100644 --- a/server/user.jobengine.osgi.commons/src/user/commons/octopus/OctopusDataMiner.java +++ b/server/user.jobengine.osgi.commons/src/user/commons/octopus/OctopusDataMiner1.java @@ -36,7 +36,7 @@ import user.commons.nosql.NoSQLUtils; import user.commons.remotestore.IProgressEventListener; import user.commons.remotestore.ProgressEvent; -public class OctopusDataMiner implements Runnable { +public class OctopusDataMiner1 implements Runnable { private static final String _TMP = "_tmp"; private static final Logger logger = LogManager.getLogger(); private static final String LINEFEED = "\r\n"; @@ -81,7 +81,7 @@ public class OctopusDataMiner implements Runnable { private Map newStoryFolders = new HashMap<>(); private Map newStories = new HashMap<>(); - public OctopusDataMiner() { + public OctopusDataMiner1() { db = NoSQLUtils.getNoSQLDB(); String apiAddress = System.getProperty("jobengine.octopus.api.address"); diff --git a/server/user.jobengine.osgi.commons/test/user/common/octopus/test/OctopusDataMinerTest.java b/server/user.jobengine.osgi.commons/test/user/common/octopus/test/OctopusDataMinerTest.java index 49c39444..b2957a28 100644 --- a/server/user.jobengine.osgi.commons/test/user/common/octopus/test/OctopusDataMinerTest.java +++ b/server/user.jobengine.osgi.commons/test/user/common/octopus/test/OctopusDataMinerTest.java @@ -10,16 +10,22 @@ import java.sql.SQLException; import java.util.List; import java.util.Properties; +import javax.ws.rs.core.Response; + +import org.jboss.resteasy.client.jaxrs.ResteasyClientBuilder; +import org.jboss.resteasy.client.jaxrs.ResteasyWebTarget; import org.junit.BeforeClass; import org.junit.Test; +import com.ibm.nosql.json.JSONUtil; import com.ibm.nosql.json.api.BasicDBList; import com.ibm.nosql.json.api.BasicDBObject; import com.ibm.nosql.json.api.DBObject; +import user.commons.nosql.NoSQLUtils; import user.commons.octopus.IOctopusAPI; import user.commons.octopus.OctopusAPI; -import user.commons.octopus.OctopusDataMiner; +import user.commons.octopus.OctopusDataMiner1; public class OctopusDataMinerTest { @@ -45,12 +51,12 @@ public class OctopusDataMinerTest { } - private OctopusDataMiner sut; + private OctopusDataMiner1 sut; @Test public void integrationExecute() throws Exception { // fixture - sut = new OctopusDataMiner(); + sut = new OctopusDataMiner1(); // Exercise sut.execute(); } @@ -58,7 +64,7 @@ public class OctopusDataMinerTest { @Test public void integrationTestClear() { // fixture - sut = new OctopusDataMiner(); + sut = new OctopusDataMiner1(); // Exercise sut.clear(); } @@ -66,7 +72,7 @@ public class OctopusDataMinerTest { @Test public void integrationTestWithClear() { // fixture - sut = new OctopusDataMiner(); + sut = new OctopusDataMiner1(); sut.clear(); // Exercise @@ -75,7 +81,7 @@ public class OctopusDataMinerTest { @Test public void run() { - sut = new OctopusDataMiner(); + sut = new OctopusDataMiner1(); sut.run(); } @@ -93,6 +99,34 @@ public class OctopusDataMinerTest { } } + @Test + public void testClient() { + ResteasyWebTarget webTarget = new ResteasyClientBuilder().build().target("http://10.10.1.29/services/rest/octopus/"); + Response response = null; + for (int i = 0; i < 10000; i++) { + response = webTarget.path("rundowns").request().get(); + List rundowns = NoSQLUtils.asList((BasicDBList) JSONUtil.jsonToDbObject(response.readEntity(String.class))); + System.out.println("Rundowns count: " + rundowns.size()); + // for (BasicDBObject rundown : rundowns) { + // response = webTarget.path("rundownStories/" + rundown.getLong(IOctopusAPI.ID)).request().get(); + // List stories = NoSQLUtils.asList((BasicDBList) JSONUtil.jsonToDbObject(response.readEntity(String.class))); + // System.out.println("Rundown '" + rundown.getString(IOctopusAPI.NAME) + "' stories count: " + stories.size()); + // } + + response = webTarget.path("storyFolders").request().get(); + List storyFolders = NoSQLUtils.asList((BasicDBList) JSONUtil.jsonToDbObject(response.readEntity(String.class))); + System.out.println("StoryFolders count: " + storyFolders.size()); + // for (BasicDBObject storyFolder : storyFolders) { + // response = webTarget.path("storyFolderStories/" + storyFolder.getLong(IOctopusAPI.ID)).request().get(); + // List stories = NoSQLUtils.asList((BasicDBList) JSONUtil.jsonToDbObject(response.readEntity(String.class))); + // System.out.println("StoryFolder '" + storyFolder.getString(IOctopusAPI.NAME) + "' stories count: " + stories.size()); + // } + response = webTarget.path("storyFolderStories/1504322").request().get(); + List stories = NoSQLUtils.asList((BasicDBList) JSONUtil.jsonToDbObject(response.readEntity(String.class))); + System.out.println("Híradó stories count: " + stories.size()); + } + } + @Test public void testListEquals() { BasicDBList list1 = new BasicDBList(new BasicDBObject("x", 1)); @@ -119,4 +153,5 @@ public class OctopusDataMinerTest { list2.add(new BasicDBObject("z", 3)); assertTrue(list1.equals(list2)); } + } diff --git a/server/user.jobengine.osgi.db/sql/cleanup.sql b/server/user.jobengine.osgi.db/sql/cleanup.sql index 0352c35a..0a7be31c 100644 --- a/server/user.jobengine.osgi.db/sql/cleanup.sql +++ b/server/user.jobengine.osgi.db/sql/cleanup.sql @@ -1,23 +1,25 @@ --MEDIAFILE +select mediaid from vw_mediafiles where mediafilecount = 1 + select * from mediafile f where f.mediaid in (select mediaid from vw_mediafiles where mediafilecount = 1) -delete from mediafile f where f.mediaid in (select mediaid from vw_mediafiles where mediafilecount = 1) +--delete from mediafile f where f.mediaid in (select mediaid from vw_mediafiles where mediafilecount = 1) select * from mediafile f where f.mediaid in (select id from media where created > '2017-12-08') -delete from mediafile f where f.mediaid in (select id from media where created > '2017-12-08') +--delete from mediafile f where f.mediaid in (select id from media where created > '2017-12-08') --MEDIA CREATE VIEW vw_mediafiles as select mediaid, count(*) as mediafilecount from mediafile group by (mediaid) -select m.id, m.title, m.created,f.mediafilecount from media m right outer join vw_mediafiles f on (f.mediaid = m.id) where m.created > '2017-12-08' +select m.id, m.title, m.created,f.mediafilecount from media m right outer join vw_mediafiles f on (f.mediaid = m.id) where m.created > '2017-12-14' and f.mediafilecount = 1 select m.itemid, m.id, m.itemtypeid, m.title, m.created,f.mediafilecount from media m left outer join vw_mediafiles f on (f.mediaid = m.id) where f.mediafilecount is null select m.itemid, m.id, m.itemtypeid, m.title, m.created,f.mediafilecount from media m left outer join vw_mediafiles f on (f.mediaid = m.id) where f.mediafilecount = 1 -delete from media where id in (select m.id from media m left outer join vw_mediafiles f on (f.mediaid = m.id) where f.mediafilecount is null) +--delete from media where id in (select m.id from media m left outer join vw_mediafiles f on (f.mediaid = m.id) where f.mediafilecount is null) --ITEM CREATE VIEW vw_medias as select itemid, count(*) as mediacount from media group by (itemid) select i.id, i.title, i.created, m.mediacount from item i left outer join vw_medias m on (m.itemid = i.id) where m.mediacount is null -delete from item where id in (select i.id from item i left outer join vw_medias m on (m.itemid = i.id) where m.mediacount is null) +--delete from item where id in (select i.id from item i left outer join vw_medias m on (m.itemid = i.id) where m.mediacount is null) diff --git a/server/user.jobengine.osgi.server/src/user/jobengine/server/actions/DoneSuspendAction.java b/server/user.jobengine.osgi.server/src/user/jobengine/server/actions/DoneSuspendAction.java index 597e41ce..77108218 100644 --- a/server/user.jobengine.osgi.server/src/user/jobengine/server/actions/DoneSuspendAction.java +++ b/server/user.jobengine.osgi.server/src/user/jobengine/server/actions/DoneSuspendAction.java @@ -1,5 +1,7 @@ package user.jobengine.server.actions; +import java.sql.Timestamp; + import user.commons.JobStatus; import user.jobengine.server.IJobEngine; import user.jobengine.server.IJobRuntime; @@ -10,6 +12,7 @@ public class DoneSuspendAction implements IJobStatusAction { public void processAction(IJobEngine jobEngine, IJobRuntime jobRuntime) { jobRuntime.saveStatus(); jobRuntime.setStatus(JobStatus.SUSPENDED); + jobRuntime.setFinished(new Timestamp(System.currentTimeMillis())); jobRuntime.NotifyUpdate(); jobRuntime.restoreStack(); jobRuntime.decrementInstructionPointer(); -- 2.54.0