From bc18c45871fbd9e536e0216e074a6b0579769008 Mon Sep 17 00:00:00 2001 From: elgekko Date: Fri, 24 Mar 2023 14:48:09 +0100 Subject: [PATCH] Aktualis HTV job allapot --- .../production/HIRTV/jobs/executors.xml | 81 +- .../production/HIRTV/jobs/schedules-orig.json | 318 -------- .../production/HIRTV/jobs/schedules.json | 351 +++++++- .../jobs/steps/ArchiveListBuilderStep.java | 170 ++++ .../jobs/steps/ArchiveMaterialSubmitStep.java | 55 ++ .../HIRTV/jobs/steps/BackupFileStep.java | 122 +++ .../jobs/steps/BatchRetrieveForkStep.java | 78 ++ .../HIRTV/jobs/steps/CancelableStep.java | 22 + .../jobs/steps/CheckLOWRESIntegrity.java | 66 ++ .../CheckMORPHEUSMissingMaterialsStep.java | 155 ++++ .../CheckTRAFFICMissingMaterialsStep.java | 97 +++ .../steps/CleanupMountedLocationStep.java | 296 +++++++ .../CopyForArchiveNEXIOMaterialsStep.java | 561 +++++++++++++ .../CopyForArchiveNEXIORecordingsStep.java | 455 +++++++++++ .../jobs/steps/CreateArchiveItemStep.java | 45 ++ .../jobs/steps/CreateMissingLowresStep.java | 92 +++ .../HIRTV/jobs/steps/DeleteFileStep.java | 36 + .../jobs/steps/DeleteNEXIOMaterialsStep.java | 121 +++ .../jobs/steps/DetectMissingLengthStep.java | 68 ++ .../steps/DownloadRecordingFromNexioStep.java | 182 +++++ .../jobs/steps/DuplicateRemoverStep.java | 102 +++ .../steps/GenerateMorpheusMetadataStep.java | 32 + .../HIRTV/jobs/steps/HLSProxyStep.java | 77 ++ .../ImportMORPHEUSMissingMaterialsStep.java | 255 ++++++ .../jobs/steps/ImportStatisticsStep.java | 365 +++++++++ .../HIRTV/jobs/steps/MXFCutterStep.java | 137 ++++ .../HIRTV/jobs/steps/MediaToolStep.java | 31 + .../jobs/steps/MetadataTransformStep.java | 161 ++++ .../steps/OutputPathAndNameSelectorStep.java | 161 ++++ .../ProjectCleanupMountedLocationStep.java | 225 ++++++ .../RecordingsArchiveItemBuilderStep.java | 322 ++++++++ .../HIRTV/jobs/steps/SyncOCTOPUSDataStep.java | 32 + .../HIRTV/jobs/steps/TSMBackupStep.java | 231 ++++++ .../jobs/steps/TSMExtendedRetrieveStep.java | 92 +++ .../HIRTV/jobs/steps/TSMRestoreStep.java | 193 +++++ .../steps/TSMRetrieveMissingMaterialStep.java | 45 ++ .../jobs/steps/TSMSystemRestoreStep.java | 151 ++++ .../jobs/steps/TranscodeFFAStranStep.java | 157 ++++ .../jobs/steps/TranscodeSELENIOStep.java | 243 ++++++ .../HIRTV/jobs/steps/TranscodeStep.java | 32 + .../jobs/steps/UpdateGhostMediaDataStep.java | 89 ++ .../steps/UploadRecordingToNexioStep.java | 147 ++++ .../HIRTV/jobs/steps/shared/Cmd.java | 82 ++ .../HIRTV/jobs/steps/shared/EscortFiles.java | 378 +++++++++ .../jobs/steps/shared/ExternalCommand.java | 79 ++ .../steps/shared/ExternalCommandExecutor.java | 32 + .../jobs/steps/shared/ExternalProfile.java | 33 + .../steps/shared/ExternalProfilesConfig.java | 15 + .../HIRTV/jobs/steps/shared/FFMpeg.java | 225 ++++++ .../steps/shared/FileSearchFilterOptions.java | 41 + .../jobs/steps/shared/IExternalCallback.java | 5 + .../steps/shared/ItemManagerExtensions.java | 95 +++ .../jobs/steps/shared/MediaCubeClient.java | 77 ++ .../jobs/steps/shared/MetadataSaver.java | 44 + .../HIRTV/jobs/steps/shared/MetadataType.java | 5 + .../steps/shared/MetadataTypeDetector.java | 49 ++ .../jobs/steps/shared/OctopusDataMiner.java | 759 ++++++++++++++++++ .../jobs/steps/shared/PlanAirExtensions.java | 300 +++++++ 58 files changed, 8478 insertions(+), 392 deletions(-) delete mode 100644 server/-product/production/HIRTV/jobs/schedules-orig.json create mode 100644 server/-product/production/HIRTV/jobs/steps/ArchiveListBuilderStep.java create mode 100644 server/-product/production/HIRTV/jobs/steps/ArchiveMaterialSubmitStep.java create mode 100644 server/-product/production/HIRTV/jobs/steps/BackupFileStep.java create mode 100644 server/-product/production/HIRTV/jobs/steps/BatchRetrieveForkStep.java create mode 100644 server/-product/production/HIRTV/jobs/steps/CancelableStep.java create mode 100644 server/-product/production/HIRTV/jobs/steps/CheckLOWRESIntegrity.java create mode 100644 server/-product/production/HIRTV/jobs/steps/CheckMORPHEUSMissingMaterialsStep.java create mode 100644 server/-product/production/HIRTV/jobs/steps/CheckTRAFFICMissingMaterialsStep.java create mode 100644 server/-product/production/HIRTV/jobs/steps/CleanupMountedLocationStep.java create mode 100644 server/-product/production/HIRTV/jobs/steps/CopyForArchiveNEXIOMaterialsStep.java create mode 100644 server/-product/production/HIRTV/jobs/steps/CopyForArchiveNEXIORecordingsStep.java create mode 100644 server/-product/production/HIRTV/jobs/steps/CreateArchiveItemStep.java create mode 100644 server/-product/production/HIRTV/jobs/steps/CreateMissingLowresStep.java create mode 100644 server/-product/production/HIRTV/jobs/steps/DeleteFileStep.java create mode 100644 server/-product/production/HIRTV/jobs/steps/DeleteNEXIOMaterialsStep.java create mode 100644 server/-product/production/HIRTV/jobs/steps/DetectMissingLengthStep.java create mode 100644 server/-product/production/HIRTV/jobs/steps/DownloadRecordingFromNexioStep.java create mode 100644 server/-product/production/HIRTV/jobs/steps/DuplicateRemoverStep.java create mode 100644 server/-product/production/HIRTV/jobs/steps/GenerateMorpheusMetadataStep.java create mode 100644 server/-product/production/HIRTV/jobs/steps/HLSProxyStep.java create mode 100644 server/-product/production/HIRTV/jobs/steps/ImportMORPHEUSMissingMaterialsStep.java create mode 100644 server/-product/production/HIRTV/jobs/steps/ImportStatisticsStep.java create mode 100644 server/-product/production/HIRTV/jobs/steps/MXFCutterStep.java create mode 100644 server/-product/production/HIRTV/jobs/steps/MediaToolStep.java create mode 100644 server/-product/production/HIRTV/jobs/steps/MetadataTransformStep.java create mode 100644 server/-product/production/HIRTV/jobs/steps/OutputPathAndNameSelectorStep.java create mode 100644 server/-product/production/HIRTV/jobs/steps/ProjectCleanupMountedLocationStep.java create mode 100644 server/-product/production/HIRTV/jobs/steps/RecordingsArchiveItemBuilderStep.java create mode 100644 server/-product/production/HIRTV/jobs/steps/SyncOCTOPUSDataStep.java create mode 100644 server/-product/production/HIRTV/jobs/steps/TSMBackupStep.java create mode 100644 server/-product/production/HIRTV/jobs/steps/TSMExtendedRetrieveStep.java create mode 100644 server/-product/production/HIRTV/jobs/steps/TSMRestoreStep.java create mode 100644 server/-product/production/HIRTV/jobs/steps/TSMRetrieveMissingMaterialStep.java create mode 100644 server/-product/production/HIRTV/jobs/steps/TSMSystemRestoreStep.java create mode 100644 server/-product/production/HIRTV/jobs/steps/TranscodeFFAStranStep.java create mode 100644 server/-product/production/HIRTV/jobs/steps/TranscodeSELENIOStep.java create mode 100644 server/-product/production/HIRTV/jobs/steps/TranscodeStep.java create mode 100644 server/-product/production/HIRTV/jobs/steps/UpdateGhostMediaDataStep.java create mode 100644 server/-product/production/HIRTV/jobs/steps/UploadRecordingToNexioStep.java create mode 100644 server/-product/production/HIRTV/jobs/steps/shared/Cmd.java create mode 100644 server/-product/production/HIRTV/jobs/steps/shared/EscortFiles.java create mode 100644 server/-product/production/HIRTV/jobs/steps/shared/ExternalCommand.java create mode 100644 server/-product/production/HIRTV/jobs/steps/shared/ExternalCommandExecutor.java create mode 100644 server/-product/production/HIRTV/jobs/steps/shared/ExternalProfile.java create mode 100644 server/-product/production/HIRTV/jobs/steps/shared/ExternalProfilesConfig.java create mode 100644 server/-product/production/HIRTV/jobs/steps/shared/FFMpeg.java create mode 100644 server/-product/production/HIRTV/jobs/steps/shared/FileSearchFilterOptions.java create mode 100644 server/-product/production/HIRTV/jobs/steps/shared/IExternalCallback.java create mode 100644 server/-product/production/HIRTV/jobs/steps/shared/ItemManagerExtensions.java create mode 100644 server/-product/production/HIRTV/jobs/steps/shared/MediaCubeClient.java create mode 100644 server/-product/production/HIRTV/jobs/steps/shared/MetadataSaver.java create mode 100644 server/-product/production/HIRTV/jobs/steps/shared/MetadataType.java create mode 100644 server/-product/production/HIRTV/jobs/steps/shared/MetadataTypeDetector.java create mode 100644 server/-product/production/HIRTV/jobs/steps/shared/OctopusDataMiner.java create mode 100644 server/-product/production/HIRTV/jobs/steps/shared/PlanAirExtensions.java diff --git a/server/-product/production/HIRTV/jobs/executors.xml b/server/-product/production/HIRTV/jobs/executors.xml index 08332f19..c843940c 100644 --- a/server/-product/production/HIRTV/jobs/executors.xml +++ b/server/-product/production/HIRTV/jobs/executors.xml @@ -1,49 +1,42 @@ - - - - - - - - + + + + + + + + + + + + + - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/server/-product/production/HIRTV/jobs/schedules-orig.json b/server/-product/production/HIRTV/jobs/schedules-orig.json deleted file mode 100644 index 076b6fc0..00000000 --- a/server/-product/production/HIRTV/jobs/schedules-orig.json +++ /dev/null @@ -1,318 +0,0 @@ -{"joblist":[ - { - "template": "cancelable.xml" - }, - { - "active": true, - "executeimmediate": true, - "name" : "OCTOPUS adatok szinkronizálása", - "template": "sync-octopus.xml", - "cronexpression": "0/30 * * * * ?", - "parameters": [ - {"name": "includeArchived", "value": false, "type": "java.lang.Boolean"}, - {"name": "address", "value": "http://10.10.1.10/api/v1", "type": "java.lang.String"}, - {"name": "user", "value": "mam", "type": "java.lang.String"}, - {"name": "pwd", "value": "napocska", "type": "java.lang.String"} - ] - }, - { - "active": true, - "executeimmediate": false, - "name" : "Limitált archiválás az ISILON/ARCHIVE mappából", - "template": "archive-limited.xml", - "cronexpression": "0 0 6-22/2 * * ?", - "parameters": [ - {"name": "sourcePath", "value": "/mnt/ISILON/ARCHIVE", "type": "java.lang.String"}, - {"name": "killDateDays", "value": 1, "type": "java.lang.Integer"}, - {"name": "limit", "value": 20, "type": "java.lang.Integer"} - ] - }, - { - "active": true, - "executeimmediate": false, - "name" : "Archiválás az ISILON/ARCHIVE mappából", - "template": "archive-ondemand.xml", - "cronexpression": "0 0 1 * * ?", - "parameters": [ - {"name": "sourcePath", "value": "/mnt/ISILON/ARCHIVE", "type": "java.lang.String"}, - {"name": "killDateDays", "value": 1, "type": "java.lang.Integer"}, - {"name": "limit", "value": 0, "type": "java.lang.Integer"} - ] - }, - { - "active": true, - "executeimmediate": false, - "name" : "PROXY pótlás FFASTRANS 71", - "template": "sys-recreate-lowres-71.xml", - "cronexpression": "0 * * * * ?", - "parameters": [ - {"name": "localLowresPath", "value": "/mnt/FIXTRANSCODER-71/OUTPUT", "type": "java.lang.String" }, - {"name": "transcoderAddress", "value": "http://10.10.1.71:65445/api/json/v1/", "type": "java.lang.String"}, - {"name": "globalRetrievePath", "value": "file://10.10.1.30/transcode", "type": "java.lang.String"}, - {"name": "localRetrievePath", "value": "/mediacube/data/lowres/www/video/IFT3/transcode", "type": "java.lang.String"}, - {"name": "localHiresPath", "value": "/mediacube/data/lowres/www/video/IFT3/transcode", "type": "java.lang.String" }, - {"name": "globalHiresPath", "value": "L:\\transcode", "type": "java.lang.String" }, - {"name": "transcoderTemplateName", "value": "MAM_proxy", "type": "java.lang.String"} - ] - }, - { - "active": false, - "executeimmediate": false, - "name" : "PROXY pótlás FFASTRANS 72", - "template": "sys-recreate-lowres-72.xml", - "cronexpression": "10 * * * * ?", - "parameters": [ - {"name": "localLowresPath", "value": "/mnt/FIXTRANSCODER-72/OUTPUT", "type": "java.lang.String" }, - {"name": "transcoderAddress", "value": "http://10.10.1.72:65445/api/json/v1/", "type": "java.lang.String"}, - {"name": "globalRetrievePath", "value": "file://10.10.1.30/transcode", "type": "java.lang.String"}, - {"name": "localRetrievePath", "value": "/mediacube/data/lowres/www/video/IFT3/transcode", "type": "java.lang.String"}, - {"name": "localHiresPath", "value": "/mediacube/data/lowres/www/video/IFT3/transcode", "type": "java.lang.String" }, - {"name": "globalHiresPath", "value": "L:\\transcode", "type": "java.lang.String" }, - {"name": "transcoderTemplateName", "value": "MAM_proxy", "type": "java.lang.String"} - ] - }, - { - "active": false, - "executeimmediate": false, - "name" : "PROXY pótlás FFASTRANS 73", - "template": "sys-recreate-lowres-73.xml", - "cronexpression": "20 * * * * ?", - "parameters": [ - {"name": "localLowresPath", "value": "/mnt/FIXTRANSCODER-73/OUTPUT", "type": "java.lang.String" }, - {"name": "transcoderAddress", "value": "http://10.10.1.73:65445/api/json/v1/", "type": "java.lang.String"}, - {"name": "globalRetrievePath", "value": "file://10.10.1.30/transcode", "type": "java.lang.String"}, - {"name": "localRetrievePath", "value": "/mediacube/data/lowres/www/video/IFT3/transcode", "type": "java.lang.String"}, - {"name": "localHiresPath", "value": "/mediacube/data/lowres/www/video/IFT3/transcode", "type": "java.lang.String" }, - {"name": "globalHiresPath", "value": "L:\\transcode", "type": "java.lang.String" }, - {"name": "transcoderTemplateName", "value": "MAM_proxy", "type": "java.lang.String"} - ] - }, - { - "active": false, - "executeimmediate": false, - "name" : "PROXY pótlás FFASTRANS 74", - "template": "sys-recreate-lowres-74.xml", - "cronexpression": "30 * * * * ?", - "parameters": [ - {"name": "localLowresPath", "value": "/mnt/FIXTRANSCODER-74/OUTPUT", "type": "java.lang.String" }, - {"name": "transcoderAddress", "value": "http://10.10.1.74:65445/api/json/v1/", "type": "java.lang.String"}, - {"name": "globalRetrievePath", "value": "file://10.10.1.30/transcode", "type": "java.lang.String"}, - {"name": "localRetrievePath", "value": "/mediacube/data/lowres/www/video/IFT3/transcode", "type": "java.lang.String"}, - {"name": "localHiresPath", "value": "/mediacube/data/lowres/www/video/IFT3/transcode", "type": "java.lang.String" }, - {"name": "globalHiresPath", "value": "L:\\transcode", "type": "java.lang.String" }, - {"name": "transcoderTemplateName", "value": "MAM_proxy", "type": "java.lang.String"} - ] - }, - { - "name" : "SYS: create-lowres-ondemand", - "template": "create-lowres-ondemand.xml", - "parameters": [ - {"name": "globalRetrievePath", "value": "file://10.10.1.30/transcode", "type": "java.lang.String"}, - {"name": "localRetrievePath", "value": "/mediacube/data/lowres/www/video/IFT3/transcode", "type": "java.lang.String"}, - {"name": "localHiresPath", "value": "/mediacube/data/lowres/www/video/IFT3/transcode", "type": "java.lang.String" }, - {"name": "globalHiresPath", "value": "L:\\transcode", "type": "java.lang.String" }, - {"name": "localLowresPath", "value": "/mnt/FIXTRANSCODER/OUTPUT", "type": "java.lang.String" }, - {"name": "transcoderAddress", "value": "http://10.10.1.74:65445/api/json/v1/", "type": "java.lang.String"}, - {"name": "transcoderTemplateName", "value": "MAM_proxy", "type": "java.lang.String"} - ] - }, - { - "active": true, - "executeimmediate": false, - "name" : "Lejárt ISISLON/ARCHIVE anyagok törlése", - "template": "delete-materials1.xml", - "cronexpression": "0 0 5 * * ?", - "parameters": [ - {"name": "sourcePath", "value": "/mnt/ISILON/ARCHIVE", "type": "java.lang.String"}, - {"name": "skipArchiveCheck", "value": true, "type": "java.lang.Boolean"} - ] - }, - { - "active": true, - "executeimmediate": false, - "name" : "Lejárt ISILON/TQC/CHECK/KESZ anyagok törlése", - "template": "delete-materials2.xml", - "cronexpression": "0 0 8 * * ?", - "parameters": [ - {"name": "sourcePath", "value": "/mnt/ISILON/TQC/CHECK/KESZ", "type": "java.lang.String"}, - {"name": "skipArchiveCheck", "value": true, "type": "java.lang.Boolean"} - ] - }, - { - "active": true, - "executeimmediate": false, - "name" : "Lejárt ISILON/TQC/PROMO/KESZ anyagok törlése", - "template": "delete-materials3.xml", - "cronexpression": "0 0 6 * * ?", - "parameters": [ - {"name": "sourcePath", "value": "/mnt/ISILON/TQC/PROMO/KESZ", "type": "java.lang.String"}, - {"name": "skipArchiveCheck", "value": true, "type": "java.lang.Boolean"} - ] - }, - { - "active": true, - "executeimmediate": false, - "name" : "Lejárt ISILON/TQC/REKLAM/KESZ anyagok törlése", - "template": "delete-materials4.xml", - "cronexpression": "0 0 7 * * ?", - "parameters": [ - {"name": "sourcePath", "value": "/mnt/ISILON/TQC/REKLAM/KESZ", "type": "java.lang.String"}, - {"name": "skipArchiveCheck", "value": true, "type": "java.lang.Boolean"} - ] - }, - { - "active": true, - "executeimmediate": false, - "name" : "Lejárt ISILON/OCTOPUS/_NAPI_MEGTEKINTO anyagok törlése", - "template": "delete-materials5.xml", - "cronexpression": "0 0 22 * * ?", - "parameters": [ - {"name": "sourcePath", "value": "/mnt/ISILON/OCTOPUS/_NAPI_MEGTEKINTO", "type": "java.lang.String"}, - {"name": "skipArchiveCheck", "value": true, "type": "java.lang.Boolean"} - ] - }, - { - "active": true, - "executeimmediate": false, - "name" : "Lejárt ISISLON/PROMO_NLE mappák törlése", - "template": "delete-promo-materials.xml", - "cronexpression": "0 0 6 * * ?", - "parameters": [ - {"name": "sourcePath", "value": "/mnt/ISILON/PROMO_NLE", "type": "java.lang.String"} - ] - }, - { - "active": true, - "executeimmediate": false, - "name" : "NEXIO bejátszó anyagok másolása az ISILON/ARCHIVE mappába", - "template": "copyforarchive-nexio-materials.xml", - "cronexpression": "0 0 10 * * ?", - "parameters": [ - {"name": "nexioPort", "value": 2098, "type": "java.lang.Integer"}, - {"name": "nexioUserName", "value": "administrator", "type": "java.lang.String"}, - {"name": "nexioPassword", "value": "system", "type": "java.lang.String"}, - {"name": "archiveFtp", "value": "ftp://10.10.1.100/ARCHIVE", "type": "java.lang.String"}, - {"name": "archiveUserName", "value": "mediacube", "type": "java.lang.String"}, - {"name": "archivePassword", "value": "Broadca5T", "type": "java.lang.String"}, - {"name": "daysBeforeNow", "value": 1, "type": "java.lang.Integer"}, - {"name": "nexioKillDateDays", "value": 21, "type": "java.lang.Integer"}, - {"name": "nexioAgency", "value": "ARCHIVED", "type": "java.lang.String"} - ] - }, - { - "active": true, - "executeimmediate": false, - "name" : "NEXIO visszarögzített anyagok másolása az ISILON/ARCHIVE mappába", - "template": "copyforarchive-nexio-recordings.xml", - "cronexpression": "0 0 12 * * ? *", - "parameters": [ - {"name": "nexioPort", "value": 2098, "type": "java.lang.Integer"}, - {"name": "nexioUserName", "value": "administrator", "type": "java.lang.String"}, - {"name": "nexioPassword", "value": "system", "type": "java.lang.String"}, - {"name": "archiveFtp", "value": "ftp://10.10.1.100/ARCHIVE", "type": "java.lang.String"}, - {"name": "archiveUserName", "value": "mediacube", "type": "java.lang.String"}, - {"name": "archivePassword", "value": "Broadca5T", "type": "java.lang.String"}, - {"name": "filterAgencies", "value": "schedule-rec", "type": "java.lang.String"}, - {"name": "limit", "value": 30, "type": "java.lang.Integer"}, - {"name": "nexioKillDateDays", "value": 7, "type": "java.lang.Integer"}, - {"name": "nexioAgency", "value": "HIRADO_ARCHIVED", "type": "java.lang.String"} - ] - }, - { - "active": true, - "executeimmediate": false, - "name" : "Híranyag statisztika importálása", - "template": "import-statistics.xml", - "cronexpression": "0 0 6 * * ?", - "parameters": [ - {"name": "daysBeforeNow", "value": 1, "type": "java.lang.Integer"} - ] - }, - { - "active": false, - "executeimmediate": true, - "name" : "TRAFFIC anyagok visszatöltése", - "template": "retrieve-traffic-missing-materials.xml", - "cronexpression": "0 0 0/1 1/1 * ? *", - "parameters": [ - {"name": "dbUrl", "value": "jdbc:sqlserver://10.10.1.45\\sql16;databaseName=PA_Echo;", "type": "java.lang.String"}, - {"name": "userName", "value": "MAM", "type": "java.lang.String"}, - {"name": "password", "value": "Echotv.hu", "type": "java.lang.String"}, - {"name": "lookupDays", "value": 7, "type": "java.lang.Integer"}, - {"name": "targetPath", "value": "/mnt/ISILON/PLAYOUT/IceGateway/Input", "type": "java.lang.String"}, - {"name": "killDateDays", "value": 10, "type": "java.lang.Integer"} - ] - }, - { - "active": true, - "executeimmediate": false, - "name" : "Lejárt NEXIO anyagok törlése", - "template": "delete-nexio-materials.xml", - "cronexpression": "0 0 6 * * ?", - "parameters": [ - {"name": "port", "value": 2098, "type": "java.lang.Integer"}, - {"name": "userName", "value": "administrator", "type": "java.lang.String"}, - {"name": "password", "value": "system", "type": "java.lang.String"}, - {"name": "filterAgencies", "value": "HIRADO_23_00,HIRADO_ARCHIVED,HIRADO_CLN", "type": "java.lang.String"}, - {"name": "gracePeriodDays", "value": 1, "type": "java.lang.Integer"}, - {"name": "notificationOnly", "value": false, "type": "java.lang.Boolean"} - ] - }, - { - "name" : "SYS: MORPHEUS 'missing materials' importálása", - "template": "import-morpheus-missing-materials.xml", - "parameters": [ - {"name": "csvFilePath", "value": "/mnt/MORPHEUS", "type": "java.lang.String"}, - {"name": "processedFolder", "value": "DONE", "type": "java.lang.String"}, - {"name": "targetPath", "value": "/mnt/ISILON/ARCHIVE_RESTORE", "type": "java.lang.String"} - ] - }, - { - "name" : "SYS: batch-retrieve-ondemand", - "template": "batch-retrieve-ondemand.xml" - }, - { - "name" : "SYS: retrieve-ondemand", - "template": "retrieve-ondemand.xml", - "parameters": [ - {"name": "globalRetrievePath", "value": "file://isilon.intra.echotv.hu", "type": "java.lang.String"}, - {"name": "localRetrievePath", "value": "/mnt/ISILON", "type": "java.lang.String"}, - {"name": "materialOutputFolder", "value": "PLAYOUT_NLE", "type": "java.lang.String"}, - {"name": "promoOutputFolder", "value": "PROMO_NLE", "type": "java.lang.String"}, - {"name": "advertisementOutputFolder", "value": "REKLAM_NLE", "type": "java.lang.String"}, - {"name": "octopusOutputFolder", "value": "OCTOPUS", "type": "java.lang.String"}, - {"name": "genericOutputFolder", "value": "ARCHIVE_RESTORE", "type": "java.lang.String"}, - {"name": "onlineOutputFolder", "value": "ONLINE", "type": "java.lang.String"}, - {"name": "killDateDays", "value": 7, "type": "java.lang.Integer"}, - {"name": "nexioAgency", "value": "ARCHIVE_RESTORE", "type": "java.lang.String"}, - {"name": "nexioPort", "value": 2098, "type": "java.lang.Integer"}, - {"name": "nexioUserName", "value": "administrator", "type": "java.lang.String"}, - {"name": "nexioPassword", "value": "system", "type": "java.lang.String"} - ] - }, - { - "name" : "sys: MORPHEUS 'missing materials' importálása", - "template": "sys-import-morpheus-missing-materials.xml", - "parameters": [ - {"name": "csvFilePath", "value": "/mnt/MORPHEUS", "type": "java.lang.String"}, - {"name": "processedFolder", "value": "DONE", "type": "java.lang.String"}, - {"name": "targetPath", "value": "/mnt/ISILON/PLAYOUT/Video", "type": "java.lang.String"} - ] - }, - { - "name" : "sys: MORPHEUS 'missing material' visszatöltése", - "template": "sys-retrieve-missing-material.xml", - "parameters": [ - {"name": "targetPath", "value": "/mnt/ISILON/PLAYOUT/Video", "type": "java.lang.String"}, - {"name": "globalRetrievePath", "value": "\\\\10.10.1.100\\BRAAVOS\\PLAYOUT\\Video", "type": "java.lang.String"}, - {"name": "morpheusDeviceID", "value": "ISILON", "type": "java.lang.String"}, - {"name": "dbUrl", "value": "jdbc:sqlserver://10.10.1.45;databaseName=PA_Echo;", "type": "java.lang.String"}, - {"name": "userName", "value": "MAM", "type": "java.lang.String"}, - {"name": "password", "value": "Echotv.hu", "type": "java.lang.String"}, - {"name": "targetMetadataPath", "value": "/mnt/ISILON/PLAYOUT/MorpheusGateway/Input", "type": "java.lang.String"} - ] - } -]} - - - diff --git a/server/-product/production/HIRTV/jobs/schedules.json b/server/-product/production/HIRTV/jobs/schedules.json index 1697c76c..3c4faed1 100644 --- a/server/-product/production/HIRTV/jobs/schedules.json +++ b/server/-product/production/HIRTV/jobs/schedules.json @@ -1,30 +1,321 @@ -{ - "joblist": [ - { - "template": "backup-file.xml" - }, - { - "active": false, - "cronexpression": "0/2 * * * * ?", - "template": "cancelable.xml", - "parameters": [ - { - "name": "param", - "value": 100, - "type": "java.lang.Integer" - } - ] - }, - { - "cronexpression": "0/2 * * * * ?", - "template": "sys-fork-recreate-lowres.xml", - "parameters": [ - { - "name": "localHiresPath", - "value": "c:/temp", - "type": "java.lang.String" - } - ] - } - ] -} +{"joblist":[ + { + "template": "backup-file.xml" + }, + { + "template": "cancelable.xml" + }, + { + "active": true, + "executeimmediate": true, + "name" : "OCTOPUS adatok szinkronizálása", + "template": "sync-octopus.xml", + "cronexpression": "0/30 * * * * ?", + "parameters": [ + {"name": "includeArchived", "value": false, "type": "java.lang.Boolean"}, + {"name": "address", "value": "http://10.10.1.10/api/v1", "type": "java.lang.String"}, + {"name": "user", "value": "mam", "type": "java.lang.String"}, + {"name": "pwd", "value": "napocska", "type": "java.lang.String"} + ] + }, + { + "active": true, + "executeimmediate": false, + "name" : "Limitált archiválás az ISILON/ARCHIVE mappából", + "template": "archive-limited.xml", + "cronexpression": "0 0 6-22/1 * * ?", + "parameters": [ + {"name": "sourcePath", "value": "/mnt/ISILON/ARCHIVE", "type": "java.lang.String"}, + {"name": "killDateDays", "value": 1, "type": "java.lang.Integer"}, + {"name": "limit", "value": 10, "type": "java.lang.Integer"} + ] + }, + { + "active": true, + "executeimmediate": false, + "name" : "Archiválás az ISILON/ARCHIVE mappából", + "template": "archive-ondemand.xml", + "cronexpression": "0 0 1 * * ?", + "parameters": [ + {"name": "sourcePath", "value": "/mnt/ISILON/ARCHIVE", "type": "java.lang.String"}, + {"name": "killDateDays", "value": 1, "type": "java.lang.Integer"}, + {"name": "limit", "value": 0, "type": "java.lang.Integer"} + ] + }, + { + "active": true, + "executeimmediate": false, + "name" : "PROXY pótlás FFASTRANS 71", + "template": "sys-recreate-lowres-71.xml", + "cronexpression": "0 0/10 * * * ?", + "parameters": [ + {"name": "localLowresPath", "value": "/mnt/FIXTRANSCODER-71/OUTPUT", "type": "java.lang.String" }, + {"name": "transcoderAddress", "value": "http://10.10.1.71:65445/api/json/v1/", "type": "java.lang.String"}, + {"name": "globalRetrievePath", "value": "file://10.10.1.30/transcode", "type": "java.lang.String"}, + {"name": "localRetrievePath", "value": "/mnt/IFT3/transcode", "type": "java.lang.String"}, + {"name": "localHiresPath", "value": "/mnt/IFT3/transcode", "type": "java.lang.String" }, + {"name": "globalHiresPath", "value": "L:\\transcode", "type": "java.lang.String" }, + {"name": "transcoderTemplateName", "value": "MAM_proxy", "type": "java.lang.String"} + ] + }, + { + "active": true, + "executeimmediate": false, + "name" : "PROXY pótlás FFASTRANS 72", + "template": "sys-recreate-lowres-72.xml", + "cronexpression": "0 0/10 * * * ?", + "parameters": [ + {"name": "localLowresPath", "value": "/mnt/FIXTRANSCODER-72/OUTPUT", "type": "java.lang.String" }, + {"name": "transcoderAddress", "value": "http://10.10.1.72:65445/api/json/v1/", "type": "java.lang.String"}, + {"name": "globalRetrievePath", "value": "file://10.10.1.30/transcode", "type": "java.lang.String"}, + {"name": "localRetrievePath", "value": "/mnt/IFT3/transcode", "type": "java.lang.String"}, + {"name": "localHiresPath", "value": "/mnt/IFT3/transcode", "type": "java.lang.String" }, + {"name": "globalHiresPath", "value": "L:\\transcode", "type": "java.lang.String" }, + {"name": "transcoderTemplateName", "value": "MAM_proxy", "type": "java.lang.String"} + ] + }, + { + "active": true, + "executeimmediate": false, + "name" : "PROXY pótlás FFASTRANS 73", + "template": "sys-recreate-lowres-73.xml", + "cronexpression": "0 0/10 * * * ?", + "parameters": [ + {"name": "localLowresPath", "value": "/mnt/FIXTRANSCODER-73/OUTPUT", "type": "java.lang.String" }, + {"name": "transcoderAddress", "value": "http://10.10.1.73:65445/api/json/v1/", "type": "java.lang.String"}, + {"name": "globalRetrievePath", "value": "file://10.10.1.30/transcode", "type": "java.lang.String"}, + {"name": "localRetrievePath", "value": "/mnt/IFT3/transcode", "type": "java.lang.String"}, + {"name": "localHiresPath", "value": "/mnt/IFT3/transcode", "type": "java.lang.String" }, + {"name": "globalHiresPath", "value": "L:\\transcode", "type": "java.lang.String" }, + {"name": "transcoderTemplateName", "value": "MAM_proxy", "type": "java.lang.String"} + ] + }, + { + "active": true, + "executeimmediate": false, + "name" : "PROXY pótlás FFASTRANS 74", + "template": "sys-recreate-lowres-74.xml", + "cronexpression": "0 0/10 * * * ?", + "parameters": [ + {"name": "localLowresPath", "value": "/mnt/FIXTRANSCODER-74/OUTPUT", "type": "java.lang.String" }, + {"name": "transcoderAddress", "value": "http://10.10.1.74:65445/api/json/v1/", "type": "java.lang.String"}, + {"name": "globalRetrievePath", "value": "file://10.10.1.30/transcode", "type": "java.lang.String"}, + {"name": "localRetrievePath", "value": "/mnt/IFT3/transcode", "type": "java.lang.String"}, + {"name": "localHiresPath", "value": "/mnt/IFT3/transcode", "type": "java.lang.String" }, + {"name": "globalHiresPath", "value": "L:\\transcode", "type": "java.lang.String" }, + {"name": "transcoderTemplateName", "value": "MAM_proxy", "type": "java.lang.String"} + ] + }, + { + "name" : "SYS: create-lowres-ondemand", + "template": "create-lowres-ondemand.xml", + "parameters": [ + {"name": "localLowresPath", "value": "/mnt/FIXTRANSCODER-71/OUTPUT", "type": "java.lang.String" }, + {"name": "transcoderAddress", "value": "http://10.10.1.71:65445/api/json/v1/", "type": "java.lang.String"}, + {"name": "globalRetrievePath", "value": "file://10.10.1.30/transcode", "type": "java.lang.String"}, + {"name": "localRetrievePath", "value": "/mnt/IFT3/transcode", "type": "java.lang.String"}, + {"name": "localHiresPath", "value": "/mnt/IFT3/transcode", "type": "java.lang.String" }, + {"name": "globalHiresPath", "value": "L:\\transcode", "type": "java.lang.String" }, + {"name": "transcoderTemplateName", "value": "MAM_proxy", "type": "java.lang.String"} + ] + }, + { + "active": true, + "executeimmediate": false, + "name" : "Lejárt ISISLON/ARCHIVE anyagok törlése", + "template": "delete-materials1.xml", + "cronexpression": "0 0 5 * * ?", + "parameters": [ + {"name": "sourcePath", "value": "/mnt/ISILON/ARCHIVE", "type": "java.lang.String"}, + {"name": "skipArchiveCheck", "value": false, "type": "java.lang.Boolean"} + ] + }, + { + "active": true, + "executeimmediate": false, + "name" : "Lejárt ISILON/TQC/CHECK/KESZ anyagok törlése", + "template": "delete-materials2.xml", + "cronexpression": "0 0 8 * * ?", + "parameters": [ + {"name": "sourcePath", "value": "/mnt/ISILON/TQC/CHECK/KESZ", "type": "java.lang.String"}, + {"name": "skipArchiveCheck", "value": true, "type": "java.lang.Boolean"} + ] + }, + { + "active": true, + "executeimmediate": false, + "name" : "Lejárt ISILON/TQC/PROMO/KESZ anyagok törlése", + "template": "delete-materials3.xml", + "cronexpression": "0 0 6 * * ?", + "parameters": [ + {"name": "sourcePath", "value": "/mnt/ISILON/TQC/PROMO/KESZ", "type": "java.lang.String"}, + {"name": "skipArchiveCheck", "value": true, "type": "java.lang.Boolean"} + ] + }, + { + "active": true, + "executeimmediate": false, + "name" : "Lejárt ISILON/TQC/REKLAM/KESZ anyagok törlése", + "template": "delete-materials4.xml", + "cronexpression": "0 0 7 * * ?", + "parameters": [ + {"name": "sourcePath", "value": "/mnt/ISILON/TQC/REKLAM/KESZ", "type": "java.lang.String"}, + {"name": "skipArchiveCheck", "value": true, "type": "java.lang.Boolean"} + ] + }, + { + "active": true, + "executeimmediate": false, + "name" : "Lejárt ISILON/OCTOPUS/_NAPI_MEGTEKINTO anyagok törlése", + "template": "delete-materials5.xml", + "cronexpression": "0 0 22 * * ?", + "parameters": [ + {"name": "sourcePath", "value": "/mnt/ISILON/OCTOPUS/_NAPI_MEGTEKINTO", "type": "java.lang.String"}, + {"name": "skipArchiveCheck", "value": true, "type": "java.lang.Boolean"} + ] + }, + { + "active": true, + "executeimmediate": false, + "name" : "Lejárt ISISLON/PROMO_NLE mappák törlése", + "template": "delete-promo-materials.xml", + "cronexpression": "0 0 6 * * ?", + "parameters": [ + {"name": "sourcePath", "value": "/mnt/ISILON/PROMO_NLE", "type": "java.lang.String"} + ] + }, + { + "active": true, + "executeimmediate": false, + "name" : "NEXIO bejátszó anyagok másolása az ISILON/ARCHIVE mappába", + "template": "copyforarchive-nexio-materials.xml", + "cronexpression": "0 0 10 * * ?", + "parameters": [ + {"name": "nexioPort", "value": 2098, "type": "java.lang.Integer"}, + {"name": "nexioUserName", "value": "administrator", "type": "java.lang.String"}, + {"name": "nexioPassword", "value": "system", "type": "java.lang.String"}, + {"name": "archiveFtp", "value": "ftp://10.10.1.100/ARCHIVE", "type": "java.lang.String"}, + {"name": "archiveUserName", "value": "mediacube", "type": "java.lang.String"}, + {"name": "archivePassword", "value": "Broadca5T", "type": "java.lang.String"}, + {"name": "daysBeforeNow", "value": 1, "type": "java.lang.Integer"}, + {"name": "nexioKillDateDays", "value": 21, "type": "java.lang.Integer"}, + {"name": "nexioAgency", "value": "ARCHIVED", "type": "java.lang.String"} + ] + }, + { + "active": true, + "executeimmediate": false, + "name" : "NEXIO visszarögzített anyagok másolása az ISILON/ARCHIVE mappába", + "template": "copyforarchive-nexio-recordings.xml", + "cronexpression": "0 0 12 * * ? *", + "parameters": [ + {"name": "nexioPort", "value": 2098, "type": "java.lang.Integer"}, + {"name": "nexioUserName", "value": "administrator", "type": "java.lang.String"}, + {"name": "nexioPassword", "value": "system", "type": "java.lang.String"}, + {"name": "archiveFtp", "value": "ftp://10.10.1.100/ARCHIVE", "type": "java.lang.String"}, + {"name": "archiveUserName", "value": "mediacube", "type": "java.lang.String"}, + {"name": "archivePassword", "value": "Broadca5T", "type": "java.lang.String"}, + {"name": "filterAgencies", "value": "schedule-rec", "type": "java.lang.String"}, + {"name": "limit", "value": 30, "type": "java.lang.Integer"}, + {"name": "nexioKillDateDays", "value": 7, "type": "java.lang.Integer"}, + {"name": "nexioAgency", "value": "HIRADO_ARCHIVED", "type": "java.lang.String"} + ] + }, + { + "active": true, + "executeimmediate": false, + "name" : "Híranyag statisztika importálása", + "template": "import-statistics.xml", + "cronexpression": "0 0 6 * * ?", + "parameters": [ + {"name": "daysBeforeNow", "value": 1, "type": "java.lang.Integer"} + ] + }, + { + "active": false, + "executeimmediate": true, + "name" : "TRAFFIC anyagok visszatöltése", + "template": "retrieve-traffic-missing-materials.xml", + "cronexpression": "0 0 0/1 1/1 * ? *", + "parameters": [ + {"name": "dbUrl", "value": "jdbc:sqlserver://10.10.1.45\\sql16;databaseName=PA_Echo;", "type": "java.lang.String"}, + {"name": "userName", "value": "MAM", "type": "java.lang.String"}, + {"name": "password", "value": "Echotv.hu", "type": "java.lang.String"}, + {"name": "lookupDays", "value": 7, "type": "java.lang.Integer"}, + {"name": "targetPath", "value": "/mnt/ISILON/PLAYOUT/IceGateway/Input", "type": "java.lang.String"}, + {"name": "killDateDays", "value": 10, "type": "java.lang.Integer"} + ] + }, + { + "active": true, + "executeimmediate": false, + "name" : "Lejárt NEXIO anyagok törlése", + "template": "delete-nexio-materials.xml", + "cronexpression": "0 0 6 * * ?", + "parameters": [ + {"name": "port", "value": 2098, "type": "java.lang.Integer"}, + {"name": "userName", "value": "administrator", "type": "java.lang.String"}, + {"name": "password", "value": "system", "type": "java.lang.String"}, + {"name": "filterAgencies", "value": "HIRADO_23_00,HIRADO_ARCHIVED,HIRADO_CLN", "type": "java.lang.String"}, + {"name": "gracePeriodDays", "value": 1, "type": "java.lang.Integer"}, + {"name": "notificationOnly", "value": false, "type": "java.lang.Boolean"} + ] + }, + { + "name" : "SYS: MORPHEUS 'missing materials' importálása", + "template": "import-morpheus-missing-materials.xml", + "parameters": [ + {"name": "csvFilePath", "value": "/mnt/MORPHEUS", "type": "java.lang.String"}, + {"name": "processedFolder", "value": "DONE", "type": "java.lang.String"}, + {"name": "targetPath", "value": "/mnt/ISILON/ARCHIVE_RESTORE", "type": "java.lang.String"} + ] + }, + { + "name" : "SYS: batch-retrieve-ondemand", + "template": "batch-retrieve-ondemand.xml" + }, + { + "name" : "SYS: retrieve-ondemand", + "template": "retrieve-ondemand.xml", + "parameters": [ + {"name": "globalRetrievePath", "value": "file://isilon.intra.echotv.hu", "type": "java.lang.String"}, + {"name": "localRetrievePath", "value": "/mnt/ISILON", "type": "java.lang.String"}, + {"name": "materialOutputFolder", "value": "PLAYOUT_NLE", "type": "java.lang.String"}, + {"name": "promoOutputFolder", "value": "PROMO_NLE", "type": "java.lang.String"}, + {"name": "advertisementOutputFolder", "value": "REKLAM_NLE", "type": "java.lang.String"}, + {"name": "octopusOutputFolder", "value": "OCTOPUS", "type": "java.lang.String"}, + {"name": "genericOutputFolder", "value": "ARCHIVE_RESTORE", "type": "java.lang.String"}, + {"name": "onlineOutputFolder", "value": "ONLINE", "type": "java.lang.String"}, + {"name": "killDateDays", "value": 7, "type": "java.lang.Integer"}, + {"name": "nexioAgency", "value": "ARCHIVE_RESTORE", "type": "java.lang.String"}, + {"name": "nexioPort", "value": 2098, "type": "java.lang.Integer"}, + {"name": "nexioUserName", "value": "administrator", "type": "java.lang.String"}, + {"name": "nexioPassword", "value": "system", "type": "java.lang.String"} + ] + }, + { + "name" : "sys: MORPHEUS 'missing materials' importálása", + "template": "sys-import-morpheus-missing-materials.xml", + "parameters": [ + {"name": "csvFilePath", "value": "/mnt/MORPHEUS", "type": "java.lang.String"}, + {"name": "processedFolder", "value": "DONE", "type": "java.lang.String"}, + {"name": "targetPath", "value": "/mnt/ISILON/PLAYOUT/Video", "type": "java.lang.String"} + ] + }, + { + "name" : "sys: MORPHEUS 'missing material' visszatöltése", + "template": "sys-retrieve-missing-material.xml", + "parameters": [ + {"name": "targetPath", "value": "/mnt/ISILON/PLAYOUT/Video", "type": "java.lang.String"}, + {"name": "globalRetrievePath", "value": "\\\\10.10.1.100\\BRAAVOS\\PLAYOUT\\Video", "type": "java.lang.String"}, + {"name": "morpheusDeviceID", "value": "ISILON", "type": "java.lang.String"}, + {"name": "dbUrl", "value": "jdbc:sqlserver://10.10.1.45;databaseName=PA_Echo;", "type": "java.lang.String"}, + {"name": "userName", "value": "MAM", "type": "java.lang.String"}, + {"name": "password", "value": "Echotv.hu", "type": "java.lang.String"}, + {"name": "targetMetadataPath", "value": "/mnt/ISILON/PLAYOUT/MorpheusGateway/Input", "type": "java.lang.String"} + ] + } +]} + + + diff --git a/server/-product/production/HIRTV/jobs/steps/ArchiveListBuilderStep.java b/server/-product/production/HIRTV/jobs/steps/ArchiveListBuilderStep.java new file mode 100644 index 00000000..a86d563f --- /dev/null +++ b/server/-product/production/HIRTV/jobs/steps/ArchiveListBuilderStep.java @@ -0,0 +1,170 @@ +package user.jobengine.server.steps; + +import java.io.File; +import java.io.IOException; +import java.nio.file.DirectoryStream; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.LinkedList; +import java.util.List; + +import org.apache.commons.lang.StringUtils; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.apache.logging.log4j.Marker; + +import user.jobengine.server.IJobEngine; +import user.jobengine.server.IJobRuntime; + +/** + * Az archivalhato mediak listazasa MediaFileWrapper objektumokban. A listazott + * media allomanyokat megjeloli .catched signal allomannyal, hogy a legkozelebbi + * listazas figyelmen kivul hagyja. + * + * @author robi + */ +public class ArchiveListBuilderStep extends JobStep { + private static final Logger logger = LogManager.getLogger(); + // private static final String UTF8 = "utf-8"; + private static final String STATUSFOLDER = ".STATUS"; + private static final String JSONEXT = ".json"; + private static final String CATCHEDEXT = ".catched"; + + public static final String ITEM_TITLE = "itemTitle"; + public static final String ITEM_HOUSEID = "itemHouseId"; + public static final String ITEM_DESCRIPTION = "itemDescription"; + public static final String MEDIA_HOUSEID = "mediaHouseId"; + public static final String MEDIA_TITLE = "mediaTitle"; + public static final String MEDIA_DESCRIPTION = "mediaDescription"; + public static final String MEDIA_TYPE = "mediaType"; + private static final String DURATION = "duration"; + private static final String EXISTING_MEDIAID = "existingMediaId"; + private static final String TAGS = "tags"; + + private Marker marker; + + private ArchiveItem createArchiveItem(Path jsonFilePath, Path mediaFilePath, Path catchedFilePath) { + ArchiveItem result = null; + try { + result = ArchiveItem.fromFile(jsonFilePath); + result.setMediaFile(mediaFilePath.toString()); + result.setCatchedFile(catchedFilePath.toString()); + } catch (Exception e) { + logger.catching(e); + } + + return result; + } + + private void createCatchedFile(Path catchedFilePath) { + try { + Files.createFile(catchedFilePath); + // Files.write(catchedFilePath, CATCHED.getBytes(UTF8), + // StandardOpenOption.CREATE); + } catch (Exception e) { + logger.catching(e); + } + } + + @StepEntry + public Object[] execute(String sourcePath, int limit, IJobEngine jobEngine, IJobRuntime jobRuntime) { + marker = jobRuntime.getSessionMarker(); + List archiveList = new LinkedList(); + DirectoryStream directoryStream = null; + try { + DirectoryStream stream = Files.newDirectoryStream(Paths.get(sourcePath)); + for (Path p : stream) { + processPathItem(p, archiveList); + } + } catch (Exception e) { + logger.catching(e); + logger.error(marker, "Az '{}' mappa elérése sikertelen. A rendszer hibaüzenete: {}", e.getMessage()); + } finally { + if (directoryStream != null) { + try { + directoryStream.close(); + } catch (IOException e) { + } + } + } + + if (limit > 0 && archiveList.size() > limit) { + archiveList = archiveList.subList(0, limit); + logger.info(marker, "A folyamat alkalmazza a beállított {} limitet.", limit); + } + + if (archiveList.size() == 0) + logger.info(marker, "Nincs archiválandó anyag."); + else + logger.info(marker, "Az archiváló folyamat {} új anyagot érzékelt.", + archiveList == null ? 0 : archiveList.size()); + + for (ArchiveItem archiveItem : archiveList) { + createCatchedFile(Paths.get(archiveItem.getCatchedFile())); + } + + return new Object[] { archiveList }; + } + + private boolean processPathItem(Path mediaFilePath, final List archiveList) { + File mediaFile = mediaFilePath.toFile(); + + // if (mediaFile.length() > 0) + // return false; + + if (mediaFile.isDirectory()) { + return false; + } + + Path dotStorePath = Paths.get(mediaFilePath.getParent().toString(), STATUSFOLDER); + Path catchedFilePath = Paths.get(dotStorePath.toString(), mediaFile.getName() + CATCHEDEXT); + File catchedFile = catchedFilePath.toFile(); + if (catchedFile.exists()) { + logger.warn("{} file is already catched.", mediaFile.getName()); + return false; + } + + Path jsonFilePath = Paths.get(dotStorePath.toString(), mediaFile.getName() + JSONEXT); + File jsonFile = jsonFilePath.toFile(); + if (!jsonFile.exists()) { + logger.warn("{} has no json metadata.", mediaFile.getName()); + return false; + } + + ArchiveItem archiveItem = createArchiveItem(jsonFilePath, mediaFilePath, catchedFilePath); + + if (archiveItem == null) { + logger.warn("{} has no metadata specified.", mediaFile.getName()); + return false; + } + + if (StringUtils.isBlank(archiveItem.getItemHouseId())) { + logger.warn("{} has no Item HouseID specified in metadata.", mediaFile.getName()); + return false; + } + + if (StringUtils.isBlank(archiveItem.getItemTitle())) { + logger.warn("{} has no Item Title specified in metadata.", mediaFile.getName()); + return false; + } + + if (StringUtils.isBlank(archiveItem.getMediaHouseId())) { + logger.warn("{} has no Media HouseID specified in metadata.", mediaFile.getName()); + return false; + } + + if (StringUtils.isBlank(archiveItem.getMediaTitle())) { + logger.warn("{} has no Media Title specified in metadata.", mediaFile.getName()); + return false; + } + + // A tenyleges archivalast vesszuk elore + if (mediaFile.length() == 0) + archiveList.add(archiveItem); + else + archiveList.add(0, archiveItem); + // createCatchedFile(catchedFilePath); + return true; + } +} diff --git a/server/-product/production/HIRTV/jobs/steps/ArchiveMaterialSubmitStep.java b/server/-product/production/HIRTV/jobs/steps/ArchiveMaterialSubmitStep.java new file mode 100644 index 00000000..9a869abf --- /dev/null +++ b/server/-product/production/HIRTV/jobs/steps/ArchiveMaterialSubmitStep.java @@ -0,0 +1,55 @@ +package user.jobengine.server.steps; + +import java.io.File; +import java.util.List; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.apache.logging.log4j.Marker; + +import user.commons.ListUtils; +import user.jobengine.server.IJobEngine; +import user.jobengine.server.IJobRuntime; + +public class ArchiveMaterialSubmitStep extends JobStep { + private static final Logger logger = LogManager.getLogger(); + private static final String JOBTEMPLATE = "archive-material.xml"; + private static final String KILL_DATE_DAYS = "killDateDays"; + private static final String ARCHIVE = "Archiválás"; + private static final String ARCHIVE_ITEM = "archiveItem"; + private Marker marker; + + @StepEntry + public Object[] execute(List archiveList, int killDateDays, IJobEngine jobEngine, + IJobRuntime jobRuntime) throws Exception { + marker = jobRuntime.getSessionMarker(); + if (archiveList == null || archiveList.size() == 0) + return null; + + if (jobRuntime.forkPrepare()) { + for (int i = 0; i < archiveList.size(); i++) { + ArchiveItem archiveItem = archiveList.get(i); + try { + IJobRuntime runtime = jobEngine.submit(jobRuntime, null, JOBTEMPLATE, ARCHIVE, + ListUtils.asMap(ARCHIVE_ITEM, archiveItem, KILL_DATE_DAYS, killDateDays)); + int progress = (i + 1) * 100 / archiveList.size(); + setProgress(progress); + // TODO kivezetni a submit hibaüzenetet + if (runtime == null) + throw new Exception("Submit returned null runtime"); + } catch (Exception e) { + logger.catching(e); + String fileName = new File(archiveItem.getMediaFile()).getName(); + logger.error(marker, "A(z) '{}' állomány archiválási kísérlete sikertelen. A rendszer üzenete: {}", + fileName, e.getMessage()); + if (!archiveItem.removeCatchedFile()) + logger.error(marker, "A(z) '{}' állomány .catched jelző állománya nem törölhető.", fileName); + throw e; + } + } + } + jobRuntime.forkWaitComplete(); + return null; + } + +} diff --git a/server/-product/production/HIRTV/jobs/steps/BackupFileStep.java b/server/-product/production/HIRTV/jobs/steps/BackupFileStep.java new file mode 100644 index 00000000..396fd24f --- /dev/null +++ b/server/-product/production/HIRTV/jobs/steps/BackupFileStep.java @@ -0,0 +1,122 @@ +package user.jobengine.server.steps; + +import java.io.IOException; +import java.nio.file.DirectoryStream; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.ArrayList; +import java.util.List; + +import org.apache.commons.lang.RandomStringUtils; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.apache.logging.log4j.Marker; +import org.apache.logging.log4j.message.Message; +import org.apache.logging.log4j.message.ParameterizedMessage; + +import user.commons.RemoteFile; +import user.commons.StoreUri; +import user.commons.configuration.SystemConfiguration; +import user.commons.remotestore.RemoteStoreProtocol; +import user.jobengine.db.MediaFile; +import user.jobengine.db.MediaFileDAO; +import user.jobengine.db.Store; +import user.jobengine.server.steps.shared.EscortFiles; + +public class BackupFileStep extends JobStep { + private static final Logger logger = LogManager.getLogger(); + private static boolean RANDOMIZE_ARCHIVES = SystemConfiguration.getInstance().value("tsm.randomize-archives"); + private Marker marker; + public static final String DOT_DONE = ".DONE"; + public static final String DOT_JSON = ".json"; + + MediaFile getMediaFile(String fileName) { + List result = new ArrayList<>(); + MediaFileDAO dao = (MediaFileDAO) getManager().getBaseDAO(MediaFile.class); + getManager().executeQuery("SELECT id FROM MEDIAFILE WHERE relativepath=?", rs -> { + long id = rs.getLong("id"); + result.add((MediaFile) dao.get(id)); + return false; + }, s -> { + s.setString(1, fileName); + }); + + return result.size() > 0 ? result.get(0) : null; + } + + String flag(Object checkObject) { + return checkObject == null ? "-" : "X"; + } + + @StepEntry + public Object[] execute() throws Exception { + marker = getSessionMarker(); + + Path sourceFolder = Paths.get("/mnt/ISILON/ARCHIVE/.CONFLICT"); + Path donePath = Paths.get(sourceFolder.toString(), DOT_DONE); + Store tsmStore = getManager().getSystemStore(false); + StoreUri tsmUri = tsmStore.getSourceStoreUri(RemoteStoreProtocol.TSM); + + try (DirectoryStream stream = Files.newDirectoryStream(sourceFolder)) { + for (Path source : stream) { + if (Files.isDirectory(source)) { + continue; + } + + String fileName = source.getFileName().toString(); + if (fileName.toLowerCase().endsWith(".mxf") || fileName.toLowerCase().endsWith(".mp4")) { + + MediaFile mf = getMediaFile(fileName); + RemoteFile tsmFile = tsmUri.getRemoteFile(fileName); + logger.info(marker, "[{}{}] {}", flag(mf), flag(tsmFile), fileName); + if (getJobRuntime().isWaitingCancel()) + break; + + if (tsmFile == null) { + backup(source, tsmUri); + tsmFile = tsmUri.getRemoteFile(fileName); + } + + if (mf != null && tsmFile != null) { + moveToDone(donePath, source, fileName); + } + } + } + } finally { + tsmUri.cleanUp(); + } + + return null; + } + + private void moveToDone(Path donePath, Path source, String fileName) throws IOException { + EscortFiles.ensureUNCFolder(donePath); + Files.move(source, Paths.get(donePath.toString(), fileName)); + Path json = Paths.get(source.getParent().toString(), fileName + DOT_JSON); + Files.move(json, Paths.get(donePath.toString(), fileName + DOT_JSON)); + } + + private void backup(Path source, StoreUri targetUri) throws Exception { + String sourceFileName = source.getFileName().toString(); + try { + getJobRuntime().setDescription(sourceFileName); + + StoreUri sourceUri = getManager().createStoreUri(RemoteStoreProtocol.LOCAL, source.getParent().toString()); + String targetFileName; + if (RANDOMIZE_ARCHIVES) { + targetFileName = String.format("%s-%s", RandomStringUtils.randomAlphanumeric(8), sourceFileName); + } else + targetFileName = sourceFileName; + + RemoteFile remoteFile = sourceUri.transferFrom(targetUri, sourceFileName, targetFileName); + + } catch (Exception e) { + logger.catching(e); + Message m = new ParameterizedMessage("Az '{}' állomány archiválása sikertelen. A rendszer hibaüzenete: {}", sourceFileName, e.getMessage()); + logger.error(marker, m); + throw new Exception(m.getFormattedMessage()); + } + } + +} diff --git a/server/-product/production/HIRTV/jobs/steps/BatchRetrieveForkStep.java b/server/-product/production/HIRTV/jobs/steps/BatchRetrieveForkStep.java new file mode 100644 index 00000000..a531a2ea --- /dev/null +++ b/server/-product/production/HIRTV/jobs/steps/BatchRetrieveForkStep.java @@ -0,0 +1,78 @@ +package user.jobengine.server.steps; + +import java.util.List; +import java.util.Map; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + +import user.commons.log4j2.marker.MediaCubeFinishMarker; +import user.commons.log4j2.marker.MediaCubeMarker; +import user.jobengine.db.ArchivedMedia; +import user.jobengine.server.IJobEngine; +import user.jobengine.server.IJobRuntime; +import user.jobengine.server.scheduler.ScheduledJob; + +public class BatchRetrieveForkStep extends JobStep { + private static final String TARGET_PATH_TYPE = "targetPathType"; + private static final Logger logger = LogManager.getLogger(); + private static final String CHILD_TEMPLATE = "retrieve-ondemand.xml"; + private static final String ARCHIVEDMEDIA = "archivedMedia"; + private static final String RECIPIENT = "successRecipient"; + private static final String HOUSEID = "houseId"; + private MediaCubeMarker marker; + + @StepEntry + public Object[] execute(List basket, String houseId, String recipient, String targetPathType, + IJobEngine jobEngine, IJobRuntime jobRuntime) throws Exception { + marker = (MediaCubeMarker) jobRuntime.getSessionMarker(); + + // session szinten csak a finishMarker cimzettje az erdekes, es ezt a cimet + // pluszban hasznalja a konfigban megadott cimmel + // a finishMarker orokli a cim bellitast a sessionMarkertol + marker.setTo(recipient); + + ((MediaCubeMarker) jobRuntime.getFinishMarker()).setTo(recipient); + + if (basket == null || basket.size() == 0) + return null; + setProgress(10); + + MediaCubeMarker mailMarker = new MediaCubeMarker(recipient); + mailMarker.setSessionName("Archívum viszatöltés"); + mailMarker.setSessionID(houseId); + logger.info(mailMarker, "A visszatöltések elindultak az alábbi állományokra:"); + + if (jobRuntime.forkPrepare()) { + for (ArchivedMedia archivedMedia : basket) { + logger.info(mailMarker, archivedMedia.getMedia().getMediaFilesName()); + submit(archivedMedia, recipient, houseId, targetPathType, jobEngine, jobRuntime); + } + } + setProgress(50); + logger.info(new MediaCubeFinishMarker(mailMarker), "A visszatöltések végeztével megerősítő üzenetet küldünk."); + jobRuntime.forkWaitComplete(); + setProgress(100); + return null; + } + + public void submit(ArchivedMedia archivedMedia, String recipient, String houseId, String targetPathType, + IJobEngine jobEngine, IJobRuntime jobRuntime) throws Exception { + try { + ScheduledJob scheduledJob = jobEngine.getScheduledJob(CHILD_TEMPLATE); + Map parameters = scheduledJob.getJobParameters(); + parameters.put(ARCHIVEDMEDIA, archivedMedia); + parameters.put(HOUSEID, houseId); + parameters.put(RECIPIENT, recipient); + parameters.put(TARGET_PATH_TYPE, targetPathType); + IJobRuntime child = jobEngine.submit(jobRuntime, null, CHILD_TEMPLATE, + String.format("Visszatöltés %s részére", recipient), parameters); + ((MediaCubeMarker) child.getSessionMarker()).setTo(recipient); + } catch (Exception e) { + logger.catching(e); + logger.error(marker, "Hiba a kötegelt visszatöltésben. A rendszer üzenete: {}", e.getMessage()); + } + + } + +} diff --git a/server/-product/production/HIRTV/jobs/steps/CancelableStep.java b/server/-product/production/HIRTV/jobs/steps/CancelableStep.java new file mode 100644 index 00000000..b830e155 --- /dev/null +++ b/server/-product/production/HIRTV/jobs/steps/CancelableStep.java @@ -0,0 +1,22 @@ +package user.jobengine.server.steps; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + +public class CancelableStep extends JobStep { + private static final Logger logger = LogManager.getLogger(); + + @StepEntry + public Object[] execute(int param) throws Exception { + logger.info(getMarker(), "Executing with param {}", param); + for (int i = 0; i < 10; i++) { + if (getJobRuntime().isWaitingCancel()) + break; + Thread.sleep(10); + int progress = (i + 1) * 100 / 10; + setProgress(progress); + } + return null; + } + +} diff --git a/server/-product/production/HIRTV/jobs/steps/CheckLOWRESIntegrity.java b/server/-product/production/HIRTV/jobs/steps/CheckLOWRESIntegrity.java new file mode 100644 index 00000000..373545eb --- /dev/null +++ b/server/-product/production/HIRTV/jobs/steps/CheckLOWRESIntegrity.java @@ -0,0 +1,66 @@ +package user.jobengine.server.steps; + +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.ArrayList; +import java.util.List; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.apache.logging.log4j.Marker; + +import user.jobengine.db.IItemManager; +import user.jobengine.db.IResultSetConsumer; +import user.jobengine.db.Media; +import user.jobengine.db.MediaFile; +import user.jobengine.server.IJobEngine; +import user.jobengine.server.IJobRuntime; + +public class CheckLOWRESIntegrity extends JobStep { + private static final Logger logger = LogManager.getLogger(); + private IItemManager manager; + private Marker marker; + private String webPath; + + private void checkIntegrity(MediaFile mediaFile) { + Path path = Paths.get(webPath, mediaFile.getRelativePath()); + if (!path.toFile().exists()) { + logger.warn(marker, "{} {}", mediaFile.getId(), path); + } + } + + @StepEntry + public Object[] execute(String webPath, IJobEngine jobEngine, IJobRuntime jobRuntime) throws Exception { + this.webPath = webPath; + marker = jobRuntime.getSessionMarker(); + manager = jobEngine.getItemManager(); + List mediaIdentities = getTranscodedMediaIdentities(); + int allCount = mediaIdentities.size(); + int currentCount = 0; + for (long id : mediaIdentities) { + Media media = manager.getMedia(id); + List mediaFiles = media.getMediaFiles(); + for (MediaFile mediaFile : mediaFiles) { + if (mediaFile.getStoreId() == 21) + checkIntegrity(mediaFile); + } + currentCount++; + jobRuntime.incrementProgress(currentCount * 100 / allCount); + } + return null; + } + + public List getTranscodedMediaIdentities() { + final List result = new ArrayList<>(); + String query = "select mediaid from vw_mediafiles where mediafilecount = 2"; + + IResultSetConsumer consumer = rs -> { + result.add(rs.getLong("mediaId")); + return true; + }; + + manager.executeQuery(query, consumer, null); + return result; + } + +} diff --git a/server/-product/production/HIRTV/jobs/steps/CheckMORPHEUSMissingMaterialsStep.java b/server/-product/production/HIRTV/jobs/steps/CheckMORPHEUSMissingMaterialsStep.java new file mode 100644 index 00000000..d28a1080 --- /dev/null +++ b/server/-product/production/HIRTV/jobs/steps/CheckMORPHEUSMissingMaterialsStep.java @@ -0,0 +1,155 @@ +package user.jobengine.server.steps; + +import java.io.File; +import java.io.IOException; +import java.nio.charset.Charset; +import java.nio.file.DirectoryStream; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.Arrays; +import java.util.List; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + +import user.commons.IEntityBase; +import user.commons.ListUtils; +import user.jobengine.db.IItemManager; +import user.jobengine.db.MediaFile; +import user.jobengine.db.MediaFileDAO; +import user.jobengine.server.IJobEngine; +import user.jobengine.server.IJobRuntime; + +public class CheckMORPHEUSMissingMaterialsStep extends JobStep { + private static final Logger logger = LogManager.getLogger(); + private static final String KILLDATEDAYS = "killDateDays"; + private static final String SUCCESSRECIPIENT = "successRecipient"; + private static final String TARGETNAMEPATTERN = "targetNamePattern"; + private static final String MATERIAL_ID = "Material ID"; + private static final String RETRIEVE_MATERIAL = "Adásanyag visszatöltés"; + private static final String TARGETPATH = "targetPath"; + private static final String MEDIACUBEMEDIA = "mediaCubeMedia"; + private static final String JOBTEMPLATE = "retrieve-material.xml"; + private static final String CSV_EXT = ".csv"; + private static final String MXF_EXT = ".MXF"; + private String targetPath; + private MediaFileDAO dao; + private IJobEngine jobEngine; + private int killDateDays; + + @StepEntry + public Object[] execute(String sourcePath, String targetPath, int killDateDays, IJobEngine jobEngine, + IJobRuntime jobRuntime) throws Exception { + this.killDateDays = killDateDays; + setAndCheck(sourcePath, targetPath, jobEngine); + DirectoryStream directoryStream = null; + try { + Files.newDirectoryStream(Paths.get(sourcePath)).forEach(p -> processPathItem(p)); + } catch (Exception e) { + logger.error("", e); + } finally { + if (directoryStream != null) { + try { + directoryStream.close(); + } catch (IOException e) { + } + } + } + return null; + } + + private void processMediaId(String mediaId) throws Exception { + List medias = dao.getByHouseId(mediaId + ".MXF"); + if (medias == null || medias.size() == 0) { + logger.error(getMarker(), "A(z) {} anyag nem található az archívumban.", mediaId); + return; + } + + if (medias.size() > 1) { + logger.error(getMarker(), "A(z) {} anyagból egynél több található az archívumban.", mediaId); + return; + } + + jobEngine.submit(JOBTEMPLATE, RETRIEVE_MATERIAL, ListUtils.asMap(MEDIACUBEMEDIA, medias.get(0), TARGETPATH, targetPath, TARGETNAMEPATTERN, + mediaId + MXF_EXT, SUCCESSRECIPIENT, null, KILLDATEDAYS, killDateDays)); + } + + private void processMissingMaterialCSV(Path csvFilePath) throws Exception { + List lines = Files.readAllLines(csvFilePath, Charset.forName("UTF-8")); + if (lines == null | lines.size() == 0) { + return; + } + + int mediaIdPosition = -1; + for (int i = 0; i < lines.size(); i++) { + String line = lines.get(i); + if (line == null) + continue; + String[] data = line.split(","); + if (i == 0) { + // Channel,Time to Air,Duration,Material ID,Title,Device ID,Reason, + List dataList = Arrays.asList(data); + mediaIdPosition = dataList.indexOf(MATERIAL_ID); + if (mediaIdPosition < 0) { + logger.error(getMarker(), "A(z) {} MORPHEUS állományban nem található a 'Material ID' mező.", + csvFilePath.toFile().getName()); + break; + } + } else { + processMediaId(data[mediaIdPosition]); + } + } + } + + private void processPathItem(Path csvFilePath) { + File csvFile = csvFilePath.toFile(); + + if (csvFile.isDirectory() || !csvFile.getName().toLowerCase().endsWith(CSV_EXT.toLowerCase())) { + return; + } + + try { + processMissingMaterialCSV(csvFilePath); + } catch (Exception e) { + logger.catching(e); + logger.error(getMarker(), + "A(z) {} MORPHEUS állomány feldolgozásakor hiba történt. A rendszer hibaüzenete: {}.", + csvFile.getName(), e.getMessage()); + } + + // TODO ne törölje, move + // if (!csvFile.delete()) + // logger.error(getMarker(), "A {} MORPHEUS állomány nem törölhető.", + // csvFile.getName()); + } + + private void setAndCheck(String sourcePath, String targetPath, IJobEngine jobEngine) { + if (jobEngine == null) { + logger.error(getMarker(), "Az folyamatkezelő réteg nem elérhető."); + throw new NullPointerException("Internal error, missing JobEngine reference."); + } + this.jobEngine = jobEngine; + + IItemManager manager = jobEngine.getItemManager(); + if (manager == null) { + logger.error(getMarker(), "Az adatbáziskezelő réteg nem elérhető."); + throw new NullPointerException("Internal error, missing ItemManager reference."); + } + dao = (MediaFileDAO) manager.getBaseDAO(MediaFile.class); + if (dao == null) { + logger.error(getMarker(), "Az adatbáziskezelő réteg MediaFile kezelöje nem elérhető."); + throw new NullPointerException("Internal error, missing MediaFile DAO reference."); + } + if (sourcePath == null) { + logger.error(getMarker(), "A folyamat 'sourcePath' bemeneti paramétere üres."); + throw new NullPointerException("System is not configured properly, 'sourcePath' input parameter missing."); + } + if (targetPath == null) { + logger.error(getMarker(), "A folyamat 'targetPath' bemeneti paramétere üres."); + throw new NullPointerException("System is not configured properly, 'targetPath' input parameter missing."); + } + this.targetPath = targetPath; + } + +} diff --git a/server/-product/production/HIRTV/jobs/steps/CheckTRAFFICMissingMaterialsStep.java b/server/-product/production/HIRTV/jobs/steps/CheckTRAFFICMissingMaterialsStep.java new file mode 100644 index 00000000..87fca790 --- /dev/null +++ b/server/-product/production/HIRTV/jobs/steps/CheckTRAFFICMissingMaterialsStep.java @@ -0,0 +1,97 @@ +package user.jobengine.server.steps; + +import java.sql.Connection; +import java.sql.DriverManager; +import java.sql.PreparedStatement; +import java.sql.ResultSet; +import java.util.List; + +import org.apache.commons.lang.StringUtils; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + +import user.commons.IEntityBase; +import user.commons.ListUtils; +import user.jobengine.db.IItemManager; +import user.jobengine.db.MediaFile; +import user.jobengine.db.MediaFileDAO; +import user.jobengine.server.IJobEngine; +import user.jobengine.server.IJobRuntime; + +public class CheckTrafficMissingMaterialsStep extends JobStep { + private static final String KILLDATEDAYS = "killDateDays"; + private static final Logger logger = LogManager.getLogger(); + private static final String TARGET_NAME_PATTERN = "targetNamePattern"; + private static final String TARGET_PATH = "targetPath"; + private static final String MEDIA_CUBE_MEDIA = "mediaCubeMedia"; + private static final String SQLSERVER_JDBC_SQL_SERVER_DRIVER = "com.microsoft.sqlserver.jdbc.SQLServerDriver"; + private static final String SQL = "{call dbo.clIFsp_EC_MAM(1001, 32, Null, Null, ?)}"; + private static final String JOBTEMPLATE = "retrieve-material.xml"; + private static final String SUCCESSRECIPIENT = "successRecipient"; + private int killDateDays; + + @StepEntry + public Object[] execute(String dbUrl, String userName, String password, int lookupDays, String targetPath, + int killDateDays, IJobEngine jobEngine, IJobRuntime jobRuntime) throws Exception { + + this.killDateDays = killDateDays; + try (Connection con = getConnection(dbUrl, userName, password); + PreparedStatement stmt = con.prepareStatement(SQL)) { + stmt.setInt(1, lookupDays); + try (ResultSet rs = stmt.executeQuery()) { + while (rs.next()) { + String trafficId = rs.getString(5); + if (StringUtils.isBlank(trafficId)) + continue; + + try { + processTrafficId(trafficId, targetPath, jobEngine); + } catch (Exception e) { + logger.error(getMarker(), + "Hiba lépett fel az {} anyag visszatöltéskor. A rendszer hibaüzenete: {}", trafficId, + e.getMessage()); + } + + } + } + } catch (Exception e) { + logger.catching(e); + throw e; + } finally { + } + return null; + } + + private Connection getConnection(String dbUrl, String userName, String password) throws Exception { + Connection result = null; + try { + Class.forName(SQLSERVER_JDBC_SQL_SERVER_DRIVER); + result = DriverManager.getConnection(dbUrl, userName, password); + } catch (Exception e) { + logger.error(getMarker(), "Hiba lépett fel a folyamat indításakor. A rendszer hibaüzenete: {}", + e.getMessage()); + throw e; + } + return result; + } + + private void processTrafficId(String trafficId, String targetPath, IJobEngine jobEngine) throws Exception { + IItemManager manager = jobEngine.getItemManager(); + MediaFileDAO dao = (MediaFileDAO) manager.getBaseDAO(MediaFile.class); + List medias = dao.getByHouseId(trafficId); + if (medias == null || medias.size() == 0) { + logger.error(getMarker(), "Az {} anyag nem található az archívumban.", trafficId); + return; + } + + if (medias.size() > 1) { + logger.error(getMarker(), "Az {} anyagból egynél több található az archívumban.", trafficId); + return; + } + + String title = "Traffic adásanyag visszatöltés: " + trafficId; + jobEngine.submit(JOBTEMPLATE, title, ListUtils.asMap(MEDIA_CUBE_MEDIA, medias.get(0), TARGET_PATH, targetPath, TARGET_NAME_PATTERN, "%s", + SUCCESSRECIPIENT, null, KILLDATEDAYS, killDateDays)); + } + +} diff --git a/server/-product/production/HIRTV/jobs/steps/CleanupMountedLocationStep.java b/server/-product/production/HIRTV/jobs/steps/CleanupMountedLocationStep.java new file mode 100644 index 00000000..d674c61e --- /dev/null +++ b/server/-product/production/HIRTV/jobs/steps/CleanupMountedLocationStep.java @@ -0,0 +1,296 @@ +package user.jobengine.server.steps; + +import java.io.File; +import java.io.IOException; +import java.nio.file.DirectoryStream; +import java.nio.file.FileVisitResult; +import java.nio.file.FileVisitor; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.nio.file.SimpleFileVisitor; +import java.nio.file.attribute.BasicFileAttributes; +import java.text.ParseException; +import java.text.SimpleDateFormat; +import java.util.ArrayList; +import java.util.Collections; +import java.util.Date; +import java.util.List; + +import org.apache.commons.io.FileUtils; +import org.apache.commons.lang.StringUtils; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.apache.logging.log4j.Marker; + +import user.jobengine.server.steps.shared.ItemManagerExtensions; + +public class CleanupMountedLocationStep extends JobStep implements FileVisitor { + + private static final Logger logger = LogManager.getLogger(); + private static final String PROJECTFOLDER = "PROJECT"; + private static final String DATEFORMAT = "yyyyMMdd"; + private static final String DOT = "."; + private static final String STATUSFOLDER = ".STATUS"; + private static final String EWC2EXT = ".ewc2"; + private static final String XMPEXT = ".xmp"; + private static final String CATCHEDEXT = ".catched"; + private static final String KILLDATEEXT = ".killdate"; + private static final String JSONEXT = ".json"; + + private static boolean isEmpty(final Path directory) throws IOException { + try (DirectoryStream dirStream = Files.newDirectoryStream(directory)) { + final int[] count = new int[] { 0 }; + final int[] specialCount = new int[] { 0 }; + dirStream.forEach(p -> { + count[0]++; + // if + // (p.getFileName().toString().toLowerCase().equals(PROJECTFOLDER.toLowerCase())) + // specialCount[0]++; + if (p.getFileName().toString().toLowerCase().equals(STATUSFOLDER.toLowerCase())) + specialCount[0]++; + + }); + if (specialCount[0] == count[0]) + return true; + } + return false; + } + + private Marker marker; + + final int[] allCount = new int[] { 0 }; + final int[] currentCount = new int[] { 0 }; + + private Path sourcePath; + private SimpleDateFormat dateFormat; + private boolean skipArchiveCheck; + + private Date checkExpiration(List killDateFiles) { + Date killDate = null; + for (Path killDateFile : killDateFiles) { + Date currentKillDate = getKillDate(killDateFile); + if (currentKillDate == null) + continue; + if ((killDate != null && currentKillDate.after(killDate)) || killDate == null) + killDate = currentKillDate; + } + return new Date().after(killDate) ? killDate : null; + } + + @StepEntry + public Object[] execute(String sourceFolder, boolean skipArchiveCheck) throws Exception { + this.skipArchiveCheck = skipArchiveCheck; + marker = getSessionMarker(); + sourcePath = Paths.get(sourceFolder); + DirectoryStream directoryStream = null; + if (StringUtils.isBlank(sourcePath.toString())) { + logger.error(marker, "A folyamat 'sourcePath' bemeneti paramétere üres."); + throw new NullPointerException( + "System is not configured properly, 'sourceFolder' input parameter missing."); + } + + if (!sourcePath.toFile().exists() || !sourcePath.toFile().isDirectory()) { + logger.error(marker, "A(z) {} mappa nem létezik.", sourceFolder); + throw new NullPointerException(String.format("Directory %s does not exist.", sourceFolder)); + } + + try { + setProgress(1); + dateFormat = new SimpleDateFormat(DATEFORMAT); + + Files.walkFileTree(sourcePath, new SimpleFileVisitor() { + @Override + public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) throws IOException { + allCount[0]++; + return super.visitFile(file, attrs); + } + }); + Files.walkFileTree(sourcePath, this); + } catch (Exception e) { + logger.catching(e); + logger.error(marker, "Hiba a '{}' mappa feldolgozásában. A rendszer hibaüzenete: {}", sourcePath, + e.getMessage()); + throw e; + } finally { + if (directoryStream != null) { + try { + directoryStream.close(); + } catch (IOException e) { + } + } + } + return null; + } + + private Date getKillDate(Path killDateFile) { + String fileName = killDateFile.getFileName().toString(); + int end = fileName.lastIndexOf(DOT); + if (end < 1) + return null; + int start = fileName.lastIndexOf(DOT, end - 1); + if (start < 0) + return null; + String strKillDate = fileName.substring(start + 1, end); + Date result = null; + if (StringUtils.isNumeric(strKillDate)) { + try { + result = dateFormat.parse(strKillDate); + } catch (ParseException e) { + logger.error(marker, + "A {} fájl 'killdate' állománya hibás formátumú, a {} karaktersorozat nem konvertálható dátummá.", + fileName, strKillDate); + return null; + } + } else + logger.error(marker, "A {} fájl 'killdate' állománya hibás formátumú, az dátum helyett ez áll: '{}'.", + fileName, strKillDate); + return result; + } + + private List getKillDateFiles(Path filePath) { + String killDateFilePattern = String.format("%s.*%s", filePath.getFileName().toString(), KILLDATEEXT); + List result = new ArrayList<>(); + Path statusPath = null; + try { + statusPath = Paths.get(filePath.getParent().toString(), STATUSFOLDER); + } catch (Exception e) { + logger.catching(e); + return null; + } + File statusPathFile = statusPath.toFile(); + if (statusPathFile.exists() && statusPathFile.isDirectory()) { + try (DirectoryStream stream = Files.newDirectoryStream(statusPath, killDateFilePattern)) { + stream.forEach(p -> result.add(p)); + } catch (Exception e) { + logger.catching(e); + } + } + Collections.sort(result); + return result; + } + + @Override + public FileVisitResult postVisitDirectory(Path dir, IOException exc) throws IOException { + if (!dir.equals(sourcePath) && isEmpty(dir)) { + if (!removeExistingSpecialDirectory(dir, PROJECTFOLDER)) + return FileVisitResult.CONTINUE; + if (!removeExistingSpecialDirectory(dir, STATUSFOLDER)) + return FileVisitResult.CONTINUE; + if (removeFile(dir)) + logger.info(marker, "A {} üres mappa törlése sikeres.", dir); + + } + return FileVisitResult.CONTINUE; + } + + @Override + public FileVisitResult preVisitDirectory(Path dir, BasicFileAttributes attrs) throws IOException { + + // A .-al kezdodo mappakat kihagyjuk + if (dir.getFileName().toString().startsWith(".")) + return FileVisitResult.SKIP_SUBTREE; + + return FileVisitResult.CONTINUE; + } + + private void processPathItem(Path filePath) { + currentCount[0]++; + + if (filePath.getFileName().toString().startsWith(".")) + return; + + int progress = currentCount[0] * 100 / allCount[0]; + setProgress(progress); + + logger.info("Checking {}", filePath); + List killDateFiles = getKillDateFiles(filePath); + if (killDateFiles == null || killDateFiles.size() == 0) { + logger.warn(marker, "A {} fájlhoz nem található 'killdate' állomány.", filePath); + return; + } + + if (killDateFiles.size() != 1) + logger.warn(marker, + "A {} fájlhoz több 'killdate' állomány található, a legújabb dátum határozza meg a törlés időpontját.", + filePath); + + Date killDate = checkExpiration(killDateFiles); + if (killDate == null) + return; + + if (!skipArchiveCheck && filePath.toFile().length() > 0) { + if (!ItemManagerExtensions.isArchived(getManager(), filePath)) { + logger.error(marker, "A(z) {} anyag törlésre van kijelölve, de nem található az archívumban.", + filePath); + return; + } + } + + if (removeFiles(filePath, killDateFiles)) + logger.info(marker, + "A {} fájl és kapcsolódó állományai a {} killdate bejegyzés alapján sikeresen törlődtek.", + filePath.getFileName(), dateFormat.format(killDate)); + else + logger.warn(marker, + "A {} fájl és kapcsolódó állományai a {} killdate bejegyzés alapján csak részlegesen vagy egyáltalán nem törlődtek.", + filePath.getFileName(), dateFormat.format(killDate)); + } + + private boolean removeExistingSpecialDirectory(Path dir, String folderName) throws IOException { + File projectPath = Paths.get(dir.toString(), folderName).toFile(); + if (projectPath.exists() && projectPath.isDirectory()) { + FileUtils.deleteDirectory(projectPath); + if (projectPath.exists()) { + logger.warn(marker, "A {} alatti {} mappa törlése nem sikerült.", dir, folderName); + return false; + } + } + return true; + } + + private boolean removeFile(Path filePath) { + boolean result = false; + try { + // logger.error("REMOVE {}", filePath); + File file = filePath.toFile(); + if (file.exists()) + result = file.delete(); + } catch (Exception e) { + logger.error(marker, "A {} fájl nem törölhető. A rendszer hibaüzenete: {}", filePath, e.getMessage()); + } + return result; + } + + private boolean removeFiles(Path filePath, List killDateFiles) { + if (!removeFile(filePath)) + return false; + + removeFile(Paths.get(filePath.toString() + EWC2EXT)); + removeFile(Paths.get(filePath.toString() + XMPEXT)); + removeFile(Paths.get(filePath.getParent().toString(), STATUSFOLDER, + filePath.getFileName().toString() + CATCHEDEXT)); + removeFile( + Paths.get(filePath.getParent().toString(), STATUSFOLDER, filePath.getFileName().toString() + JSONEXT)); + + boolean result = true; + for (Path killDateFile : killDateFiles) { + if (!removeFile(killDateFile)) + result = false; + } + + return result; + } + + @Override + public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) throws IOException { + processPathItem(file); + return FileVisitResult.CONTINUE; + } + + @Override + public FileVisitResult visitFileFailed(Path file, IOException e) throws IOException { + logger.error(marker, "A {} fájl nem érhető el. A rendszer hibaüzenete: {}", file.toString(), e.getMessage()); + return FileVisitResult.CONTINUE; + } +} diff --git a/server/-product/production/HIRTV/jobs/steps/CopyForArchiveNEXIOMaterialsStep.java b/server/-product/production/HIRTV/jobs/steps/CopyForArchiveNEXIOMaterialsStep.java new file mode 100644 index 00000000..0637fb33 --- /dev/null +++ b/server/-product/production/HIRTV/jobs/steps/CopyForArchiveNEXIOMaterialsStep.java @@ -0,0 +1,561 @@ +package user.jobengine.server.steps; + +import java.io.OutputStream; +import java.net.InetAddress; +import java.net.URI; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Calendar; +import java.util.Date; +import java.util.List; + +import org.apache.commons.lang.StringUtils; +import org.apache.commons.net.ftp.FTP; +import org.apache.commons.net.ftp.FTPClient; +import org.apache.commons.net.ftp.FTPReply; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.apache.logging.log4j.Marker; + +import com.ibm.nosql.json.api.BasicDBObject; +import com.ibm.nosql.json.api.DB; +import com.ibm.nosql.json.api.DBCollection; +import com.ibm.nosql.json.api.DBCursor; +import com.ibm.nosql.json.api.DBObject; + +import user.commons.CalendarUtils; +import user.commons.ListUtils; +import user.commons.StoreUri; +import user.commons.configuration.SystemConfiguration; +import user.commons.nosql.NoSQLUtils; +import user.commons.octopus.IOctopusAPI; +import user.commons.octopus.OctopusAPI; +import user.commons.remotestore.FtpDirectoryLister; +import user.commons.remotestore.RemoteStoreProtocol; +import user.jobengine.db.IItemManager; +import user.jobengine.server.IJobEngine; +import user.jobengine.server.IJobRuntime; +import user.jobengine.server.steps.shared.EscortFiles; +import user.jobengine.server.steps.shared.ItemManagerExtensions; +import user.jobengine.server.steps.shared.MetadataType; +import user.jobengine.server.steps.shared.MetadataTypeDetector; + +public class CopyForArchiveNEXIOMaterialsStep extends JobStep { + private static final String SCHEDULED_FORMAT = "yyyy.MM.dd HH:mm"; + private static final Logger logger = LogManager.getLogger("CopyForArchiveNEXIOMaterialsStep"); + private static final String UTF_8 = "utf-8"; + private static final String JSON_EXT = ".json"; + private static final String XML_EXT = ".xml"; + private static final String DURATION = "duration"; + private static final String MXFEXT = ".MXF"; + private static final String NEXIOCLIPS = SystemConfiguration.getInstance().value("services.nexio.collection-name", + "nexioclips"); + private static final String LONGNAMEID = "longnameid"; + private static final String ARCHIVEDRUNDOWNS = "archivedrundowns"; + private static final String ID = "id"; + private static final String MEDIATYPE = "Hír bejátszó"; + private static String NEXIO_HOST = SystemConfiguration.getInstance().value("services.nexio.host"); + private OctopusAPI octopusAPI; + private IItemManager manager; + + private DB db; + private FTPClient sourceFtp; + private FTPClient targetFtp; + private StoreUri sourceUri; + private StoreUri targetUri; + private int nexioKillDateDays; + private String nexioAgency; + private Marker systemMarker; + private List transferredFileNames = null; + private boolean demo = false; + + private int check(int value, String name) { + if (value == 0) { + logger.error(systemMarker, "A folyamat '{}' bemeneti paramétere 0.", name); + throw new NullPointerException( + String.format("System is not configured properly, missing '%s' input parameter.", name)); + } + return value; + } + + private String check(String value, String name) { + if (value == null) { + logger.error(systemMarker, "A folyamat '{}' bemeneti paramétere üres.", name); + throw new NullPointerException( + String.format("System is not configured properly, missing '%s' input parameter.", name)); + } + return value; + } + + private void copy(RundownArchive rundownArchive) throws Exception { + for (StoryArchive storyArchive : rundownArchive.getStoryArchives()) { + for (FileArchive fileArchive : storyArchive.getFileArchives()) { + try { + copyFile(fileArchive, rundownArchive, storyArchive); + } catch (Exception e) { + logger.error(systemMarker, "A '{}' clip archiválása sikertelen. A rendszer üzenete: {}", + fileArchive.getFileName(), e.getMessage()); + throw e; + } + } + } + } + + private void copyFile(FileArchive fileArchive, RundownArchive rundownArchive, StoryArchive storyArchive) + throws Exception { + String origFileName = fileArchive.getFileName(); + String fileName = String.format("%s-%s", origFileName, rundownArchive.getItemHouseId()); + String videoFileName = fileName + MXFEXT; + + if (transferredFileNames == null) + transferredFileNames = new ArrayList<>(); + + // A mar letezo mozikat nem archivaljuk le ujra, csak a metaadatot + long existingMediaId = 0; + boolean transferDone = false; + if (transferredFileNames.contains(origFileName)) { + logger.info(systemMarker, + "A '{}' file archiválásra felkészítése egy másik story kapcsán már megtörtént, ezért csak metaadat archiválás szükséges.", + origFileName); + if (!demo) + transferDone = transferChunk(videoFileName); + } else { + existingMediaId = ItemManagerExtensions.getExistingRundownMedia(manager, origFileName); + if (existingMediaId == 0) { + transferredFileNames.add(origFileName); + if (!demo) + transferDone = transferFile(origFileName + MXFEXT, videoFileName); + logger.info(systemMarker, "A '{}' file archiválásra felkészítése sikeres volt.", origFileName); + } else { + + logger.info(systemMarker, + "A '{}' file archiválása már megtörtént, ezért csak metaadat archiválás szükséges.", + origFileName); + if (!demo) + transferDone = transferChunk(videoFileName); + } + } + + if (!demo && transferDone) { + BasicDBObject metadata = createMetadata(rundownArchive, storyArchive, fileArchive, existingMediaId); + transferMetadata(videoFileName, metadata); + createSourceKillDateFile(rundownArchive, origFileName); + } + } + + private BasicDBObject createMetadata(RundownArchive rundownArchive, StoryArchive storyArchive, + FileArchive fileArchive, long existingMediaId) { + BasicDBObject result = new BasicDBObject(); + result.put("itemHouseId", rundownArchive.getItemHouseId()); + result.put("itemTitle", rundownArchive.getItemTitle()); + result.put("itemDescription", rundownArchive.getItemDesc()); + result.put("userName", "mediacube"); + result.put("mediaHouseId", storyArchive.getMediaHouseId()); + result.put("mediaTitle", storyArchive.getMediaTitle()); + result.put("mediaDescription", storyArchive.getMediaDesc()); + result.put("mediaType", MEDIATYPE); + result.put("duration", fileArchive.getDuration()); + result.put("existingMediaId", existingMediaId); + return result; + } + + private void createSourceKillDateFile(RundownArchive rundownArchive, String fileName) throws Exception { + logger.info("Create killdate/agency for {}", fileName); + OutputStream outStream = null; + try { + sourceFtp = ((FtpDirectoryLister) sourceUri.getLister()).connect(); + Calendar killDate = CalendarUtils.createCalendar(rundownArchive.getScheduleDate()); + killDate.add(Calendar.DAY_OF_YEAR, nexioKillDateDays); + byte[] killDateFile = EscortFiles.createNEXIOKillDateFile(fileName, killDate.getTime(), null, nexioAgency); + outStream = sourceFtp.storeFileStream(fileName + XML_EXT); + if (outStream == null) { + throw new NullPointerException( + "Can not open: " + fileName + XML_EXT + " Reply:" + sourceFtp.getReplyString()); + } + outStream.write(killDateFile); + outStream.flush(); + } catch (Exception e) { + logger.catching(e); + throw e; + } finally { + if (outStream != null) + outStream.close(); + sourceUri.cleanUp(); + } + } + + @StepEntry + public Object[] execute(int nexioPort, String nexioUserName, String nexioPassword, String archiveFtp, + String archiveUserName, String archivePassword, int daysBeforeNow, int nexioKillDateDays, + String nexioAgency, IJobEngine jobEngine, IJobRuntime jobRuntime) throws Exception { + systemMarker = getMarker(); + logger.debug(systemMarker, "debug test"); + setAndCheck(nexioPort, nexioUserName, nexioPassword, archiveFtp, archiveUserName, archivePassword, + nexioKillDateDays, nexioAgency, jobEngine); + octopusAPI = new OctopusAPI(); + Calendar scheduledDate = Calendar.getInstance(); + scheduledDate.add(Calendar.DAY_OF_YEAR, -1 * daysBeforeNow); + List rundowns = octopusAPI.getRundowns(scheduledDate.getTime()); + if (rundowns == null) { + logger.warn(systemMarker, "Nem található adástükör a {} napra.", CalendarUtils.toDateString(scheduledDate)); + return null; + } + + processRundowns(rundowns); + return null; + } + + // private String getVersionedFileName(String fileName, String extension) throws + // Exception { + // String result = fileName; + // try { + // targetFtp = ((FtpDirectoryLister) targetUri.getLister()).connect(); + // FTPFile[] listFiles = targetFtp.listFiles(); + // List fileNames = new ArrayList<>(); + // for (FTPFile ftpFile : listFiles) { + // fileNames.add(ftpFile.getName()); + // } + // while (fileNames.contains(result + extension)) { + // + // } + // + // } catch (Exception e) { + // logger.catching(e); + // throw e; + // } finally { + // targetUri.cleanUp(); + // } + // return result; + // } + + private FileArchive processMosObject(BasicDBObject rundown, BasicDBObject story, BasicDBObject mosObject) + throws Exception { + String mosID = mosObject.getString(IOctopusAPI.OBJ_ID); + if (MetadataTypeDetector.GuessMetadataType(mosID) != MetadataType.OctopusPlaceholder) { + logger.info(systemMarker, "Skipping MOS object {}", mosID); + return null; + } + DBCollection clips = db.getCollection(NEXIOCLIPS); + BasicDBObject clip = (BasicDBObject) clips.findOne(new BasicDBObject(LONGNAMEID, mosID)); + if (clip == null) { + logger.info(systemMarker, "File {} does NOT exist", mosID); + return null; + // throw new Exception(String.format("File does NOT exist %s", mosID)); + } else { + logger.debug(systemMarker, "File {} exists", mosID); + } + long duration = NoSQLUtils.asLong(clip, DURATION); + + if (duration == 0) { + logger.info(systemMarker, "File {} exists with 0 frame length", mosID); + return null; + } + if (duration == 1) { + logger.info(systemMarker, "File {} exists with 1 frame length", mosID); + return null; + } + return new FileArchive(mosID, duration); + } + + private RundownArchive processRundow(DBObject r) throws Exception { + BasicDBObject rundown = (BasicDBObject) r; + long rundownID = rundown.getLong(ID); + if (!demo) + logger.info(systemMarker, "Processing rundown {} {}", rundownID, rundown.getString(IOctopusAPI.NAME)); + List stories = octopusAPI.getRundownFullStories(rundownID); + if (stories == null) { + logger.info(systemMarker, "Rundown {} {} is empty", rundownID, rundown.getString(IOctopusAPI.NAME)); + return null; + } + RundownArchive result = new RundownArchive(); + + long id = NoSQLUtils.asLong(rundown, IOctopusAPI.ID); + if (id == 0) + return null; + String name = NoSQLUtils.asString(NoSQLUtils.asDBObject(rundown, IOctopusAPI.RUNDOWN_TYPE), IOctopusAPI.NAME); + if (StringUtils.isBlank(name)) { + logger.info(systemMarker, "Rundown {} {} type is empty", rundownID, rundown.getString(IOctopusAPI.NAME)); + return null; + } + String channel = NoSQLUtils.asString(NoSQLUtils.asDBObject(rundown, IOctopusAPI.CHANNEL), IOctopusAPI.NAME); + Date scheduledStart = rundown.getDate(IOctopusAPI.SCHEDULED_START); + if (scheduledStart == null) { + logger.info(systemMarker, "Rundown {} {} schedule is empty", rundownID, + rundown.getString(IOctopusAPI.NAME)); + return null; + } + String start = CalendarUtils.toString(CalendarUtils.createCalendar(scheduledStart), SCHEDULED_FORMAT); + result.setScheduleDate(scheduledStart); + result.setItemHouseId(String.valueOf(id)); + result.setItemTitle(String.format("%s %s %s", start, name, channel)); + + logger.info(systemMarker, "Processing stories in rundown {} {}", rundownID, + rundown.getString(IOctopusAPI.NAME)); + for (DBObject s : stories) { + StoryArchive storyArchive = processStory(rundown, s); + if (storyArchive == null) + continue; + result.addStoryArchive(storyArchive); + } + return result; + } + + private void processRundowns(List rundowns) { + + // TODO kiveni publikálás előtt + // db.getCollection(v).drop(); + List archivedRundowns = queryArchivedRundowns(); + + logger.info(systemMarker, "Found {} rundowns to archive", archivedRundowns.size()); + int index = 1; + + for (DBObject r : rundowns) { + BasicDBObject rundown = (BasicDBObject) r; + int progress = index * 100 / rundowns.size(); + setProgress(progress); + long rundownID = NoSQLUtils.asLong(rundown, IOctopusAPI.ID); + String rundownName = NoSQLUtils.asString(rundown, IOctopusAPI.NAME); + try { + BasicDBObject currentRundownID = new BasicDBObject(IOctopusAPI.ID, rundownID); + if (!demo && archivedRundowns != null && archivedRundowns.contains(currentRundownID)) { + logger.info(systemMarker, "Skipping archived rundown {} {}", rundownID, rundownName); + continue; + } + + RundownArchive rundownArchive = processRundow(r); + if (rundownArchive == null || rundownArchive.isEmpty()) { + if (!demo) + logger.info(systemMarker, "Skipping rundown {} {}", rundownID, rundownName); + continue; + } + + if (!demo) + logger.info(systemMarker, "Saving rundown {} {}", rundownID, rundownName); + copy(rundownArchive); + if (!demo) + logger.info(systemMarker, "A '{}' tükör {}db bejátszójának archiválása sikeres volt", + rundownArchive.getItemTitle(), rundownArchive.getStoryArchives().size()); + + if (!demo) + db.getCollection(ARCHIVEDRUNDOWNS).save(currentRundownID); + } catch (Exception e) { + logger.catching(e); + logger.error(systemMarker, String.format( + "A %s %s tükör archiválása nem lehetséges, mert a annak ellenőrzése hibát jelzett. A rendszer üzenete: %s", + rundownID, rundownName, e.getMessage())); + } + index++; + } + + } + + private StoryArchive processStory(BasicDBObject rundown, DBObject s) throws Exception { + BasicDBObject story = (BasicDBObject) s; + String parentStoryID = story.getString(IOctopusAPI.PARENT_STORY_ID); + String storyID = story.getString(IOctopusAPI.ID); + + if (StringUtils.isBlank(parentStoryID)) { + logger.warn(systemMarker, "Story parentStoryID is null: {}", story.toPrettyString(null)); + return null; + } else { + if ("475048225".equals(storyID)) + logger.info(systemMarker, "Processing story {}", story.toPrettyString(null)); + } + List mosObjects = NoSQLUtils.asList(story, IOctopusAPI.MOS_OBJECTS); + if (mosObjects == null) { + logger.debug(systemMarker, "No MOS in story {}", storyID); + return null; + } + StoryArchive storyArchive = null; + for (BasicDBObject mosObject : mosObjects) { + FileArchive fileArchive = processMosObject(rundown, story, mosObject); + if (fileArchive == null) + continue; + if (storyArchive == null) { + storyArchive = new StoryArchive(); + storyArchive.setMediaHouseId(parentStoryID); + storyArchive.setMediaTitle(story.getString(IOctopusAPI.NAME)); + storyArchive.setMediaDesc(story.getString(IOctopusAPI.SCRIPT_CONTENT)); + + } + storyArchive.addFileArchive(fileArchive); + } + return storyArchive; + + } + + private List queryArchivedRundowns() { + List result = null; + DBCollection collection = db.getCollection(ARCHIVEDRUNDOWNS); + DBCursor find = collection.find(new BasicDBObject(), + new BasicDBObject(IOctopusAPI._ID, 0).append(IOctopusAPI.ID, 1)); + if (find.hasNext()) + result = ListUtils.cast(find.toArray()); + return result; + } + + private void setAndCheck(int nexioPort, String nexioUserName, String nexioPassword, String archiveFtp, + String archiveUserName, String archivePassword, int nexioKillDateDays, String nexioAgency, + IJobEngine jobEngine) throws Exception { + db = NoSQLUtils.getNoSQLDB(); + if (db == null) { + logger.error(systemMarker, "Az NoSQL adatkezelő réteg nem elérhető."); + throw new NullPointerException("Internal error, missing NoSQL DB reference."); + } + + if (jobEngine == null) { + logger.error(systemMarker, "Az folyamatkezelő réteg nem elérhető."); + throw new NullPointerException("Internal error, missing JobEngine reference."); + } + manager = jobEngine.getItemManager(); + if (manager == null) { + logger.error(systemMarker, "Az adatbáziskezelő réteg nem elérhető."); + throw new NullPointerException("Internal error, missing ItemManager reference."); + } + + if (StringUtils.isBlank(NEXIO_HOST)) { + logger.error(systemMarker, "A 'nexio.host' rendszer paraméter nem található."); + throw new NullPointerException( + "System is not configured properly, 'jobengine.selenio.address' startup parameter missing."); + } + check(nexioPort, "nexioPort"); + check(nexioUserName, "nexioUserName"); + check(nexioPassword, "nexioPassword"); + + check(nexioKillDateDays, "nexioKillDateDays"); + this.nexioKillDateDays = nexioKillDateDays; + check(nexioAgency, "nexioAgency"); + this.nexioAgency = nexioAgency; + + sourceUri = manager.createStoreUri(RemoteStoreProtocol.FTP, NEXIO_HOST); + sourceUri.setPortNumber(nexioPort); + sourceUri.setUserName(nexioUserName); + sourceUri.setPassword(nexioPassword); + if (sourceUri == null) { + logger.error(systemMarker, "A forrás nem elérhető."); + throw new NullPointerException("Internal error, missing 'sourceUri'."); + } + + check(archiveFtp, "archiveFtp"); + check(archiveUserName, "archiveUserName"); + check(archivePassword, "archivePassword"); + + targetUri = manager.createStoreUri(new URI(archiveFtp)); + targetUri.setUserName(archiveUserName); + targetUri.setPassword(archivePassword); + if (targetUri == null) { + logger.error(systemMarker, "A cél nem elérhető."); + throw new NullPointerException("Internal error, missing 'targetUri'."); + } + + } + + private boolean transferChunk(String fileName) throws Exception { + logger.info("Transfer chunk {}", fileName); + OutputStream outStream = null; + try { + targetFtp = ((FtpDirectoryLister) targetUri.getLister()).connect(); + + // ha ugyan abban a tukorben szerepel meg 1x a mozi, akkor a chunk felulirja a + // tenyleges file-t, es az sosem archivalodik + String[] names = targetFtp.listNames(); + if (names != null) { + if (Arrays.asList(names).contains(fileName)) + return false; + } + + outStream = targetFtp.storeFileStream(fileName); + if (outStream == null) { + throw new NullPointerException("Can not open: " + fileName + " Reply:" + targetFtp.getReplyString()); + } + } catch (Exception e) { + logger.catching(e); + throw e; + } finally { + if (outStream != null) + outStream.close(); + targetUri.cleanUp(); + } + return true; + } + + private boolean transferFile(String sourceFileName, String targetFileName) throws Exception { + boolean result = true; + int reply = 0; + logger.info("Transfer clip {}", sourceFileName); + try { + sourceFtp = ((FtpDirectoryLister) sourceUri.getLister()).connect(); + targetFtp = ((FtpDirectoryLister) targetUri.getLister()).connect(); + if (!targetFtp.enterRemotePassiveMode()) + throw new Exception("!PASV"); + + reply = sourceFtp.port(InetAddress.getByName(targetFtp.getPassiveHost()), targetFtp.getPassivePort()); + if (!FTPReply.isPositiveCompletion(reply)) + throw new Exception("PORT parancs válasza: " + sourceFtp.getReplyString()); + + if (!sourceFtp.setFileType(FTP.BINARY_FILE_TYPE)) + throw new Exception("!SOURCE TYPE"); + + reply = sourceFtp.retr(sourceFileName); + if (!FTPReply.isPositivePreliminary(reply)) + throw new Exception("RETR parancs válasza: " + sourceFtp.getReplyString()); + + if (!targetFtp.setFileType(FTP.BINARY_FILE_TYPE)) + throw new Exception("!TARGET TYPE"); + + reply = targetFtp.stor(targetFileName); + if (!FTPReply.isPositivePreliminary(reply)) + throw new Exception("STOR parancs válasza: " + sourceFtp.getReplyString()); + + while (true) { + reply = sourceFtp.stat(); + if (!FTPReply.isPositiveCompletion(reply)) + throw new Exception("STAT parancs válasza: " + sourceFtp.getReplyString()); + + logger.info("Status: {}", sourceFtp.getReplyString()); + if (reply == 226) { + break; + } + Thread.sleep(1000); + } + } catch (Exception e) { + logger.catching(e); + result = false; + } finally { + sourceUri.cleanUp(); + targetUri.cleanUp(); + } + + return result; + } + + private void transferMetadata(String fileName, BasicDBObject metadata) throws Exception { + logger.info("Transfer metadata {}", fileName); + OutputStream outStream = null; + try { + targetFtp = ((FtpDirectoryLister) targetUri.getLister()).connect(); + if (!targetFtp.changeWorkingDirectory(EscortFiles.STATUSFOLDER)) { + targetFtp.makeDirectory(EscortFiles.STATUSFOLDER); + if (!targetFtp.changeWorkingDirectory(EscortFiles.STATUSFOLDER)) + throw new Exception("!STATUSFOLDER"); + } + + outStream = targetFtp.storeFileStream(fileName + JSON_EXT); + if (outStream == null) { + throw new NullPointerException( + "Can not open: " + fileName + JSON_EXT + " Reply:" + targetFtp.getReplyString()); + } + outStream.write(metadata.toString().getBytes(UTF_8)); + outStream.flush(); + // targetFtp.changeToParentDirectory(); + } catch (Exception e) { + logger.catching(e); + throw e; + } finally { + if (outStream != null) + outStream.close(); + targetUri.cleanUp(); + } + } + +} diff --git a/server/-product/production/HIRTV/jobs/steps/CopyForArchiveNEXIORecordingsStep.java b/server/-product/production/HIRTV/jobs/steps/CopyForArchiveNEXIORecordingsStep.java new file mode 100644 index 00000000..941b65c6 --- /dev/null +++ b/server/-product/production/HIRTV/jobs/steps/CopyForArchiveNEXIORecordingsStep.java @@ -0,0 +1,455 @@ +package user.jobengine.server.steps; + +import java.io.OutputStream; +import java.net.InetAddress; +import java.net.URI; +import java.text.ParseException; +import java.text.SimpleDateFormat; +import java.util.Calendar; +import java.util.Date; +import java.util.List; +import java.util.TimeZone; + +import org.apache.commons.lang.StringUtils; +import org.apache.commons.net.ftp.FTP; +import org.apache.commons.net.ftp.FTPClient; +import org.apache.commons.net.ftp.FTPReply; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.apache.logging.log4j.Marker; + +import com.ibm.nosql.json.api.BasicDBList; +import com.ibm.nosql.json.api.BasicDBObject; +import com.ibm.nosql.json.api.DB; +import com.ibm.nosql.json.api.DBCollection; +import com.ibm.nosql.json.api.DBCursor; +import com.ibm.nosql.json.api.DBObject; +import com.ibm.nosql.json.api.QueryBuilder; + +import user.commons.CalendarUtils; +import user.commons.ListUtils; +import user.commons.StoreUri; +import user.commons.configuration.SystemConfiguration; +import user.commons.nexio.NexioDispatcher; +import user.commons.nosql.NoSQLUtils; +import user.commons.octopus.IOctopusAPI; +import user.commons.octopus.OctopusAPI; +import user.commons.remotestore.FtpDirectoryLister; +import user.commons.remotestore.RemoteStoreProtocol; +import user.jobengine.db.IItemManager; +import user.jobengine.server.IJobEngine; +import user.jobengine.server.IJobRuntime; +import user.jobengine.server.steps.shared.EscortFiles; + +public class CopyForArchiveNEXIORecordingsStep extends JobStep { + private static final String MEDIATYPE = "Visszarögzített"; + private static final String SCHEDULED_FORMAT = "yyyy.MM.dd HH:mm"; + private static final String STARTTIME_FORMAT = "HHmm"; + private static final String RUNDOWNDATE_FORMAT = "yyyyMMdd"; + private static final Logger logger = LogManager.getLogger(); + private static final String UTF_8 = "utf-8"; + private static final String JSON_EXT = ".json"; + private static final String XML_EXT = ".xml"; + private static final String DURATION = "duration"; + private static final String MXFEXT = ".MXF"; + private static final String NEXIOCLIPS = NexioDispatcher.CLIP_COLLECTION_NAME; + private static final String LONGNAMEID = "longnameid"; + private static final String EXTAGENCY = "extagency"; + private static final String RECORDDATE = "recorddate"; + private static final SimpleDateFormat startTimeformat = new SimpleDateFormat(STARTTIME_FORMAT); + private static final SimpleDateFormat rundownDateformat = new SimpleDateFormat(RUNDOWNDATE_FORMAT); + private static String NEXIO_HOST = SystemConfiguration.getInstance().value("services.nexio.host"); + + private OctopusAPI octopusAPI; + private IItemManager manager; + + private DB db; + private FTPClient sourceFtp; + private FTPClient targetFtp; + private StoreUri sourceUri; + private StoreUri targetUri; + private int nexioKillDateDays; + private String nexioAgency; + private Object[] filterAgencies; + private Marker systemMarker; + + private int check(int value, String name) { + if (value == 0) { + logger.error(systemMarker, "A folyamat '{}' bemeneti paramétere 0.", name); + throw new NullPointerException(String.format("System is not configured properly, missing '%s' input parameter.", name)); + } + return value; + } + + private String check(String value, String name) { + if (StringUtils.isBlank(value)) { + logger.error(systemMarker, "A folyamat '{}' bemeneti paramétere üres.", name); + throw new NullPointerException(String.format("System is not configured properly, missing '%s' input parameter.", name)); + } + return value; + } + + private void copy(RundownArchive rundownArchive) { + for (StoryArchive storyArchive : rundownArchive.getStoryArchives()) { + for (FileArchive fileArchive : storyArchive.getFileArchives()) { + try { + copyFile(fileArchive, rundownArchive, storyArchive); + logger.info(systemMarker, "Sikeres fájl archiválás a '{}' tükörhöz: '{}'.", rundownArchive.getItemTitle(), fileArchive.getFileName()); + } catch (Exception e) { + logger.error(systemMarker, "Az '{}' fájl archiválása sikertelen. A rendszer üzenete: {}", fileArchive.getFileName(), e.getMessage()); + } + } + } + } + + private void copyFile(FileArchive fileArchive, RundownArchive rundownArchive, StoryArchive storyArchive) throws Exception { + String sourceFileName = fileArchive.getFileName(); + String targetFileName = getTargetFileName(rundownArchive, sourceFileName); + transferFile(sourceFileName, targetFileName); + BasicDBObject metadata = createMetadata(rundownArchive, storyArchive, fileArchive); + transferMetadata(targetFileName, metadata); + createSourceKillDateFile(rundownArchive, sourceFileName); + } + + private BasicDBObject createMetadata(RundownArchive rundownArchive, StoryArchive storyArchive, FileArchive fileArchive) { + BasicDBObject result = new BasicDBObject(); + result.put("itemHouseId", rundownArchive.getItemHouseId()); + result.put("itemTitle", rundownArchive.getItemTitle()); + result.put("itemDescription", rundownArchive.getItemDesc()); + result.put("userName", "mediacube"); + + result.put("mediaHouseId", storyArchive.getMediaHouseId()); + result.put("mediaTitle", storyArchive.getMediaTitle()); + result.put("mediaDescription", storyArchive.getMediaDesc()); + + result.put("duration", fileArchive.getDuration()); + result.put("mediaType", MEDIATYPE); + return result; + } + + private void createSourceKillDateFile(RundownArchive rundownArchive, String fileName) throws Exception { + logger.info("Create killdate/agency for {}", fileName); + OutputStream outStream = null; + try { + sourceFtp = ((FtpDirectoryLister) sourceUri.getLister()).connect(); + Calendar killDate = CalendarUtils.createCalendar(rundownArchive.getScheduleDate()); + killDate.add(Calendar.DAY_OF_YEAR, nexioKillDateDays); + byte[] killDateFile = EscortFiles.createNEXIOKillDateFile(fileName, killDate.getTime(), null, nexioAgency); + outStream = sourceFtp.storeFileStream(fileName + XML_EXT); + if (outStream == null) { + throw new NullPointerException("Can not open: " + fileName + XML_EXT + " Reply:" + sourceFtp.getReplyString()); + } + outStream.write(killDateFile); + outStream.flush(); + } catch (Exception e) { + logger.catching(e); + throw e; + } finally { + if (outStream != null) + outStream.close(); + sourceUri.cleanUp(); + } + } + + @StepEntry + public Object[] execute(int nexioPort, String nexioUserName, String nexioPassword, String archiveFtp, String archiveUserName, String archivePassword, + String filterAgencies, int limit, int nexioKillDateDays, String targetAgency, IJobEngine jobEngine, IJobRuntime jobRuntime) throws Exception { + systemMarker = jobRuntime.getSessionMarker(); + setAndCheck(nexioPort, nexioUserName, nexioPassword, archiveFtp, archiveUserName, archivePassword, filterAgencies, limit, nexioKillDateDays, + targetAgency, jobEngine); + octopusAPI = new OctopusAPI(); + List clips = queryClips(); + processClips(clips, limit); + return null; + } + + private Date getScheduledStart(String clipName, Date recordDate) { + if (StringUtils.isBlank(clipName)) { + logger.warn(systemMarker, "A fájlnak nincs neve, ezért nem archiválható."); + return null; + } + if (recordDate == null) { + logger.warn(systemMarker, "Az '{}' fájl rögzítésének ideje nem meghatározható, ezért nem archiválható.", clipName); + return null; + } + + Date timePart = null; + try { + String clipNameTime = clipName.split("_")[0]; + timePart = startTimeformat.parse(clipNameTime); + } catch (ParseException e) { + logger.warn(systemMarker, "A '{}' fájl neve nem időbélyeggel kezdődik, ezért nem archiválható.", clipName); + return null; + } + Calendar dateCal = CalendarUtils.createCalendar(recordDate); + dateCal.setTimeZone(TimeZone.getTimeZone("Europe/Budapest")); + Calendar wholeCal = CalendarUtils.createCalendar(CalendarUtils.createCalendar(recordDate), timePart); + wholeCal.setTimeZone(TimeZone.getTimeZone("Europe/Budapest")); + return wholeCal.getTime(); + } + + private String getTargetFileName(RundownArchive rundownArchive, String sourceFileName) { + String date = rundownDateformat.format(rundownArchive.getScheduleDate()); + return String.format("%s-%s%s", date, sourceFileName, MXFEXT); + } + + private RundownArchive processClip(BasicDBObject clip) { + RundownArchive result = null; + String clipName = NoSQLUtils.asString(clip, LONGNAMEID); + Date recordDate = clip.getDate(RECORDDATE); + long duration = NoSQLUtils.asLong(clip, DURATION); + logger.info("Processing {} {}", clipName, recordDate); + Date scheduledStart = getScheduledStart(clipName, recordDate); + if (scheduledStart == null) + return null; + DBObject rundown = octopusAPI.getRundown(scheduledStart); + if (rundown == null) { + logger.error(systemMarker, "A '{}' anyaghoz nem található tükör '{}' kezdéssel, ezért nem archiválható.", clipName, scheduledStart); + return null; + } + + try { + result = processRundow(rundown, clipName, duration); + } catch (Exception e) { + logger.catching(e); + logger.error(systemMarker, "A '{}' anyag metaadatainak transzformálása sikertelen, ezért az nem archiválható. A rendszer hibaüzenete: {}", + e.getMessage()); + return null; + } + /* + if (clipName.startsWith("1900_")) { + String clipNameNext = clipName.replace("1900_", "1908_"); + scheduledStart = getScheduledStart(clipNameNext, recordDate); + rundown = octopusAPI.getRundown(scheduledStart); + if (rundown == null) { + Calendar calendar = CalendarUtils.createCalendar(scheduledStart); + int dow = calendar.get(Calendar.DAY_OF_WEEK); + if (dow == Calendar.SATURDAY || dow == Calendar.SUNDAY) { + logger.info(systemMarker, "A '{}' anyaghoz nem található tükör '{}' kezdéssel, de a hétvégi kivétel miatt archiválható.", clipName, + scheduledStart); + return result; + } else { + logger.error(systemMarker, "A '{}' anyaghoz nem található tükör '{}' kezdéssel, ezért nem archiválható.", clipName, scheduledStart); + return null; + } + } + + RundownArchive item2 = null; + + try { + item2 = processRundow(rundown, clipName, duration); + } catch (Exception e) { + logger.catching(e); + logger.error(systemMarker, "A '{}' anyag metaadatainak transzformálása sikertelen, ezért az nem archiválható. A rendszer hibaüzenete: {}", + e.getMessage()); + return null; + } + + result.setItemTitle(result.getItemTitle() + " + NAPIAKT"); + StoryArchive storyArchive = result.getStoryArchives().get(0); + StoryArchive storyArchive2 = item2.getStoryArchives().get(0); + storyArchive.setMediaDesc(storyArchive.getMediaDesc() + "\r\n\r\n****** NAPIAKT ******\r\n\r\n" + storyArchive2.getMediaDesc()); + } + */ + return result; + } + + private void processClips(List clips, int limit) { + logger.info(systemMarker, "A folyamat {} archiválható anyagot érzékelt.", clips.size()); + int current = 0; + for (BasicDBObject clip : clips) { + RundownArchive rundownArchive = processClip(clip); + if (rundownArchive == null) + continue; + + copy(rundownArchive); + + current++; + if (current == limit) + break; + int progress = current * 100 / limit; + setProgress(progress); + } + } + + private RundownArchive processRundow(DBObject r, String clipName, long duration) throws Exception { + BasicDBObject rundown = (BasicDBObject) r; + long rundownID = rundown.getLong(IOctopusAPI.ID); + logger.info("Processing rundown {} {}", rundownID, rundown.getString(IOctopusAPI.NAME)); + + List stories = octopusAPI.getRundownFullStories(rundownID); + if (stories == null) + return null; + RundownArchive result = new RundownArchive(); + + String name = NoSQLUtils.asString(NoSQLUtils.asDBObject(rundown, IOctopusAPI.RUNDOWN_TYPE), IOctopusAPI.NAME); + if (StringUtils.isBlank(name)) + return null; + String channel = NoSQLUtils.asString(NoSQLUtils.asDBObject(rundown, IOctopusAPI.CHANNEL), IOctopusAPI.NAME); + Date scheduledStart = rundown.getDate(IOctopusAPI.SCHEDULED_START); + if (scheduledStart == null) + return null; + String start = CalendarUtils.toString(CalendarUtils.createCalendar(scheduledStart), SCHEDULED_FORMAT); + result.setScheduleDate(scheduledStart); + result.setItemHouseId(String.valueOf(rundownID)); + result.setItemTitle(String.format("%s %s %s", start, name, channel)); + + StoryArchive storyArchive = new StoryArchive(); + storyArchive.setMediaHouseId(result.getItemHouseId()); + storyArchive.setMediaTitle(clipName); + storyArchive.setMediaDesc(octopusAPI.getRundownContent(stories)); + result.addStoryArchive(storyArchive); + storyArchive.addFileArchive(new FileArchive(clipName, duration)); + return result; + } + + private List queryClips() { + DBCollection collection = db.getCollection(NEXIOCLIPS); + BasicDBList agencies = new BasicDBList(filterAgencies); + + Calendar now = CalendarUtils.createZeroCalendar(); + QueryBuilder dateQueryBuilder = QueryBuilder.start(RECORDDATE).lessThan(now.getTime()); + QueryBuilder queryBuilder = QueryBuilder.start(EXTAGENCY).in(agencies).and(dateQueryBuilder.get()); + DBCursor cursor = collection.find(queryBuilder.get()).sort(RECORDDATE, -1); + if (!cursor.hasNext()) + return null; + return ListUtils.cast(cursor.toArray()); + } + + private void setAndCheck(int nexioPort, String nexioUserName, String nexioPassword, String archiveFtp, String archiveUserName, String archivePassword, + String agencies, int limit, int nexioKillDateDays, String nexioAgency, IJobEngine jobEngine) throws Exception { + db = NoSQLUtils.getNoSQLDB(); + if (db == null) { + logger.error(systemMarker, "Az NoSQL adatkezelő réteg nem elérhető."); + throw new NullPointerException("Internal error, missing NoSQL DB reference."); + } + + if (jobEngine == null) { + logger.error(systemMarker, "Az folyamatkezelő réteg nem elérhető."); + throw new NullPointerException("Internal error, missing JobEngine reference."); + } + manager = jobEngine.getItemManager(); + if (manager == null) { + logger.error(systemMarker, "Az adatbáziskezelő réteg nem elérhető."); + throw new NullPointerException("Internal error, missing ItemManager reference."); + } + + if (StringUtils.isBlank(NEXIO_HOST)) { + logger.error(systemMarker, "A 'nexio.host' rendszer paraméter nem található."); + throw new NullPointerException("System is not configured properly, 'jobengine.selenio.address' startup parameter missing."); + } + check(nexioPort, "nexioPort"); + check(nexioUserName, "nexioUserName"); + check(nexioPassword, "nexioPassword"); + + check(agencies, "filterAgencies"); + filterAgencies = agencies.split(","); + + check(limit, "limit"); + + check(nexioKillDateDays, "nexioKillDateDays"); + this.nexioKillDateDays = nexioKillDateDays; + check(nexioAgency, "nexioAgency"); + this.nexioAgency = nexioAgency; + + sourceUri = manager.createStoreUri(RemoteStoreProtocol.FTP, NEXIO_HOST); + sourceUri.setPortNumber(nexioPort); + sourceUri.setUserName(nexioUserName); + sourceUri.setPassword(nexioPassword); + if (sourceUri == null) { + logger.error(systemMarker, "A forrás nem elérhető."); + throw new NullPointerException("Internal error, missing 'sourceUri'."); + } + + check(archiveFtp, "archiveFtp"); + check(archiveUserName, "archiveUserName"); + check(archivePassword, "archivePassword"); + + targetUri = manager.createStoreUri(new URI(archiveFtp)); + targetUri.setUserName(archiveUserName); + targetUri.setPassword(archivePassword); + if (targetUri == null) { + logger.error(systemMarker, "A cél nem elérhető."); + throw new NullPointerException("Internal error, missing 'targetUri'."); + } + + } + + private void transferFile(String sourceFileName, String targetFileName) throws Exception { + int reply = 0; + logger.info("Transfer clip {}", sourceFileName); + try { + sourceFtp = ((FtpDirectoryLister) sourceUri.getLister()).connect(); + targetFtp = ((FtpDirectoryLister) targetUri.getLister()).connect(); + + if (!targetFtp.enterRemotePassiveMode()) + throw new Exception("!PASV"); + + reply = sourceFtp.port(InetAddress.getByName(targetFtp.getPassiveHost()), targetFtp.getPassivePort()); + if (!FTPReply.isPositiveCompletion(reply)) + throw new Exception("PORT parancs válasza: " + sourceFtp.getReplyString()); + + if (!sourceFtp.setFileType(FTP.BINARY_FILE_TYPE)) + throw new Exception("!SOURCE TYPE"); + + reply = sourceFtp.retr(sourceFileName); + if (!FTPReply.isPositivePreliminary(reply)) + throw new Exception("RETR parancs válasza: " + sourceFtp.getReplyString()); + + if (!targetFtp.setFileType(FTP.BINARY_FILE_TYPE)) + throw new Exception("!TARGET TYPE"); + + reply = targetFtp.stor(targetFileName); + if (!FTPReply.isPositivePreliminary(reply)) + throw new Exception("STOR parancs válasza: " + sourceFtp.getReplyString()); + + while (true) { + reply = sourceFtp.stat(); + if (!FTPReply.isPositiveCompletion(reply)) + throw new Exception("STAT parancs válasza: " + sourceFtp.getReplyString()); + + //logger.info("Status: {}", sourceFtp.getReplyString()); + if (reply == 226) { + break; + } + Thread.sleep(1000); + } + } catch (Exception e) { + logger.catching(e); + throw e; + } finally { + sourceUri.cleanUp(); + targetUri.cleanUp(); + } + + } + + private void transferMetadata(String fileName, BasicDBObject metadata) throws Exception { + logger.info("Transfer metadata {}", fileName); + OutputStream outStream = null; + try { + targetFtp = ((FtpDirectoryLister) targetUri.getLister()).connect(); + if (!targetFtp.changeWorkingDirectory(EscortFiles.STATUSFOLDER)) { + targetFtp.makeDirectory(EscortFiles.STATUSFOLDER); + if (!targetFtp.changeWorkingDirectory(EscortFiles.STATUSFOLDER)) + throw new Exception("!STATUSFOLDER"); + } + + outStream = targetFtp.storeFileStream(fileName + JSON_EXT); + if (outStream == null) { + throw new NullPointerException("Can not open: " + fileName + JSON_EXT + " Reply:" + targetFtp.getReplyString()); + } + outStream.write(metadata.toString().getBytes(UTF_8)); + outStream.flush(); + //targetFtp.changeToParentDirectory(); + } catch (Exception e) { + logger.catching(e); + throw e; + } finally { + if (outStream != null) + outStream.close(); + targetUri.cleanUp(); + } + } + +} diff --git a/server/-product/production/HIRTV/jobs/steps/CreateArchiveItemStep.java b/server/-product/production/HIRTV/jobs/steps/CreateArchiveItemStep.java new file mode 100644 index 00000000..b169edf5 --- /dev/null +++ b/server/-product/production/HIRTV/jobs/steps/CreateArchiveItemStep.java @@ -0,0 +1,45 @@ +package user.jobengine.server.steps; + +import java.nio.file.Paths; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + +import com.ibm.nosql.json.api.BasicDBObject; +import com.ibm.nosql.json.api.DB; +import com.ibm.nosql.json.api.DBCollection; + +import user.commons.nosql.NoSQLUtils; +import user.jobengine.db.Media; +import user.jobengine.db.Store; + +public class CreateArchiveItemStep extends JobStep { + private static final Logger logger = LogManager.getLogger(); + + @StepEntry + public Object[] execute(Media mediaCubeMedia, String localHiresPath) throws Exception { + DB db = NoSQLUtils.getNoSQLDB(); + DBCollection collection = db.getCollection("missing_lowres"); + Store highResStore = getManager().getSystemStore(false); + + ArchiveItem archiveItem = null; + try { + if (mediaCubeMedia.getMediaFilesCount() != 1) + throw new Exception("Expected media count is 1, found " + mediaCubeMedia.getMediaFilesCount()); + if (mediaCubeMedia.getMediaFiles().get(0).getStoreId() != highResStore.getId()) + throw new Exception("Expected media store is a high-res store"); + + String name = mediaCubeMedia.getMediaFileRealName(); + archiveItem = new ArchiveItem(); + archiveItem.setMediaFile(Paths.get(localHiresPath, name).toString()); + collection.save(new BasicDBObject("name", name)); + } catch (Exception e) { + logger.catching(e); + logger.info(getMarker(), e.getMessage()); + throw e; + } finally { + setProgress(100); + } + return new Object[] { archiveItem }; + } +} diff --git a/server/-product/production/HIRTV/jobs/steps/CreateMissingLowresStep.java b/server/-product/production/HIRTV/jobs/steps/CreateMissingLowresStep.java new file mode 100644 index 00000000..519b28a7 --- /dev/null +++ b/server/-product/production/HIRTV/jobs/steps/CreateMissingLowresStep.java @@ -0,0 +1,92 @@ +package user.jobengine.server.steps; + +import java.nio.file.Paths; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + +import com.ibm.nosql.json.api.BasicDBObject; +import com.ibm.nosql.json.api.DB; +import com.ibm.nosql.json.api.DBCollection; + +import user.commons.log4j2.marker.MediaCubeUndoMarker; +import user.commons.nosql.NoSQLUtils; +import user.jobengine.db.IItemManager; +import user.jobengine.db.Media; +import user.jobengine.db.MediaFile; +import user.jobengine.server.IJobEngine; +import user.jobengine.server.IJobRuntime; + +public class CreateMissingLowresStep extends JobStep { + private static final Logger logger = LogManager.getLogger(); + + @StepEntry + public Object[] execute(String localHiresPath, IJobEngine jobEngine, IJobRuntime jobRuntime) throws Exception { + Object[] result = new Object[] { null, null, "%s", null, 0, true }; + + DB db = NoSQLUtils.getNoSQLDB(); + DBCollection collection = db.getCollection("missing_lowres"); + IItemManager manager = jobEngine.getItemManager(); + Media media = getFirstUntranscodedMedia(manager, collection); + + try { + if (media == null) { + logger.info(new MediaCubeUndoMarker(getSessionMarker().getSessionID()), "Nincs feldolgozandó hiány."); + // throw new Exception("Nincs feldolgozandó hiány."); + cancel(); + return null; + } + + String name = media.getMediaFileRealName(); + result[0] = media; + ArchiveItem archiveItem = new ArchiveItem(); + archiveItem.setMediaFile(Paths.get(localHiresPath, name).toString()); + result[1] = archiveItem; + collection.save(new BasicDBObject("name", name)); + logger.info(getSessionMarker(), "Processing mediaID: {}", media.getId()); + + } catch (Exception e) { + logger.catching(e); + logger.error(getSessionMarker(), e.getMessage()); + throw e; + } finally { + setProgress(100); + } + return result; + } + + private Media getFirstUntranscodedMedia(IItemManager manager, DBCollection collection) { + Media[] result = new Media[] { null }; + // MV + String query = "SELECT mediaid FROM VW_MISSING_PROXY_IDS WHERE HOUSEID like 'M%' or HOUSEID like 'P%' or HOUSEID like 'R%' ORDER BY modified DESC"; + + // HTV + // String query = "SELECT mediaid FROM VW_MISSING_PROXY_IDS"; + manager.executeQuery(query, rs -> { + try { + long mediaId = rs.getLong(1); + Media media = manager.getMedia(mediaId); + // a nevgeneralas miatt az eredeti MediaFilesName nem jo, a pontos nev kell + // nekunk + String name = media.getMediaFileRealName(); + logger.info(getSessionMarker(), "Checking {}", name); + long existing = collection.find(new BasicDBObject("name", name)).count(); + if (existing > 0) { + // logger.info(getSessionMarker(), "{} is on missing_lowres list", name); + return true; + } + + // 210617 proxy keszites tiltasa + MediaFile mf = manager.getSystemMediaFile(media); + if (mf.isDisableProxy()) + return true; + + result[0] = media; + } catch (Exception e) { + logger.error(e); + } + return false; + }, null); + return result[0]; + } +} diff --git a/server/-product/production/HIRTV/jobs/steps/DeleteFileStep.java b/server/-product/production/HIRTV/jobs/steps/DeleteFileStep.java new file mode 100644 index 00000000..79734f57 --- /dev/null +++ b/server/-product/production/HIRTV/jobs/steps/DeleteFileStep.java @@ -0,0 +1,36 @@ +package user.jobengine.server.steps; + +import java.io.File; +import java.nio.file.Path; +import java.nio.file.Paths; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.apache.logging.log4j.Marker; + +public class DeleteFileStep extends JobStep { + private static final Logger logger = LogManager.getLogger(); + private Marker marker; + + @StepEntry + public Object[] execute(ArchiveItem archiveItem, boolean isDelete) throws Exception { + marker = getJobRuntime().getSessionMarker(); + if (isDelete) { + Path filePath = Paths.get(archiveItem.getMediaFile()); + File file = filePath.toFile(); + if (file.exists()) { + if (!file.isDirectory()) { + try { + file.delete(); + } catch (Exception e) { + logger.error(marker, "A '{}' fájl nem törölhető. A rendszer üzenete: {}", filePath, e.getMessage()); + } + + } else + logger.warn(marker, "A '{}' elérés egy mappa.", filePath); + } else + logger.warn(marker, "A '{}' fájl nem található.", filePath); + } + return null; + } +} diff --git a/server/-product/production/HIRTV/jobs/steps/DeleteNEXIOMaterialsStep.java b/server/-product/production/HIRTV/jobs/steps/DeleteNEXIOMaterialsStep.java new file mode 100644 index 00000000..d13b3afd --- /dev/null +++ b/server/-product/production/HIRTV/jobs/steps/DeleteNEXIOMaterialsStep.java @@ -0,0 +1,121 @@ +package user.jobengine.server.steps; + +import java.text.SimpleDateFormat; +import java.util.Calendar; +import java.util.List; + +import org.apache.commons.lang.StringUtils; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + +import com.ibm.nosql.json.api.BasicDBList; +import com.ibm.nosql.json.api.BasicDBObject; +import com.ibm.nosql.json.api.DB; +import com.ibm.nosql.json.api.DBCollection; +import com.ibm.nosql.json.api.DBCursor; +import com.ibm.nosql.json.api.QueryBuilder; + +import user.commons.ListUtils; +import user.commons.RemoteFile; +import user.commons.StoreUri; +import user.commons.configuration.SystemConfiguration; +import user.commons.nosql.NoSQLUtils; +import user.commons.remotestore.RemoteStoreProtocol; +import user.jobengine.db.IItemManager; +import user.jobengine.server.IJobEngine; +import user.jobengine.server.IJobRuntime; + +public class DeleteNEXIOMaterialsStep extends JobStep { + private static final Logger logger = LogManager.getLogger(DeleteNEXIOMaterialsStep.class); + //"nexioclips" + private static final String NEXIOCLIPS = SystemConfiguration.getInstance().value("services.nexio.collection-name"); + private static final String KILLDATE = "killdate"; + private static final String RECORDDATE = "recorddate"; + private static final String LONGNAMEID = "longnameid"; + private static final String EXTAGENCY = "extagency"; + private static String NEXIO_HOST = SystemConfiguration.getInstance().value("services.nexio.host"); + private StoreUri sourceUri; + private IJobRuntime jobRuntime; + + private void delete(String name) { + RemoteFile remoteFile = null; + try { + remoteFile = sourceUri.getRemoteFile(name); + if (remoteFile == null) { + logger.warn(jobRuntime.getSessionMarker(), "A '{}' fájl már nem található meg a NEXIO szerveren", name); + logger.warn(getMarker(), "A '{}' fájl már nem található meg a NEXIO szerveren", name); + return; + } + sourceUri.delete(remoteFile); + logger.info(jobRuntime.getSessionMarker(), "A '{}' fájl törlése sikeres volt.", name); + logger.info(getMarker(), "A '{}' fájl törlése sikeres volt.", name); + } catch (Exception e) { + logger.error(jobRuntime.getSessionMarker(), "A '{}' fájl nem törölhető. A rendszer hibaüzenete: {}", name, + e.getMessage()); + logger.error(getMarker(), "A '{}' fájl nem törölhető. A rendszer hibaüzenete: {}", name, e.getMessage()); + } + } + + @StepEntry + public Object[] execute(int port, String userName, String password, String filterAgencies, int gracePeriodDays, + boolean notificationOnly, IJobEngine jobEngine, IJobRuntime jobRuntime) throws Exception { + this.jobRuntime = jobRuntime; + logger.info(getMarker(), "TESZT"); + if (StringUtils.isBlank(NEXIO_HOST)) { + logger.error(jobRuntime.getSessionMarker(), "A 'nexio.host' rendszer paraméter nem található."); + throw new NullPointerException( + "System is not configured properly, 'nexio.host' startup parameter missing."); + } + + if (StringUtils.isBlank(filterAgencies)) { + logger.error(jobRuntime.getSessionMarker(), "A 'nexioAgencies' folyamat paraméter nem található."); + throw new NullPointerException("System is not configured properly, 'nexioAgencies' job parameter missing."); + } + + DB db = NoSQLUtils.getNoSQLDB(); + DBCollection collection = db.getCollection(NEXIOCLIPS); + //https://docs.oracle.com/javase/7/docs/api/java/util/regex/Pattern.html + //http://www.vogella.com/tutorials/JavaRegularExpressions/article.html + // if (StringUtils.isNotBlank(fileNameRegex)) + // queryBuilder.and(QueryBuilder.start(LONGNAMEID).regex(Pattern.compile(fileNameRegex)).get()); + Calendar now = Calendar.getInstance(); + now.add(Calendar.DAY_OF_YEAR, gracePeriodDays * -1); + BasicDBList agencies = new BasicDBList((Object[]) filterAgencies.split(",")); + + QueryBuilder dateQueryBuilder = QueryBuilder.start(KILLDATE).lessThan(now.getTime()); + QueryBuilder queryBuilder = QueryBuilder.start(EXTAGENCY).in(agencies).and(dateQueryBuilder.get()); + DBCursor cursor = collection.find(queryBuilder.get()).sort(KILLDATE, -1); + + if (!cursor.hasNext()) + return null; + + IItemManager manager = jobEngine.getItemManager(); + sourceUri = manager.createStoreUri(RemoteStoreProtocol.FTP, NEXIO_HOST); + sourceUri.setPortNumber(port); + sourceUri.setUserName(userName); + sourceUri.setPassword(password); + + List clips = ListUtils.cast(cursor.toArray()); + SimpleDateFormat dffull = new SimpleDateFormat("yyyy.MM.dd HH:mm:ss"); + logger.info(getMarker(), "{} fájl vizsgálata elindul a {} lista alapján", clips.size(), NEXIOCLIPS); + int i = 0; + for (BasicDBObject clip : clips) { + String name = String.valueOf(clip.get(LONGNAMEID)); + //logger.info(getMarker(), name); + String agency = String.valueOf(clip.get(EXTAGENCY)); + + logger.info(getMarker(), "Az {} fájl törölhető. Rögzítve: {}, Lejárt: {}, Agency: {} ", name, + dffull.format(clip.getDate(RECORDDATE)), dffull.format(clip.getDate(KILLDATE)), agency); + + if (!notificationOnly) + delete(name); + + i++; + int progress = i * 100 / clips.size(); + setProgress(progress); + } + + return null; + } + +} diff --git a/server/-product/production/HIRTV/jobs/steps/DetectMissingLengthStep.java b/server/-product/production/HIRTV/jobs/steps/DetectMissingLengthStep.java new file mode 100644 index 00000000..830e41fc --- /dev/null +++ b/server/-product/production/HIRTV/jobs/steps/DetectMissingLengthStep.java @@ -0,0 +1,68 @@ +package user.jobengine.server.steps; + +import java.nio.file.Paths; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.apache.logging.log4j.Marker; + +import com.ibm.nosql.json.api.BasicDBObject; +import com.ibm.nosql.json.api.DB; +import com.ibm.nosql.json.api.DBCollection; +import com.ibm.nosql.json.api.DBObject; + +import user.commons.nosql.NoSQLUtils; +import user.jobengine.db.IItemManager; +import user.jobengine.db.IResultSetConsumer; +import user.jobengine.db.Media; +import user.jobengine.server.IJobEngine; +import user.jobengine.server.IJobRuntime; + +public class DetectMissingLengthStep extends JobStep { + private static final Logger logger = LogManager.getLogger(); + private Marker marker; + + @StepEntry + public Object[] execute(String localHiresPath, IJobEngine jobEngine, IJobRuntime jobRuntime) throws Exception { + Object[] result = new Object[] { null, null, "%s", null, 0, true }; + marker = jobRuntime.getSessionMarker(); + + DB db = NoSQLUtils.getNoSQLDB(); + DBCollection collection = db.getCollection("missing_length"); + IItemManager manager = jobEngine.getItemManager(); + + try { + String query = "select id from media where length=0 order by archived desc"; + IResultSetConsumer consumer = rs -> { + Media media = manager.getMedia(rs.getLong("id")); + media.setPersister(manager); + return process(collection, result, localHiresPath, media); + }; + manager.executeQuery(query.toString(), consumer, null); + if (result[0] == null) + throw new Exception("Nincs feldolgozandó hiány."); + } catch (Exception e) { + logger.catching(e); + logger.error(marker, e.getMessage()); + } finally { + setProgress(100); + } + return result; + } + + public boolean process(DBCollection collection, Object[] result, String localHiresPath, Media media) { + String name = media.getMediaFilesName(); + if (name == null) + return true; + DBObject existing = collection.findOne(new BasicDBObject("name", name)); + if (existing != null) + return true; + + result[0] = media; + ArchiveItem archiveItem = new ArchiveItem(); + archiveItem.setMediaFile(Paths.get(localHiresPath, name).toString()); + result[1] = archiveItem; + collection.save(new BasicDBObject("name", name)); + return false; + } +} diff --git a/server/-product/production/HIRTV/jobs/steps/DownloadRecordingFromNexioStep.java b/server/-product/production/HIRTV/jobs/steps/DownloadRecordingFromNexioStep.java new file mode 100644 index 00000000..c65ab01b --- /dev/null +++ b/server/-product/production/HIRTV/jobs/steps/DownloadRecordingFromNexioStep.java @@ -0,0 +1,182 @@ +package user.jobengine.server.steps; + +import java.io.File; +import java.nio.file.Paths; + +import org.apache.commons.lang.StringUtils; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.apache.logging.log4j.Marker; +import org.apache.logging.log4j.message.Message; +import org.apache.logging.log4j.message.ParameterizedMessage; + +import com.ibm.nosql.json.api.BasicDBObject; +import com.ibm.nosql.json.api.DB; +import com.ibm.nosql.json.api.DBCollection; + +import user.commons.RemoteFile; +import user.commons.StoreUri; +import user.commons.configuration.SystemConfiguration; +import user.commons.nosql.NoSQLUtils; +import user.commons.remotestore.IProgressEventListener; +import user.commons.remotestore.IStatusEventListener; +import user.commons.remotestore.ProgressEvent; +import user.commons.remotestore.RemoteStoreProtocol; +import user.commons.remotestore.StatusEvent; +import user.jobengine.db.IItemManager; +import user.jobengine.server.IJobEngine; +import user.jobengine.server.IJobRuntime; + +public class DownloadRecordingFromNexioStep extends JobStep { + private static final String MXFEXT = ".MXF"; + private static final String NEXIOCLIPS = "nexioclips"; + private static final String LONGNAMEID = "longnameid"; + private static final String DURATION = "duration"; + private static final Logger logger = LogManager.getLogger(); + + private String NEXIO_HOST = SystemConfiguration.getInstance().value("services.nexio.host"); + + private IItemManager manager; + private StoreUri targetUri; + private StoreUri sourceUri; + private Marker marker; + private DBCollection clipsCollection; + + private int check(int value, String name) { + if (value == 0) { + logger.error(marker, "A folyamat '{}' bemeneti paramétere 0.", name); + throw new NullPointerException( + String.format("System is not configured properly, missing '%s' input parameter.", name)); + } + return value; + } + + private String check(String value, String name) { + if (StringUtils.isBlank(value)) { + logger.error(marker, "A folyamat '{}' bemeneti paramétere üres.", name); + throw new NullPointerException( + String.format("System is not configured properly, missing '%s' input parameter.", name)); + } + return value; + } + + @StepEntry + public Object[] execute(ArchiveItem archiveItem, String targetPath, String targetFileName, int nexioPort, + String nexioUserName, String nexioPassword, IJobEngine jobEngine, IJobRuntime jobRuntime) throws Exception { + marker = jobRuntime.getSessionMarker(); + manager = jobEngine.getItemManager(); + setAndCheck(archiveItem, targetPath, targetFileName, nexioPort, nexioUserName, nexioPassword); + String sourceFileName = targetFileName; + try { + final IJobRuntime runtime = jobRuntime; + + RemoteFile sourceFile = null; + while (sourceFile == null) { + try { + sourceFile = sourceUri.getRemoteFile(sourceFileName); + Thread.sleep(1000); + } catch (Exception e) { + logger.warn(e.getMessage()); + } + } + + targetUri.addProgressListener(new IProgressEventListener() { + @Override + public void progressChanged(ProgressEvent evt) { + runtime.incrementProgress(evt.getProgress()); + } + }); + targetUri.addStatusListener(new IStatusEventListener() { + @Override + public void statusChanged(StatusEvent evt) { + evt.setCancel(!canContinue()); + } + }); + + String targetName = targetFileName + MXFEXT; + + File targetFile = Paths.get(targetPath, targetName).toFile(); + if (targetFile.exists()) + throw new Exception("Exists!"); + + RemoteFile remoteFile = sourceUri.transferFrom(targetUri, sourceFileName, targetName); + + logger.info(marker, "Az '{}' állomány letöltése sikeres volt '{}' néven.", sourceFileName, targetFile); + + BasicDBObject clip = (BasicDBObject) clipsCollection.findOne(new BasicDBObject(LONGNAMEID, sourceFileName)); + if (clip == null) + throw new Exception("Clip does not exist in NEXIO"); + + long duration = NoSQLUtils.asLong(clip, DURATION); + if (duration == 0) + throw new Exception("Clip duration is 0"); + + archiveItem.setDuration(duration); + archiveItem.setMediaFile(targetFile.toString()); + + RemoteFile sourceRemoteFile = sourceUri.getRemoteFile(sourceFileName); + sourceUri.delete(sourceRemoteFile); + + } catch (Exception e) { + logger.catching(e); + if (!archiveItem.removeCatchedFile()) + logger.error(getMarker(), "A {} állomány .catched jelző állománya nem törölhető.", + new File(archiveItem.getMediaFile()).getName()); + Message m = new ParameterizedMessage("Az '{}' állomány feldolgozása sikertelen. A rendszer hibaüzenete: {}", + sourceFileName, e.getMessage()); + logger.error(marker, m); + throw new Exception(m.getFormattedMessage()); + } + return new Object[] { 0 }; + } + + private void setAndCheck(ArchiveItem archiveItem, String targetPath, String targetFileName, int nexioPort, + String nexioUserName, String nexioPassword) throws Exception { + DB db = NoSQLUtils.getNoSQLDB(); + if (db == null) { + logger.error(marker, "Az NoSQL adatkezelő réteg nem elérhető."); + throw new NullPointerException("Internal error, missing NoSQL DB reference."); + } + + clipsCollection = db.getCollection(NEXIOCLIPS); + + if (archiveItem == null) { + logger.error(marker, "A folyamat 'archiveItem' bemeneti paramétere üres."); + throw new NullPointerException("Internal error, missing 'archiveItem'."); + } + + if (archiveItem.getMediaFile() == null) { + logger.error(marker, "A folyamat 'archiveItem.mediaFile' paramétere üres."); + throw new NullPointerException("Internal error, missing 'archiveItem.mediaFile'."); + } + + check(targetFileName, "targetFileName"); + check(targetPath, "targetPath"); + + if (StringUtils.isBlank(NEXIO_HOST)) { + logger.error(marker, "A 'nexio.host' rendszer paraméter nem található."); + throw new NullPointerException( + "System is not configured properly, 'jobengine.selenio.address' startup parameter missing."); + } + check(nexioPort, "nexioPort"); + check(nexioUserName, "nexioUserName"); + check(nexioPassword, "nexioPassword"); + + sourceUri = manager.createStoreUri(RemoteStoreProtocol.FTP, NEXIO_HOST); + sourceUri.setPortNumber(nexioPort); + sourceUri.setUserName(nexioUserName); + sourceUri.setPassword(nexioPassword); + if (sourceUri == null) { + logger.error(marker, "A forrás nem elérhető."); + throw new NullPointerException("Internal error, missing 'sourceUri'."); + } + + targetUri = manager.createStoreUri(RemoteStoreProtocol.LOCAL, targetPath); + if (targetUri == null) { + logger.error(marker, "A cél nem elérhető."); + throw new NullPointerException("Internal error, missing 'targetUri'."); + } + + } + +} diff --git a/server/-product/production/HIRTV/jobs/steps/DuplicateRemoverStep.java b/server/-product/production/HIRTV/jobs/steps/DuplicateRemoverStep.java new file mode 100644 index 00000000..5e407eb3 --- /dev/null +++ b/server/-product/production/HIRTV/jobs/steps/DuplicateRemoverStep.java @@ -0,0 +1,102 @@ +package user.jobengine.server.steps; + +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.apache.logging.log4j.Marker; + +import user.jobengine.db.IItemManager; +import user.jobengine.db.IResultSetConsumer; +import user.jobengine.db.IStatementDecorator; +import user.jobengine.db.MediaFile; +import user.jobengine.server.IJobEngine; +import user.jobengine.server.IJobRuntime; +import user.jobengine.server.steps.shared.EscortFiles; + +public class DuplicateRemoverStep extends JobStep { + private static final Logger logger = LogManager.getLogger(); + private static final String LOWRES_ROOT = "/mediacube/data/lowres/www/video"; + private static final String DUPLICATES_ROOT = "/mediacube/data/lowres/www"; + + private Marker marker; + private IItemManager manager; + + @StepEntry + public Object[] execute(int limit, IJobEngine jobEngine, IJobRuntime jobRuntime) throws Exception { + marker = jobRuntime.getSessionMarker(); + manager = jobEngine.getItemManager(); + processLowresDuplicates(limit); + return null; + } + + public void processLowresDuplicates(int limit) { + long count[] = new long[] { 0, 0 }; + + manager.executeQuery("select count(filecount) from vw_items_rd_dup", rs -> { + count[0] = rs.getLong(1); + return false; + }, null); + + if (count[0] == 0) { + setProgress(100); + return; + } + + if (limit > 0) + count[0] = limit; + + manager.executeQuery("select filename from vw_items_rd_dup order by filecount desc", rs -> { + String fileName = rs.getString("filename"); + innerProcessLowresDuplicates(fileName); + count[1]++; + int progress = (int) ((double) count[1] * 100 / count[0]); + setProgress(progress); + if (count[0] == count[1]) + return false; + else + return true; + }, null); + } + + private void innerProcessLowresDuplicates(String fileName) { + String query = "select mediafileid, mediafilehouseid, relativepath from vw_items_rd_lh where filename = ?"; + IStatementDecorator decorator = st -> { + st.setString(1, fileName); + }; + + MediaFile[] masterMediaFile = { null }; + + IResultSetConsumer consumer = rs -> { + long mediaFileId = rs.getLong("mediafileid"); + MediaFile mediaFile = (MediaFile) manager.get(MediaFile.class, mediaFileId); + if (masterMediaFile[0] == null) { + masterMediaFile[0] = mediaFile; + return true; + } + + String path = rs.getString("relativepath"); + boolean moved = false; + try { + Path target = Paths.get(DUPLICATES_ROOT, path); + EscortFiles.ensureUNCFolder(target.getParent()); + Files.move(Paths.get(LOWRES_ROOT, path), target); + moved = true; + } catch (Exception e) { + logger.catching(e); + } + + if (moved) { + logger.info(marker, "{} {} {}", mediaFile.getMediaId(), mediaFile.getRelativePath(), + masterMediaFile[0].getRelativePath()); + mediaFile.setRelativePath(masterMediaFile[0].getRelativePath()); + manager.modify(mediaFile); + } + return true; + }; + manager.executeQuery(query, consumer, decorator); + } + +} diff --git a/server/-product/production/HIRTV/jobs/steps/GenerateMorpheusMetadataStep.java b/server/-product/production/HIRTV/jobs/steps/GenerateMorpheusMetadataStep.java new file mode 100644 index 00000000..f27dd07b --- /dev/null +++ b/server/-product/production/HIRTV/jobs/steps/GenerateMorpheusMetadataStep.java @@ -0,0 +1,32 @@ +package user.jobengine.server.steps; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + +import com.ibm.nosql.json.api.BasicDBObject; + +import user.commons.morpheus.MorpheusStrings; +import user.jobengine.server.IJobEngine; +import user.jobengine.server.IJobRuntime; +import user.jobengine.server.steps.shared.EscortFiles; +import user.jobengine.server.steps.shared.PlanAirExtensions; + +public class GenerateMorpheusMetadataStep extends JobStep { + private static final Logger logger = LogManager.getLogger(); + + @StepEntry + public Object[] execute(BasicDBObject material, String morpheusDeviceID, String dbUrl, String userName, String password, String targetMetadataPath, + IJobEngine jobEngine, IJobRuntime jobRuntime) throws Exception { + try { + String name = material.getString(MorpheusStrings.MATERIALID); + String content = PlanAirExtensions.getMorpeusXML(jobEngine.getItemManager(), dbUrl, userName, password, name, morpheusDeviceID); + EscortFiles.createMorpheusXML(targetMetadataPath, name + ".xml", content); + } catch (Exception e) { + logger.catching(e); + logger.error(jobRuntime.getSessionMarker(), "Hiba a Morpheus kísérő XML létrehozásakor. A rendszer üzenete: {}", e.getMessage()); + throw e; + } + return null; + } + +} diff --git a/server/-product/production/HIRTV/jobs/steps/HLSProxyStep.java b/server/-product/production/HIRTV/jobs/steps/HLSProxyStep.java new file mode 100644 index 00000000..7d2704c4 --- /dev/null +++ b/server/-product/production/HIRTV/jobs/steps/HLSProxyStep.java @@ -0,0 +1,77 @@ +package user.jobengine.server.steps; + +import java.nio.file.Path; +import java.nio.file.Paths; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.apache.logging.log4j.Marker; + +import user.commons.StoreUri; +import user.commons.remotestore.RemoteStoreProtocol; +import user.jobengine.db.Media; +import user.jobengine.db.MediaFile; +import user.jobengine.db.Store; +import user.jobengine.server.steps.shared.EscortFiles; +import user.jobengine.server.steps.shared.FFMpeg; + +public class HLSProxyStep extends JobStep { + private static final Logger logger = LogManager.getLogger(); + private static final String LOWRES_FILETYPE = "Low-res"; + private Marker marker; + + @StepEntry + public Object[] execute(ArchiveItem archiveItem, Media mediaCubeMedia) throws Exception { + marker = getSessionMarker(); + + Path sourceFilePath = Paths.get(archiveItem.getMediaFile()); + try { + String fileName = sourceFilePath.getFileName().toString(); + String proxyName = fileName.substring(0, fileName.lastIndexOf(".")) + "S01" + fileName.substring(fileName.lastIndexOf(".")); + Path lowresSourcePath = Paths.get(sourceFilePath.getParent().toString(), EscortFiles.STATUSFOLDER, proxyName); + if (!lowresSourcePath.toFile().exists()) { + logger.info(marker, "HLS proxy not found for {}", archiveItem.getMediaFile()); + return null; + } + + Store lowresStore = getManager().getCurrentLowresStore(); + StoreUri lowresStoreUri = lowresStore.getTargetStoreUri(RemoteStoreProtocol.LOCAL); + String webPath = lowresStoreUri.toString(true); + + Path subdirPath = null; + if (proxyName.indexOf(".") > 2) { + subdirPath = Paths.get(proxyName.substring(0, 1), proxyName.substring(1, 2), proxyName.substring(2, 3), proxyName); + } else { + subdirPath = Paths.get(proxyName); + } + + String subDir = subdirPath.toString(); + Path lowresTargetPath = Paths.get(webPath, subDir); + + int version = 1; + while (lowresTargetPath.toFile().exists()) { + subDir = subDir + version; + lowresTargetPath = Paths.get(webPath, subDir); + version++; + } + + EscortFiles.ensureUNCFolder(webPath, subDir); + lowresTargetPath = Paths.get(webPath, subDir); + // Files.move(lowresSourcePath, lowresTargetPath); + + FFMpeg.hls_audio4ch(lowresSourcePath.toAbsolutePath().toString(), lowresTargetPath.toAbsolutePath().toString(), p -> { + setProgress((int) p); + }); + + Path lowresHTTPPath = Paths.get(subDir, "index.m3u8"); + MediaFile mediaFile = getManager().createMediaFile(lowresHTTPPath.toString(), LOWRES_FILETYPE, lowresStore.getName()); + mediaFile.setMediaId(mediaCubeMedia.getId()); + getManager().add(mediaFile); + } catch (Exception e) { + logger.catching(e); + logger.error(marker, "A HLS proxy létrehozása sikertelen a '{}' fájlból. A rendszer üzenete: {}", sourceFilePath, e.getMessage()); + } + return null; + } + +} diff --git a/server/-product/production/HIRTV/jobs/steps/ImportMORPHEUSMissingMaterialsStep.java b/server/-product/production/HIRTV/jobs/steps/ImportMORPHEUSMissingMaterialsStep.java new file mode 100644 index 00000000..f2216a33 --- /dev/null +++ b/server/-product/production/HIRTV/jobs/steps/ImportMORPHEUSMissingMaterialsStep.java @@ -0,0 +1,255 @@ +package user.jobengine.server.steps; + +import java.io.File; +import java.io.IOException; +import java.nio.charset.Charset; +import java.nio.file.DirectoryStream; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.text.SimpleDateFormat; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Date; +import java.util.HashMap; +import java.util.List; +import java.util.Locale; +import java.util.Map; +import java.util.Set; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + +import com.ibm.nosql.json.api.BasicDBObject; +import com.ibm.nosql.json.api.DB; +import com.ibm.nosql.json.api.DBCollection; +import com.ibm.nosql.json.api.DBObject; +import com.ibm.nosql.json.api.QueryBuilder; + +import user.commons.IEntityBase; +import user.commons.morpheus.MorpheusStrings; +import user.commons.nosql.NoSQLUtils; +import user.jobengine.db.IItemManager; +import user.jobengine.db.MediaFile; +import user.jobengine.db.MediaFileDAO; +import user.jobengine.server.IJobEngine; +import user.jobengine.server.IJobRuntime; +import user.jobengine.server.steps.shared.EscortFiles; + +public class ImportMORPHEUSMissingMaterialsStep extends JobStep { + private static final Logger logger = LogManager.getLogger(); + + private static final String CSV_EXT = ".csv"; + private static final String MXF_EXT = ".MXF"; + private MediaFileDAO dao; + private String processedFolder; + private DB db; + private String targetPath; + private IJobRuntime jobRuntime; + private int overall; + private int current; + private final SimpleDateFormat enDateFormat = new SimpleDateFormat(MorpheusStrings.FORMAT_TIME_TO_AIR, Locale.ENGLISH); + + private Map buildMetadataMap(Path csvFilePath, String[] data) throws Exception { + Map result = new HashMap<>(); + List dataList = Arrays.asList(data); + storeMetadataPosition(csvFilePath, dataList, MorpheusStrings.MATERIAL_ID, result); + storeMetadataPosition(csvFilePath, dataList, MorpheusStrings.CHANNEL, result); + storeMetadataPosition(csvFilePath, dataList, MorpheusStrings.TIME_TO_AIR, result); + storeMetadataPosition(csvFilePath, dataList, MorpheusStrings.DURATION, result); + storeMetadataPosition(csvFilePath, dataList, MorpheusStrings.TITLE, result); + // storeMetadataPosition(csvFilePath, dataList, MorpheusStrings.DEVICE_ID, + // result); + // storeMetadataPosition(csvFilePath, dataList, MorpheusStrings.REASON, result); + return result; + } + + @StepEntry + public Object[] execute(String sourceCSVPath, String processedFolder, String targetPath, IJobEngine jobEngine, IJobRuntime jobRuntime) throws Exception { + this.jobRuntime = jobRuntime; + setAndCheck(sourceCSVPath, processedFolder, targetPath, jobEngine); + + List filePaths = new ArrayList<>(); + try (DirectoryStream directoryStream = Files.newDirectoryStream(Paths.get(sourceCSVPath))) { + for (Path path : directoryStream) { + filePaths.add(path); + } + + processPathItems(filePaths); + } catch (Exception e) { + logger.catching(e); + logger.error(getSessionMarker(), "Hiba a végrehajtás során. A rendszer üzenete: {}", e.getMessage()); + } + return null; + } + + private void moveProcessedCSV(Path csvFilePath) throws IOException { + EscortFiles.ensureUNCFolder(csvFilePath.getParent().toString(), processedFolder); + String fileName = csvFilePath.getFileName().toString() + "." + EscortFiles.composeKillDate(0); + Path targetPath = Paths.get(csvFilePath.getParent().toString(), processedFolder, fileName); + Files.move(csvFilePath, targetPath); + } + + private void processLine(String[] data, Map metadatas) throws Exception { + String channel = data[metadatas.get(MorpheusStrings.CHANNEL)]; + String timeToAir = data[metadatas.get(MorpheusStrings.TIME_TO_AIR)]; + String duration = data[metadatas.get(MorpheusStrings.DURATION)]; + String materialID = data[metadatas.get(MorpheusStrings.MATERIAL_ID)]; + String title = data[metadatas.get(MorpheusStrings.TITLE)]; + // String deviceID = data[metadatas.get(MorpheusStrings.DEVICE_ID)]; + // String reason = data[metadatas.get(MorpheusStrings.REASON)]; + + DBObject query = QueryBuilder.start() + .and(QueryBuilder.start(MorpheusStrings.MATERIALID).is(materialID).get(), QueryBuilder.start(MorpheusStrings.TIMETOAIR).is(timeToAir).get()) + .get(); + DBCollection collection = db.getCollection(MorpheusStrings.COLLECTION_NAME); + BasicDBObject existingObject = (BasicDBObject) collection.findOne(query); + if (existingObject != null) { + logger.warn(getSessionMarker(), "Az '{}' anyag már feldolgozásra került az {} időpontban.", materialID, + existingObject.getDate(MorpheusStrings.IMPORTED)); + return; + } + + BasicDBObject dbObject = new BasicDBObject(MorpheusStrings.IMPORTED, new Date()); + dbObject.put(MorpheusStrings.CHANNEL, channel); + dbObject.put(MorpheusStrings.TIMETOAIR, enDateFormat.parse(timeToAir)); + dbObject.put(MorpheusStrings.DURATION, duration); + dbObject.put(MorpheusStrings.MATERIALID, materialID); + dbObject.put(MorpheusStrings.TITLE, title); + // dbObject.put(MorpheusStrings.DEVICEID, deviceID); + // dbObject.put(MorpheusStrings.REASON, reason); + + String fileName = materialID + MXF_EXT; + + Path targetFilePath = Paths.get(targetPath, fileName); + boolean exists = Files.exists(targetFilePath); + if (exists && targetFilePath.toFile().length() > 0) { + logger.warn(getSessionMarker(), "Az '{}' anyag már be van töltve.", materialID); + dbObject.put(MorpheusStrings.STATUS, MorpheusStrings.STATUS_SKIPPED); + } else { + List medias = dao.getByHouseId(fileName); + if (medias == null || medias.size() == 0) { + logger.warn(getSessionMarker(), "Az '{}' anyag nem található az archívumban.", materialID); + dbObject.put(MorpheusStrings.STATUS, MorpheusStrings.STATUS_UNAVAILABLE); + } else if (medias.size() > 1) { + logger.warn(getSessionMarker(), "Az '{}' anyagból egynél több található az archívumban.", materialID); + dbObject.put(MorpheusStrings.STATUS, MorpheusStrings.STATUS_MULTIPLE); + } else { + logger.info(getSessionMarker(), "Az '{}' anyag megtalálható az archívumban.", materialID); + dbObject.put(MorpheusStrings.STATUS, MorpheusStrings.STATUS_RESTORABLE); + + MediaFile mf = (MediaFile) medias.get(0); + dbObject.put(MorpheusStrings.MEDIAID, mf.getMediaId()); + } + + } + + collection.insert(dbObject); + } + + // Channel,Time to Air,Duration,Material ID,Title,Device ID,Reason, + // TX02,10-JAN-2018 13:25:21:08,00:05:00:00,M009572A,Tiéd a pálya/26. - 1. seg - + // Eredeti ** mc ,ICELE-02,On Domain (ISILON, ICELE-01, ICELE-05) , + private void processMissingMaterialCSV(Path csvFilePath, List lines) throws Exception { + if (lines == null | lines.size() == 0) { + return; + } + + Map metadatas = null; + for (int i = 0; i < lines.size(); i++) { + String line = lines.get(i); + if (line == null) + continue; + String[] data = line.split(","); + if (i == 0) + metadatas = buildMetadataMap(csvFilePath, data); + else + processLine(data, metadatas); + + current++; + int progress = current * 100 / overall; + setProgress(progress); + } + + } + + private void processPathItem(Path csvFilePath, List lines) { + File csvFile = csvFilePath.toFile(); + + try { + processMissingMaterialCSV(csvFilePath, lines); + moveProcessedCSV(csvFilePath); + } catch (Exception e) { + logger.catching(e); + logger.error(getSessionMarker(), "A {} MORPHEUS állomány mozgatásakor hiba történt. A rendszer hibaüzenete: {}.", csvFile.getName(), + e.getMessage()); + } + } + + private void processPathItems(List filePaths) throws IOException { + overall = 0; + Map> contents = new HashMap<>(); + for (Path filePath : filePaths) { + File csvFile = filePath.toFile(); + if (csvFile.isDirectory() || !csvFile.getName().toLowerCase().endsWith(CSV_EXT.toLowerCase())) + continue; + logger.info(getSessionMarker(), "Processing {}", filePath); + List lines = Files.readAllLines(filePath, Charset.forName("UTF-8")); + overall += lines.size(); + contents.put(filePath, lines); + } + + Set csvPaths = contents.keySet(); + for (Path csvPath : csvPaths) { + processPathItem(csvPath, contents.get(csvPath)); + } + } + + private void setAndCheck(String sourcePath, String processedFolder, String targetPath, IJobEngine jobEngine) { + if (jobEngine == null) { + logger.error(getSessionMarker(), "Az folyamatkezelő réteg nem elérhető."); + throw new NullPointerException("Internal error, missing JobEngine reference."); + } + + IItemManager manager = jobEngine.getItemManager(); + if (manager == null) { + logger.error(getSessionMarker(), "Az adatbáziskezelő réteg nem elérhető."); + throw new NullPointerException("Internal error, missing ItemManager reference."); + } + dao = (MediaFileDAO) manager.getBaseDAO(MediaFile.class); + if (dao == null) { + logger.error(getSessionMarker(), "Az adatbáziskezelő réteg MediaFile kezelöje nem elérhető."); + throw new NullPointerException("Internal error, missing MediaFile DAO reference."); + } + if (sourcePath == null) { + logger.error(getSessionMarker(), "A folyamat 'sourcePath' bemeneti paramétere üres."); + throw new NullPointerException("System is not configured properly, 'sourcePath' input parameter missing."); + } + if (processedFolder == null) { + logger.error(getSessionMarker(), "A folyamat 'processedFolder' bemeneti paramétere üres."); + throw new NullPointerException("System is not configured properly, 'processedFolder' input parameter missing."); + } + this.processedFolder = processedFolder; + + if (targetPath == null) { + logger.error(getSessionMarker(), "A folyamat 'targetPath' bemeneti paramétere üres."); + throw new NullPointerException("System is not configured properly, 'targetPath' input parameter missing."); + } + this.targetPath = targetPath; + + db = NoSQLUtils.getNoSQLDB(); + if (db == null) { + logger.error(getSessionMarker(), "Sikertelen kapcsolódás a NoSQL adatbázishoz."); + throw new NullPointerException("Can not connect to NoSQL database."); + } + + } + + private void storeMetadataPosition(Path csvFilePath, List dataList, String name, Map metadatas) throws Exception { + int pos = dataList.indexOf(name); + if (pos < 0) + throw new Exception(String.format("A '%s' MORPHEUS állományban nem található a '%s' mező.", csvFilePath.getFileName(), name)); + metadatas.put(name, pos); + } + +} diff --git a/server/-product/production/HIRTV/jobs/steps/ImportStatisticsStep.java b/server/-product/production/HIRTV/jobs/steps/ImportStatisticsStep.java new file mode 100644 index 00000000..7a34a4df --- /dev/null +++ b/server/-product/production/HIRTV/jobs/steps/ImportStatisticsStep.java @@ -0,0 +1,365 @@ +package user.jobengine.server.steps; + +import java.util.ArrayList; +import java.util.Calendar; +import java.util.Date; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.TimeZone; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.apache.logging.log4j.Marker; + +import com.ibm.nosql.json.api.BasicDBList; +import com.ibm.nosql.json.api.BasicDBObject; +import com.ibm.nosql.json.api.DB; +import com.ibm.nosql.json.api.DBCollection; +import com.ibm.nosql.json.api.DBObject; + +import user.commons.CalendarUtils; +import user.commons.nosql.NoSQLUtils; +import user.commons.octopus.IOctopusAPI; +import user.commons.octopus.OctopusAPI; +import user.jobengine.db.IItemManager; +import user.jobengine.server.IJobEngine; +import user.jobengine.server.IJobRuntime; +import user.jobengine.server.steps.shared.ItemManagerExtensions; + +public class ImportStatisticsStep extends JobStep { + private static final Logger logger = LogManager.getLogger(); + private static final String SCHEDULED_FORMAT = "yyyy.MM.dd HH:mm"; + private static final String NEXIOCLIPS = "nexioclips"; + private static final String LONGNAMEID = "longnameid"; + private static final String DURATION = "duration"; + private IOctopusAPI octopusAPI; + private Marker marker; + private int overall; + private int current; + private DB db; + private DBCollection clipCollection; + private IItemManager manager; + + private void createAPI(IJobRuntime jobRuntime) throws Exception { + try { + octopusAPI = new OctopusAPI(); + } catch (Exception e) { + logger.catching(e); + logger.error(jobRuntime.getSessionMarker(), "Az Octopus adatbázis nem érhető el. A rendszer üzenete: {}", e.getMessage()); + throw e; + } + } + + private BasicDBObject createFieldsObject() { + BasicDBObject result = new BasicDBObject(); + result.put(IOctopusAPI._ID, 0); + result.put(IOctopusAPI.ID, 1); + result.put(IOctopusAPI.NAME, 1); + result.put(IOctopusAPI.REF_RUNDOWN, 1); + result.put(IOctopusAPI.REF_STORYFOLDER, 1); + result.put(IOctopusAPI.MOS_OBJECTS, 1); + result.put(IOctopusAPI.PARENT_STORY_ID, 1); + result.put(IOctopusAPI.SCHEDULEFROM, 1); + result.put(IOctopusAPI.REPORTERS, 1); + return result; + } + + private BasicDBObject createStory(BasicDBObject story, String label, String objId, long duration, long rdCount, long sfCount, long parentStoryId, + String reporters) { + BasicDBObject raw = new BasicDBObject(); + raw.put(IOctopusAPI.ID, NoSQLUtils.asLong(story, IOctopusAPI.ID)); + raw.put(IOctopusAPI.NAME, NoSQLUtils.asString(story, IOctopusAPI.NAME)); + raw.put(IOctopusAPI.PARENT_STORY_ID, parentStoryId); + raw.put(IOctopusAPI.REPORTERS, reporters); + raw.put(IOctopusAPI.OBJ_ID, objId); + raw.put(IOctopusAPI.LABEL, label); + raw.put("duration", duration); + raw.put(IOctopusAPI.REF_RUNDOWN, rdCount); + raw.put(IOctopusAPI.REF_STORYFOLDER, sfCount); + return raw; + } + + @StepEntry + public Object[] execute(int daysBeforeNow, IJobEngine jobEngine, IJobRuntime jobRuntime) throws Exception { + marker = jobRuntime.getSessionMarker(); + manager = jobEngine.getItemManager(); + db = NoSQLUtils.getNoSQLDB(); + clipCollection = db.getCollection(NEXIOCLIPS); + + createAPI(jobRuntime); + + Calendar scheduledDate = CalendarUtils.createZeroCalendar(Calendar.getInstance()); + scheduledDate.setTimeZone(TimeZone.getTimeZone("Europe/Budapest")); + scheduledDate.add(Calendar.DAY_OF_YEAR, -1 * daysBeforeNow); + List rundowns = octopusAPI.getRundowns(scheduledDate.getTime()); + if (rundowns == null) + logger.warn(marker, "Nincs adástükör a {} napra.", CalendarUtils.toDateString(scheduledDate)); + else + overall += rundowns.size(); + + List folders = octopusAPI.getStoryFolders(); + if (folders == null) + logger.warn(marker, "Nincs gyűjtőmappa."); + else + overall += folders.size(); + + Map stories = processRundowns(rundowns); + logger.info(jobRuntime.getSessionMarker(), "Adástükörben megtalálható anyagok száma {}", stories.size()); + Map folderStories = processFolders(folders, scheduledDate); + stories.putAll(folderStories); + logger.info(jobRuntime.getSessionMarker(), "Gyűjtőkben megtalálható anyagok száma {}", stories.size()); + + storeStories(scheduledDate, stories); + return null; + } + + private Map processFolders(List folders, Calendar scheduledDate) { + Map result = new HashMap<>(); + for (DBObject f : folders) { + BasicDBObject folder = (BasicDBObject) f; + String folderName = null; + long folderID = 0; + try { + folderID = NoSQLUtils.asLong(folder, IOctopusAPI.ID); + folderName = folder.getString(IOctopusAPI.NAME); + List folderStories = octopusAPI.getStoryFolderStories(folderID, createFieldsObject()); + if (folderStories != null) { + + Map stories = processStories(folderStories, scheduledDate); + result.putAll(stories); + } + } catch (Exception e) { + logger.catching(e); + logger.error(marker, String.format("A %s %s mappa feldolgozása nem lehetséges. A rendszer üzenete: %s", folderID, folderName, e.getMessage())); + throw e; + } + current++; + int progress = current * 100 / overall; + setProgress(progress); + } + return result; + } + + private Map processRundowns(List rundowns) { + Map result = new HashMap<>(); + for (DBObject r : rundowns) { + BasicDBObject rundown = (BasicDBObject) r; + String rundownName = null; + long rundownID = 0; + try { + rundownID = NoSQLUtils.asLong(rundown, IOctopusAPI.ID); + String name = NoSQLUtils.asString(NoSQLUtils.asDBObject(rundown, IOctopusAPI.RUNDOWN_TYPE), IOctopusAPI.NAME); + String channel = NoSQLUtils.asString(NoSQLUtils.asDBObject(rundown, IOctopusAPI.CHANNEL), IOctopusAPI.NAME); + Date scheduledStart = rundown.getDate(IOctopusAPI.SCHEDULED_START); + String start = CalendarUtils.toString(CalendarUtils.createCalendar(scheduledStart), SCHEDULED_FORMAT); + // String rundownName = NoSQLUtils.asString(rundown, IOctopusAPI.NAME); + rundownName = String.format("%s %s %s", start, name, channel); + List stories = octopusAPI.getRundownStories(rundownID, createFieldsObject()); + if (stories != null) + result.putAll(processStories(stories, null)); + } catch (Exception e) { + logger.catching(e); + logger.error(marker, + String.format("A %s %s tükör feldolgozása nem lehetséges. A rendszer üzenete: %s", rundownID, rundownName, e.getMessage())); + throw e; + } + current++; + int progress = current * 100 / overall; + setProgress(progress); + } + return result; + } + + private Map processStories(List stories, Calendar scheduledDate) { + Map result = new HashMap<>(); + for (DBObject storyObject : stories) { + BasicDBObject story = (BasicDBObject) storyObject; + + long storyID = NoSQLUtils.asLong(story, IOctopusAPI.ID); + if (storyID == 0) + continue; + if (result.containsKey(storyID)) + continue; + + if (scheduledDate != null) { + Date scheduleFrom = story.getDate(IOctopusAPI.SCHEDULEFROM); + if (scheduleFrom == null) + continue; + if (scheduleFrom.getTime() != scheduledDate.getTime().getTime()) + continue; + } + + List mosObjects = NoSQLUtils.asList(story, IOctopusAPI.MOS_OBJECTS); + if (mosObjects == null || mosObjects.isEmpty()) + continue; + + boolean foundAnyClip = false; + for (BasicDBObject mosObject : mosObjects) { + String clipName = NoSQLUtils.asString(mosObject, IOctopusAPI.OBJ_ID); + if (clipName != null && clipName.length() > 0) { + BasicDBObject clip = (BasicDBObject) clipCollection.findOne(new BasicDBObject(LONGNAMEID, clipName)); + if (clip != null) { + long duration = NoSQLUtils.asLong(clip, DURATION); + // az ures klipp 1 kocka hosszu + if (duration > 1) { + mosObject.put("duration", duration); + foundAnyClip = true; + } + } + } + } + if (!foundAnyClip) + continue; + + List rdList = NoSQLUtils.asList(story, IOctopusAPI.REF_RUNDOWN); + story.put(IOctopusAPI.REF_RUNDOWN, rdList == null ? 0 : rdList.size()); + List sfList = NoSQLUtils.asList(story, IOctopusAPI.REF_STORYFOLDER); + story.put(IOctopusAPI.REF_STORYFOLDER, sfList == null ? 0 : sfList.size()); + + List reporters = NoSQLUtils.asList(story, IOctopusAPI.REPORTERS); + String reporterNames = ""; + if (reporters != null && reporters.size() > 0) { + for (BasicDBObject reporter : reporters) { + String userName = NoSQLUtils.asString(reporter, "longName"); + if (userName != null) { + if (reporterNames.length() > 0) + reporterNames += ", "; + reporterNames += userName; + } + } + } + story.put(IOctopusAPI.REPORTERS, reporterNames); + + result.put(storyID, story); + } + return result; + } + + private void store(Calendar scheduledDate, List stories, BasicDBObject typeStat, BasicDBObject planStat, BasicDBObject reporterStat) { + try { + BasicDBObject dailyHistory = new BasicDBObject(); + dailyHistory.put("dateTime", scheduledDate.getTime()); + BasicDBList list = new BasicDBList(); + dailyHistory.put("rawData", list); + for (BasicDBObject story : stories) { + list.add(story); + } + dailyHistory.put("typeStat", typeStat); + dailyHistory.put("planStat", planStat); + dailyHistory.put("reporterStat", reporterStat); + + DBCollection collection = db.getCollection("daily_news_history"); + collection.remove(new BasicDBObject("dateTime", scheduledDate.getTime())); + collection.save(dailyHistory); + + } catch (Exception e) { + logger.error(e); + throw e; + } + } + + private void storeStories(Calendar scheduledDate, Map stories) { + BasicDBObject planStat = new BasicDBObject(); + BasicDBObject typeStat = new BasicDBObject(); + BasicDBObject reporterStat = new BasicDBObject(); + + List rawData = new ArrayList<>(); + List processed = new ArrayList<>(); + + for (BasicDBObject story : stories.values()) { + List mosObjects = NoSQLUtils.asList(story, IOctopusAPI.MOS_OBJECTS); + for (BasicDBObject mos : mosObjects) { + String objId = NoSQLUtils.asString(mos, IOctopusAPI.OBJ_ID); + if (!mos.containsKey("duration")) + continue; + + long parentStoryId = NoSQLUtils.asLong(story, IOctopusAPI.PARENT_STORY_ID); + logger.info("Processing parentStoryId {}", parentStoryId); + + String key = String.format("%d-%s", parentStoryId, objId); + if (processed.contains(key)) + continue; + processed.add(key); + + String label = NoSQLUtils.asString(mos, IOctopusAPI.LABEL); + if (label == null) { + logger.error(marker, "Az anyag MOS objektuma nem tartalmaz formátum meghatározást, feldolgozása nem lehetséges. Részletek:"); + logger.error(marker, "STORY: {}", story.toString()); + logger.error(marker, "MOS: {}", mos.toString()); + continue; + } + long duration = NoSQLUtils.asLong(mos, "duration"); + long rdCount = NoSQLUtils.asLong(story, IOctopusAPI.REF_RUNDOWN); + long sfCount = NoSQLUtils.asLong(story, IOctopusAPI.REF_STORYFOLDER); + String reporters = NoSQLUtils.asString(story, IOctopusAPI.REPORTERS); + + BasicDBObject raw = createStory(story, label, objId, duration, rdCount, sfCount, parentStoryId, reporters); + rawData.add(raw); + + NoSQLUtils.addLong(planStat, "all_count", 1); + NoSQLUtils.addLong(planStat, "all_duration", duration); + + BasicDBObject perTypeStat = NoSQLUtils.asDBObjectOrCreate(typeStat, label); + NoSQLUtils.addLong(perTypeStat, "all_count", 1); + NoSQLUtils.addLong(perTypeStat, "all_duration", duration); + + BasicDBObject perReporterStat = NoSQLUtils.asDBObjectOrCreate(reporterStat, reporters); + NoSQLUtils.addLong(perReporterStat, "all_count", 1); + NoSQLUtils.addLong(perReporterStat, "all_duration", duration); + + BasicDBObject ingestInfo = manager.getIngestInfo(scheduledDate, String.valueOf(parentStoryId)); + if (ingestInfo != null) { + long ingestCount = ingestInfo.getLong("count"); + long ingestDuration = ingestInfo.getLong("duration"); + raw.put("ingest_count", ingestCount); + raw.put("ingest_duration", ingestDuration); + NoSQLUtils.addLong(planStat, "ingest_count", ingestCount); + NoSQLUtils.addLong(planStat, "ingest_duration", ingestDuration); + NoSQLUtils.addLong(perTypeStat, "ingest_count", ingestCount); + NoSQLUtils.addLong(perTypeStat, "ingest_duration", ingestDuration); + NoSQLUtils.addLong(perReporterStat, "ingest_count", ingestCount); + NoSQLUtils.addLong(perReporterStat, "ingest_duration", ingestDuration); + } + + BasicDBObject archiveInfo = ItemManagerExtensions.getArchiveInfo(manager, parentStoryId); + if (archiveInfo != null) { + long archiveCount = archiveInfo.getLong("count"); + long archiveDuration = archiveInfo.getLong("duration"); + raw.put("archive_count", archiveCount); + raw.put("archive_duration", archiveDuration); + NoSQLUtils.addLong(planStat, "archive_count", archiveCount); + NoSQLUtils.addLong(planStat, "archive_duration", archiveDuration); + NoSQLUtils.addLong(perTypeStat, "archive_count", archiveCount); + NoSQLUtils.addLong(perTypeStat, "archive_duration", archiveDuration); + NoSQLUtils.addLong(perReporterStat, "archive_count", archiveCount); + NoSQLUtils.addLong(perReporterStat, "archive_duration", archiveDuration); + } + + // skipped + if (sfCount > 0 && rdCount == 0) { + NoSQLUtils.addLong(planStat, "skip_count", 1); + NoSQLUtils.addLong(planStat, "skip_duration", duration); + NoSQLUtils.addLong(perTypeStat, "skip_count", 1); + NoSQLUtils.addLong(perTypeStat, "skip_duration", duration); + NoSQLUtils.addLong(perReporterStat, "skip_count", 1); + NoSQLUtils.addLong(perReporterStat, "skip_duration", duration); + } + + // onair + if (rdCount > 0) { + NoSQLUtils.addLong(planStat, "onair_count", 1); + NoSQLUtils.addLong(planStat, "onair_duration", duration); + NoSQLUtils.addLong(perTypeStat, "onair_count", 1); + NoSQLUtils.addLong(perTypeStat, "onair_duration", duration); + NoSQLUtils.addLong(perReporterStat, "onair_count", 1); + NoSQLUtils.addLong(perReporterStat, "onair_duration", duration); + } + + } + } + + store(scheduledDate, rawData, typeStat, planStat, reporterStat); + } + +} diff --git a/server/-product/production/HIRTV/jobs/steps/MXFCutterStep.java b/server/-product/production/HIRTV/jobs/steps/MXFCutterStep.java new file mode 100644 index 00000000..f1209d9d --- /dev/null +++ b/server/-product/production/HIRTV/jobs/steps/MXFCutterStep.java @@ -0,0 +1,137 @@ +package user.jobengine.server.steps; + +import org.apache.commons.lang.StringUtils; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.apache.logging.log4j.Marker; + +import user.commons.StoreUri; +import user.commons.configuration.SystemConfiguration; +import user.commons.remotestore.RemoteStoreProtocol; +//import user.jobengine.db.Media; +import user.jobengine.db.ArchivedMedia; +import user.jobengine.db.IItemManager; +import user.jobengine.db.Store; +import user.jobengine.server.IJobEngine; +import user.jobengine.server.IJobRuntime; + +public class MXFCutterStep extends JobStep { + private static final String TARGETNAMEPATTERN = "-ARCH-%s"; + private static final Logger logger = LogManager.getLogger(); + private static String NEXIO_HOST = SystemConfiguration.getInstance().value("services.nexio.host"); + private IItemManager manager; + private StoreUri tempTargetUri; + private StoreUri tempSourceUri; + private String sourceFileName; + private Marker marker; + private int nexioPort; + private String nexioUserName, nexioPassword; + + protected void checkTargetPath(String targetPath) { + if (StringUtils.isBlank(targetPath)) { + logger.error(marker, "A folyamat 'targetPath' bemeneti paramétere üres."); + throw new NullPointerException("System is not configured properly, 'targetPath' input parameter missing."); + } + } + + protected StoreUri createTargetUri(IItemManager manager, String targetPath) throws NullPointerException { + StoreUri result = null; + result = manager.createStoreUri(RemoteStoreProtocol.FTP, NEXIO_HOST); + result.setPortNumber(nexioPort); + result.setUserName(nexioUserName); + result.setPassword(nexioPassword); + + return result; + } + + @StepEntry + public Object[] execute(ArchivedMedia archivedMedia, String targetPath, String houseId, String successRecipient, int killDateDays, boolean useNexioTarget, + String nexioAgency, int nexioPort, String nexioUserName, String nexioPassword, IJobEngine jobEngine, IJobRuntime jobRuntime) throws Exception { + this.nexioPort = nexioPort; + this.nexioUserName = nexioUserName; + this.nexioPassword = nexioPassword; + marker = jobRuntime.getSessionMarker(); + +// if (!useNexioTarget) { +// String targetFileName = String.format(targetNamePattern, sourceFileName); +// MediaArea mediaArea = new MediaArea(Paths.get(targetPath)); +// try { +// mediaArea.process(); +// logger.info(getSessionMarker(), mediaArea.getInform().toPrettyString("")); +// } catch (Exception e) { +// logger.error(getSessionMarker(), "Can't analyze input {}, skipping. System message is: {}", e.getMessage()); +// } +// } + +// if (useNexioTarget && archivedMedia.getTcIn() != null && archivedMedia.getTcOut() != null) { +// setAndCheck(archivedMedia, houseId, targetPath, useNexioTarget, jobEngine); +// +// final IJobRuntime runtime = jobRuntime; +// sourceFileName = houseId + TARGETNAMEPATTERN; +// tempSourceUri.addProgressListener(new IProgressEventListener() { +// @Override +// public void progressChanged(ProgressEvent evt) { +// runtime.incrementProgress(evt.getProgress()); +// } +// }); +// +// RemoteFile result = tempSourceUri.transferFrom(tempTargetUri, sourceFileName, sourceFileName); +// +// EscortFiles.setNEXIOKillDate(killDateDays, houseId, nexioAgency, tempTargetUri); +// +// logger.info("A {} videó kivágva {}s - {}s", sourceFileName, archivedMedia.getTcIn(), archivedMedia.getTcOut()); +// } + + return null; + } + + // private String getSourceFileName(ArchivedMedia archivedMedia, Store + // store) { + // List mediaFiles = archivedMedia.getMedia().getMediaFiles(); + // if (mediaFiles == null) + // return null; + // for (MediaFile mediaFile : mediaFiles) { + // if (mediaFile.getStore().getId() == store.getId()) + // return mediaFile.getRelativePath(); + // } + // return null; + // } + + private void setAndCheck(ArchivedMedia archivedMedia, String houseId, String targetPath, boolean useNexioTarget, IJobEngine jobEngine) { + if (jobEngine == null) { + logger.error(marker, "A folyamatkezelő réteg nem elérhető."); + throw new NullPointerException("Internal error, missing JobEngine reference."); + } + manager = jobEngine.getItemManager(); + if (manager == null) { + logger.error(marker, "Az adatbáziskezelő réteg nem elérhető."); + throw new NullPointerException("Internal error, missing ItemManager reference."); + } + if (archivedMedia == null) { + logger.error(marker, "A folyamat 'mediaCubeMedia' bemeneti paramétere üres."); + throw new NullPointerException("System is not configured properly, 'mediaCubeMedia' input parameter missing."); + } + checkTargetPath(targetPath); + if (StringUtils.isBlank(houseId)) { + logger.error(marker, "A folyamat 'houseId' bemeneti paramétere üres."); + throw new NullPointerException("System is not configured properly, 'houseId' input parameter missing."); + } + Store tsmStore = manager.getSystemStore(false); + if (tsmStore == null) { + logger.error(marker, "A TSM rendszer beállítás nem elérhető."); + throw new NullPointerException("System is not configured properly, missing TSM Store."); + } + + tempSourceUri = manager.createStoreUri(RemoteStoreProtocol.LOCAL, targetPath); + if (tempSourceUri == null) { + logger.error(marker, "A TSM rendszer beállítás paraméterei nem elérhetőek."); + throw new NullPointerException("System is not configured properly, missing TSM StoreUri."); + } + tempTargetUri = createTargetUri(manager, targetPath); + // sourceFileName = getSourceFileName(archivedMedia, tsmStore); + if (sourceFileName == null) { + logger.error(marker, "Adatbázis bejegyzés hiba, a visszatöltendő fájl neve nem található."); + throw new NullPointerException("Database error, missing MediaFile 'relativePath'."); + } + } +} diff --git a/server/-product/production/HIRTV/jobs/steps/MediaToolStep.java b/server/-product/production/HIRTV/jobs/steps/MediaToolStep.java new file mode 100644 index 00000000..81bea7bd --- /dev/null +++ b/server/-product/production/HIRTV/jobs/steps/MediaToolStep.java @@ -0,0 +1,31 @@ +package user.jobengine.server.steps; + +import java.nio.file.Path; +import java.nio.file.Paths; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + +import user.commons.mediaarea.MediaArea; +import user.jobengine.db.Media; + +public class MediaToolStep extends JobStep { + private static final Logger logger = LogManager.getLogger(); + + @StepEntry + public Object[] execute(ArchiveItem archiveItem, Media mediaCubeMedia) throws Exception { + Path filePath = Paths.get(archiveItem.getMediaFile()); + if (filePath.toFile().length() != 0) { + MediaArea ma = new MediaArea(filePath); + ma.process(); + long frames = ma.getFrameCount(); + if (frames > 0) { + logger.info("Media {} length is {}", filePath, frames); + mediaCubeMedia.setLength(frames); + getManager().modify(mediaCubeMedia); + } + } + return null; + } + +} diff --git a/server/-product/production/HIRTV/jobs/steps/MetadataTransformStep.java b/server/-product/production/HIRTV/jobs/steps/MetadataTransformStep.java new file mode 100644 index 00000000..72a5e5f5 --- /dev/null +++ b/server/-product/production/HIRTV/jobs/steps/MetadataTransformStep.java @@ -0,0 +1,161 @@ +package user.jobengine.server.steps; + +import java.io.File; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.List; + +import org.apache.commons.io.FilenameUtils; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.apache.logging.log4j.Marker; + +import com.ibm.nosql.json.api.BasicDBList; + +import user.jobengine.db.Item; +import user.jobengine.db.ItemType; +import user.jobengine.db.Media; +import user.jobengine.server.steps.shared.EscortFiles; + +/** + * Itemek es mediak krealasa a ArchiveItem objektum alapjan. + * + * @author robi + */ +public class MetadataTransformStep extends JobStep { + private static final String CONFLICT = ".CONFLICT"; + private static final Logger logger = LogManager.getLogger(); + public static final String DEFAULT_MEDIATYPE = "Generic"; + private Marker marker; + + private void addTags(ArchiveItem archiveItem, Media mediaCubeMedia) { + BasicDBList tags = archiveItem.getTags(); + if (tags != null) { + for (Object tag : tags) { + + try { + String tagText = String.valueOf(tag); + getManager().addMediaTag(tagText, mediaCubeMedia.getId()); + System.out.println(); + + } catch (Exception e) { + logger.catching(e); + } + } + } + } + + private void moveToDuplicates(ArchiveItem archiveItem, String sourceFileName) { + try { + Path sourcePath = Paths.get(archiveItem.getMediaFile()); + Path conflictPath = Paths.get(sourcePath.getParent().toString(), CONFLICT); + EscortFiles.ensureUNCFolder(conflictPath); + Files.move(sourcePath, Paths.get(conflictPath.toString(), sourceFileName)); + Path metadataPath = EscortFiles.constructMetadataPath(sourcePath); + if (metadataPath.toFile().exists()) + Files.move(metadataPath, Paths.get(conflictPath.toString(), metadataPath.getFileName().toString())); + } catch (Exception e1) { + logger.catching(e1); + logger.error(marker, "Hiba az '{}' állomány mappába mozgatásakor. A rendszer üzenete: {}", CONFLICT, + e1.getMessage()); + } + } + + private Item createItem(ArchiveItem archiveItem) { + Item mediaCubeItem = getExistingItem(archiveItem.getItemHouseId(), archiveItem.getItemTitle()); + if (mediaCubeItem == null) + mediaCubeItem = getManager().createItem(DEFAULT_MEDIATYPE, archiveItem.getItemTitle(), + archiveItem.getItemDescription(), archiveItem.getItemHouseId()); + return mediaCubeItem; + } + + private Media createMedia(ArchiveItem archiveItem, Item mediaCubeItem, String mediaType) { + Media mediaCubeMedia; + mediaCubeMedia = getManager().createMedia(mediaType, archiveItem.getMediaTitle(), + archiveItem.getMediaDescription(), archiveItem.getMediaHouseId()); + mediaCubeMedia.setLength(archiveItem.getDuration()); + mediaCubeItem.appendMedia(mediaCubeMedia); + + return mediaCubeMedia; + } + + @StepEntry + public Object[] execute(ArchiveItem archiveItem) throws Exception { + marker = getSessionMarker(); + Media mediaCubeMedia = null; + try { + File sourceMediaFile = new File(archiveItem.getMediaFile()); + String sourceFileName = sourceMediaFile.getName(); + + getJobRuntime().setRelated(FilenameUtils.removeExtension(sourceFileName)); + + if (getManager().isMediaFileExists(sourceFileName)) { + cleanUpOnDuplicate(archiveItem, sourceFileName); + logger.warn(marker, "Az '" + sourceFileName + + "' állomány már megtalálható az archívumban, archiválása nem lehetséges."); + cancel(); + return null; + } + + Item mediaCubeItem = createItem(archiveItem); + setProgress(50); + String mediaType = getCreateType(archiveItem); + mediaCubeMedia = createMedia(archiveItem, mediaCubeItem, mediaType); + // ha itemid 0 akkor merge, egyebkent media insert + + if (mediaCubeItem.getId() == 0) + getManager().mergeItemStructure(mediaCubeItem); + else { + mediaCubeMedia.setItemId(mediaCubeItem.getId()); + mediaCubeMedia.add(); + } + + addTags(archiveItem, mediaCubeMedia); + + } catch (Exception e) { + logger.catching(e); + String fileName = new File(archiveItem.getMediaFile()).getName(); + logger.error(marker, + "Az '{}' állomány nem archiválható, mert a metaadat transzformáció sikertelen. A rendszer üzenete: {}", + fileName, e.getMessage()); + if (!archiveItem.removeCatchedFile()) + logger.error(marker, "Az '{}' állomány .catched jelző állománya nem törölhető.", fileName); + throw e; + } finally { + setProgress(100); + } + return new Object[] { mediaCubeMedia }; + } + + private void cleanUpOnDuplicate(ArchiveItem archiveItem, String sourceFileName) { + moveToDuplicates(archiveItem, sourceFileName); + archiveItem.removeCatchedFile(); + List killDateFiles = EscortFiles.getKillDateFiles(Paths.get(archiveItem.getMediaFile())); + EscortFiles.remove(killDateFiles); + } + + private String getCreateType(ArchiveItem archiveItem) { + String mediaType = archiveItem.getMediaType(); + if (mediaType == null || mediaType.length() == 0) + mediaType = DEFAULT_MEDIATYPE; + else { + ItemType mediaItemType = getManager().getItemType(mediaType); + if (mediaItemType == null) + getManager().createItemType(mediaType, mediaType).add(); + } + return mediaType; + } + + private Item getExistingItem(String itemHouseId, String itemTitle) { + Item[] result = new Item[] { null }; + String sql = String.format("select id from item where houseid='%s' and title='%s'", itemHouseId, itemTitle); + getManager().executeQuery(sql, rs -> { + long id = rs.getLong("id"); + result[0] = getManager().getItem(id); + return true; + }, null); + return result[0]; + } + +} diff --git a/server/-product/production/HIRTV/jobs/steps/OutputPathAndNameSelectorStep.java b/server/-product/production/HIRTV/jobs/steps/OutputPathAndNameSelectorStep.java new file mode 100644 index 00000000..52e30e91 --- /dev/null +++ b/server/-product/production/HIRTV/jobs/steps/OutputPathAndNameSelectorStep.java @@ -0,0 +1,161 @@ +package user.jobengine.server.steps; + +import java.io.IOException; +import java.nio.file.FileVisitResult; +import java.nio.file.FileVisitor; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.nio.file.SimpleFileVisitor; +import java.nio.file.attribute.BasicFileAttributes; + +import org.apache.commons.lang.StringUtils; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.apache.logging.log4j.Marker; + +import user.commons.configuration.SystemConfiguration; +import user.jobengine.db.ArchivedMedia; +import user.jobengine.server.IJobEngine; +import user.jobengine.server.IJobRuntime; +import user.jobengine.server.steps.shared.EscortFiles; +import user.jobengine.server.steps.shared.MetadataType; +import user.jobengine.server.steps.shared.MetadataTypeDetector; + +public class OutputPathAndNameSelectorStep extends JobStep { + + private static final String TARGETNAMEPATTERN = "-ARCH-%s"; + + private static final Logger logger = LogManager.getLogger(); + + private Marker marker; + + private String octopusAddress = SystemConfiguration.getInstance().value("services.octopus.api.address", ""); + + private void check(String localRetrievePath, String materialOutputFolder, String promoOutputFolder, String advertisementOutputFolder, + String octopusOutputFolder, String genericOutputFolder, String houseId, String targetPathType) { + if (StringUtils.isBlank(localRetrievePath)) { + logger.error(marker, "A folyamat 'localRetrievePath' bemeneti paramétere üres."); + throw new NullPointerException("System is not configured properly, 'localRetrievePath' input parameter missing."); + } + if (StringUtils.isBlank(materialOutputFolder)) { + logger.error(marker, "A folyamat 'materialOutputFolder' bemeneti paramétere üres."); + throw new NullPointerException("System is not configured properly, 'materialOutputFolder' input parameter missing."); + } + if (StringUtils.isBlank(promoOutputFolder)) { + logger.error(marker, "A folyamat 'promoOutputFolder' bemeneti paramétere üres."); + throw new NullPointerException("System is not configured properly, 'promoOutputFolder' input parameter missing."); + } + if (StringUtils.isBlank(advertisementOutputFolder)) { + logger.error(marker, "A folyamat 'advertisementOutputFolder' bemeneti paramétere üres."); + throw new NullPointerException("System is not configured properly, 'advertisementOutputFolder' input parameter missing."); + } + if (StringUtils.isBlank(octopusOutputFolder)) { + logger.error(marker, "A folyamat 'octopusOutputFolder' bemeneti paramétere üres."); + throw new NullPointerException("System is not configured properly, 'octopusOutputFolder' input parameter missing."); + } + if (StringUtils.isBlank(genericOutputFolder)) { + logger.error(marker, "A folyamat 'genericOutputFolder' bemeneti paramétere üres."); + throw new NullPointerException("System is not configured properly, 'genericOutputFolder' input parameter missing."); + } + if (StringUtils.isBlank(houseId)) { + logger.error(marker, "A folyamat 'houseId' bemeneti paramétere üres."); + throw new NullPointerException("System is not configured properly, 'houseId' input parameter missing."); + } + if (StringUtils.isBlank(targetPathType)) { + logger.error(marker, "A folyamat 'targetPathType' bemeneti paramétere üres."); + throw new NullPointerException("System is not configured properly, 'targetPathType' input parameter missing."); + } + } + + @StepEntry + public Object[] execute(String localRetrievePath, String materialOutputFolder, String promoOutputFolder, String advertisementOutputFolder, + String octopusOutputFolder, String genericOutputFolder, String onlineOutputFolder, String houseId, String targetPathType, + ArchivedMedia archivedMedia, IJobEngine jobEngine, IJobRuntime jobRuntime) throws Exception { + marker = jobRuntime.getSessionMarker(); + check(localRetrievePath, materialOutputFolder, promoOutputFolder, advertisementOutputFolder, octopusOutputFolder, genericOutputFolder, houseId, + targetPathType); + Object[] result = null; + switch (Integer.parseInt(targetPathType)) { + case 0: + String outputFolder = getFolderById(materialOutputFolder, promoOutputFolder, advertisementOutputFolder, octopusOutputFolder, genericOutputFolder, + houseId, archivedMedia); + result = localTargetInit(localRetrievePath, outputFolder, houseId, jobRuntime); + break; + case 1: + result = localTargetInit(localRetrievePath, onlineOutputFolder, houseId, jobRuntime); + break; + case 2: + + if (archivedMedia.getTcIn() != null && archivedMedia.getTcOut() != null) + result = new Object[] { genericOutputFolder, houseId, true }; + else + result = new Object[] { null, houseId, true }; + break; + } + return result; + } + + private String getFolderById(String materialOutputFolder, String promoOutputFolder, String advertisementOutputFolder, String octopusOutputFolder, + String genericOutputFolder, String houseId, ArchivedMedia archivedMedia) throws Exception { + String id = houseId.toUpperCase(); + MetadataType mdType = MetadataTypeDetector.GuessMetadataType(id); + String result = null; + + // a groovy nem latja enumnak, hanem az objektum tulajdonsaganak + switch (mdType.toString()) { + case "OctopusPlaceholder": + case "OctopusStory": + // 220429 MV-ben nincs Octopus + if (StringUtils.isBlank(octopusAddress)) + result = genericOutputFolder; + else + result = octopusOutputFolder; + break; + case "TrafficMaterial": + result = materialOutputFolder; + break; + case "TrafficPromo": + result = promoOutputFolder; + break; + case "TrafficAD": + result = advertisementOutputFolder; + break; + case "Generic": + result = genericOutputFolder; + break; + } + return result; + } + + private String getPossiblePath(String id, Path targetPath) throws IOException { + String[] result = new String[] { targetPath.toString() }; + FileVisitor matcherVisitor = new SimpleFileVisitor() { + @Override + public FileVisitResult preVisitDirectory(Path dir, BasicFileAttributes attrs) throws IOException { + String dirName = dir.getFileName().toString(); + if (dirName.startsWith(id + "-") || dirName.equals(id)) { + result[0] = dir.toString(); + return FileVisitResult.TERMINATE; + } + return FileVisitResult.CONTINUE; + } + }; + Files.walkFileTree(targetPath.getParent(), matcherVisitor); + return result[0]; + } + + private Object[] localTargetInit(String localRetrievePath, String outputFolder, String houseId, IJobRuntime jobRuntime) throws IOException { + String id = houseId.toUpperCase(); + String targetPath = getPossiblePath(id, Paths.get(localRetrievePath, outputFolder, id)).toString(); + String targetNamePattern = houseId + TARGETNAMEPATTERN; + try { + EscortFiles.ensureUNCFolder(Paths.get(targetPath)); + } catch (Exception e) { + logger.error(jobRuntime.getSessionMarker(), "A cél mappa '{}' nem létezik és nem hozható létre. A rendszer hibaüzenete: {}", targetPath, + e.getMessage()); + throw e; + } + return new Object[] { targetPath, targetNamePattern, false }; + } +} diff --git a/server/-product/production/HIRTV/jobs/steps/ProjectCleanupMountedLocationStep.java b/server/-product/production/HIRTV/jobs/steps/ProjectCleanupMountedLocationStep.java new file mode 100644 index 00000000..06d1cc65 --- /dev/null +++ b/server/-product/production/HIRTV/jobs/steps/ProjectCleanupMountedLocationStep.java @@ -0,0 +1,225 @@ +package user.jobengine.server.steps; + +import java.io.File; +import java.io.IOException; +import java.nio.file.DirectoryStream; +import java.nio.file.FileVisitResult; +import java.nio.file.FileVisitor; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.nio.file.SimpleFileVisitor; +import java.nio.file.attribute.BasicFileAttributes; +import java.text.ParseException; +import java.text.SimpleDateFormat; +import java.util.ArrayList; +import java.util.Collections; +import java.util.Date; +import java.util.List; + +import org.apache.commons.io.FileUtils; +import org.apache.commons.lang.StringUtils; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.apache.logging.log4j.Marker; + +import user.jobengine.server.IJobEngine; +import user.jobengine.server.IJobRuntime; +import user.jobengine.server.steps.shared.MetadataType; +import user.jobengine.server.steps.shared.MetadataTypeDetector; + +public class ProjectCleanupMountedLocationStep extends JobStep implements FileVisitor { + + private static final Logger logger = LogManager.getLogger(); + private static final String DATEFORMAT = "yyyyMMdd"; + private static final String DOT = "."; + private static final String STATUSFOLDER = ".STATUS"; + private static final String KILLDATEEXT = ".killdate"; + private Marker marker; + final int[] allCount = new int[] { 0 }; + final int[] currentCount = new int[] { 0 }; + private Path sourcePath; + private SimpleDateFormat dateFormat; + + private Date checkExpiration(List killDateFiles) { + Date killDate = null; + for (Path killDateFile : killDateFiles) { + Date currentKillDate = getKillDate(killDateFile); + if (currentKillDate == null) + continue; + if ((killDate != null && currentKillDate.after(killDate)) || killDate == null) + killDate = currentKillDate; + } + return new Date().after(killDate) ? killDate : null; + } + + @StepEntry + public Object[] execute(String sourceFolder, IJobEngine jobEngine, IJobRuntime jobRuntime) throws Exception { + marker = jobRuntime.getSessionMarker(); + sourcePath = Paths.get(sourceFolder); + DirectoryStream directoryStream = null; + if (StringUtils.isBlank(sourcePath.toString())) { + logger.error(marker, "A folyamat 'sourcePath' bemeneti paramétere üres."); + throw new NullPointerException("System is not configured properly, 'sourceFolder' input parameter missing."); + } + + if (!sourcePath.toFile().exists() || !sourcePath.toFile().isDirectory()) { + logger.error(marker, "A(z) {} mappa nem létezik.", sourceFolder); + throw new NullPointerException(String.format("Directory %s does not exist.", sourceFolder)); + } + try { + setProgress(1); + dateFormat = new SimpleDateFormat(DATEFORMAT); + Files.walkFileTree(sourcePath, new SimpleFileVisitor() { + @Override + public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) throws IOException { + allCount[0]++; + return super.visitFile(file, attrs); + } + }); + Files.walkFileTree(sourcePath, this); + } catch (Exception e) { + logger.catching(e); + logger.error(marker, "Az '{}' mappa elérése sikertelen. A rendszer hibaüzenete: {}", sourcePath, e.getMessage()); + throw e; + } finally { + if (directoryStream != null) { + try { + directoryStream.close(); + } catch (IOException e) { + } + } + } + return null; + } + + private Date getKillDate(Path killDateFile) { + String fileName = killDateFile.getFileName().toString(); + int end = fileName.lastIndexOf(DOT); + if (end < 1) + return null; + int start = fileName.lastIndexOf(DOT, end - 1); + if (start < 0) + return null; + String strKillDate = fileName.substring(start + 1, end); + Date result = null; + if (StringUtils.isNumeric(strKillDate)) { + try { + result = dateFormat.parse(strKillDate); + } catch (ParseException e) { + logger.error(marker, "A {} fájl 'killdate' állománya hibás formátumú, a {} karaktersorozat nem konvertálható dátummá.", fileName, strKillDate); + return null; + } + } else + logger.error(marker, "A {} fájl 'killdate' állománya hibás formátumú, az dátum helyett ez áll: '{}'.", fileName, strKillDate); + return result; + } + + private List getKillDateFiles(Path filePath) { + String killDateFilePattern = String.format("%s.*%s", filePath.getFileName().toString(), KILLDATEEXT); + List result = new ArrayList<>(); + Path statusPath = null; + try { + statusPath = Paths.get(filePath.getParent().toString(), STATUSFOLDER); + } catch (Exception e) { + logger.catching(e); + return null; + } + File statusPathFile = statusPath.toFile(); + if (statusPathFile.exists() && statusPathFile.isDirectory()) { + try (DirectoryStream stream = Files.newDirectoryStream(statusPath, killDateFilePattern)) { + stream.forEach(p -> result.add(p)); + } catch (Exception e) { + logger.catching(e); + } + } + Collections.sort(result); + return result; + } + + private Path getProjectRootPath(Path filePath) { + Path result = null; + if (filePath.getNameCount() > sourcePath.getNameCount()) { + String dir = filePath.getName(sourcePath.getNameCount()).toString(); + String[] tokens = dir.split("-"); + if (tokens.length != 0 && MetadataTypeDetector.GuessMetadataType(tokens[0]) == MetadataType.TrafficPromo) + result = Paths.get(sourcePath.toAbsolutePath().toString(), dir); + } + return result; + } + + @Override + public FileVisitResult postVisitDirectory(Path dir, IOException exc) throws IOException { + return FileVisitResult.CONTINUE; + } + + @Override + public FileVisitResult preVisitDirectory(Path dir, BasicFileAttributes attrs) throws IOException { + if (dir.equals(sourcePath)) + return FileVisitResult.CONTINUE; + + Path parent = getProjectRootPath(dir); + if (parent == null) { + System.out.println("Skipping " + dir); + return FileVisitResult.SKIP_SUBTREE; + } + + return FileVisitResult.CONTINUE; + } + + private boolean processPathItem(Path filePath) { + currentCount[0]++; + int progress = currentCount[0] * 100 / allCount[0]; + setProgress(progress); + + if (filePath.toFile().isDirectory() || !filePath.toString().toUpperCase().endsWith(".EZP")) + return false; + + // level check + if (filePath.getNameCount() != sourcePath.getNameCount() + 3) { + // logger.warn(marker, "A {} fájl elérési útja a várttól eltérő.", filePath); + return false; + } + + List killDateFiles = getKillDateFiles(filePath); + if (killDateFiles == null || killDateFiles.size() == 0) { + logger.warn(marker, "A {} fájlhoz nem található 'killdate' állomány.", filePath); + return false; + } + + if (killDateFiles.size() != 1) + logger.warn(marker, "A {} fájlhoz több 'killdate' állomány található, a legújabb dátum határozza meg a törlés időpontját.", filePath); + + Date killDate = checkExpiration(killDateFiles); + if (killDate == null) + return false; + + Path parent = getProjectRootPath(filePath); + if (parent != null) { + Path pathToDelete = Paths.get(sourcePath.toAbsolutePath().toString(), filePath.getName(sourcePath.getNameCount()).toString()); + try { + FileUtils.deleteDirectory(pathToDelete.toFile()); + logger.info(marker, "A {} mappa sikeresen törlődött. Lejárat: {}", pathToDelete, dateFormat.format(killDate)); + } catch (Exception e) { + logger.error(marker, "A {} mappa törlése nem lehetséges, a rendszer üzenete: {}", pathToDelete, e.getMessage()); + logger.error(e); + } + } + // else + // logger.warn(marker, "A {} fájlhoz nem határozható meg a törlendő + // szülőkönyvtár.", filePath); + return true; + } + + @Override + public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) throws IOException { + processPathItem(file); + return FileVisitResult.CONTINUE; + } + + @Override + public FileVisitResult visitFileFailed(Path file, IOException e) throws IOException { + logger.error(marker, "A {} fájl nem érhető el. A rendszer hibaüzenete: {}", file.toString(), e.getMessage()); + return FileVisitResult.CONTINUE; + } +} diff --git a/server/-product/production/HIRTV/jobs/steps/RecordingsArchiveItemBuilderStep.java b/server/-product/production/HIRTV/jobs/steps/RecordingsArchiveItemBuilderStep.java new file mode 100644 index 00000000..cb3f5d58 --- /dev/null +++ b/server/-product/production/HIRTV/jobs/steps/RecordingsArchiveItemBuilderStep.java @@ -0,0 +1,322 @@ +package user.jobengine.server.steps; + +import java.io.File; +import java.io.IOException; +import java.nio.file.FileVisitResult; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.nio.file.SimpleFileVisitor; +import java.nio.file.attribute.BasicFileAttributes; +import java.text.ParseException; +import java.text.SimpleDateFormat; +import java.util.Calendar; +import java.util.Date; +import java.util.List; + +import org.apache.commons.lang.StringUtils; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.apache.logging.log4j.Marker; + +import com.ibm.nosql.json.api.BasicDBObject; +import com.ibm.nosql.json.api.DB; +import com.ibm.nosql.json.api.DBCollection; +import com.ibm.nosql.json.api.DBObject; + +import user.commons.CalendarUtils; +import user.commons.nosql.NoSQLUtils; +import user.commons.octopus.IOctopusAPI; +import user.commons.octopus.OctopusAPI; +import user.jobengine.server.IJobEngine; +import user.jobengine.server.IJobRuntime; +import user.jobengine.server.steps.shared.EscortFiles; + +public class RecordingsArchiveItemBuilderStep extends JobStep { + private static final String MEDIATYPE = "Visszarögzített"; + private static final Logger logger = LogManager.getLogger(); + private static final String STATUSFOLDER = ".STATUS"; + private static final String LXFEXT = ".lxf"; + private static final String CATCHEDEXT = ".catched"; + private static final SimpleDateFormat startTimeformat = new SimpleDateFormat("HHmm"); + private static final SimpleDateFormat startDateformat = new SimpleDateFormat("yyMMdd"); + private static final String SCHEDULED_FORMAT = "yyyy.MM.dd HH:mm"; + + private Marker marker; + private DBCollection existingRecordings; + + private ArchiveItem createArchiveItem(Path mediaFilePath, Path catchedFilePath) { + ArchiveItem result = null; + try { + + Date recordDate = startDateformat.parse(mediaFilePath.getParent().toFile().getName()); + String clipName = mediaFilePath.toFile().getName(); + Date scheduledStart = getScheduledStart(clipName, recordDate); + IOctopusAPI octopusAPI = new OctopusAPI(); + + DBObject rundown = octopusAPI.getRundown(scheduledStart); + if (rundown == null) { + logger.error(marker, "A '{}' anyaghoz nem található tükör '{}' kezdéssel, ezért nem archiválható.", + clipName, scheduledStart); + return null; + } + result = processRundow(octopusAPI, rundown); + if (result == null) + return null; + + if (clipName.startsWith("1900")) { + Calendar cal = CalendarUtils.createCalendar(scheduledStart); + cal.add(Calendar.MINUTE, 5); + rundown = octopusAPI.getRundown(cal.getTime()); + if (rundown == null) { + logger.error(marker, "A '{}' anyaghoz nem található tükör '{}' kezdéssel, ezért nem archiválható.", + clipName, scheduledStart); + return null; + } + ArchiveItem item2 = processRundow(octopusAPI, rundown); + if (item2 == null) + return null; + + result.setItemTitle(result.getItemTitle() + " + NAPIAKT"); + result.setMediaDescription(result.getMediaDescription() + "\r\n\r\n****** NAPIAKT ******\r\n\r\n" + + item2.getMediaDescription()); + } + + result.setMediaTitle(clipName); + result.setMediaType(MEDIATYPE); + result.setMediaFile(mediaFilePath.toString()); + result.setCatchedFile(catchedFilePath.toString()); + } catch (Exception e) { + logger.catching(e); + logger.error(getJobRuntime().getSessionMarker(), "A metaadat nem elérhető. A rendszer üzenete: {}", + e.getMessage()); + return null; + } + + return result; + } + + private void createCatchedFile(Path catchedFilePath) throws IOException { + try { + EscortFiles.ensureUNCFolder(catchedFilePath.getParent()); + Files.createFile(catchedFilePath); + } catch (IOException e) { + logger.catching(e); + logger.error(marker, "A '{}' jelzőfájl nem hozható létre. A rendszer üzenete: {}", catchedFilePath, + e.getMessage()); + throw e; + } + } + + @StepEntry + public Object[] execute(String sourcePath, IJobEngine jobEngine, IJobRuntime jobRuntime) throws Exception { + final ArchiveItem[] archiveItems = { null }; + DB db = NoSQLUtils.getNoSQLDB(); + existingRecordings = db.getCollection("tmp_existing_recordings"); + marker = getJobRuntime().getSessionMarker(); + try { + Files.walkFileTree(Paths.get(sourcePath), new SimpleFileVisitor() { + + @Override + public FileVisitResult preVisitDirectory(Path dir, BasicFileAttributes attrs) throws IOException { + FileVisitResult result = FileVisitResult.SKIP_SUBTREE; + + if (dir.equals(Paths.get(sourcePath)) || "2017".equals(dir.toFile().getName().toUpperCase()) + || "2018".equals(dir.toFile().getName().toUpperCase())) + result = FileVisitResult.CONTINUE; + else { + if ("2017".equals(dir.getParent().toFile().getName().toUpperCase()) + || "2018".equals(dir.getParent().toFile().getName().toUpperCase())) { + try { + startDateformat.parse(dir.toFile().getName()); + result = FileVisitResult.CONTINUE; + } catch (ParseException e) { + } + } + } + + if (result == FileVisitResult.CONTINUE) + logger.info(dir); + return result; + } + + @Override + public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) throws IOException { + FileVisitResult result = FileVisitResult.TERMINATE; + ArchiveItem item = null; + try { + item = processPathItem(file); + } catch (IOException e) { + logger.catching(e); + logger.error(marker, "Az '{}' állomány feldolgozása sikertelen. A rendszer hibaüzenete: {}", + file, e.getMessage()); + throw e; + } + if (item == null) { + result = FileVisitResult.CONTINUE; + } else { + logger.info(file); + archiveItems[0] = item; + } + return result; + } + + }); + + } catch (Exception e) { + logger.catching(e); + logger.error(marker, "Az '{}' mappa elérése sikertelen. A rendszer hibaüzenete: {}", sourcePath, + e.getMessage()); + throw e; + } finally { + } + ArchiveItem archiveItem = archiveItems[0]; + String targetFileName = null; + + if (archiveItem == null || archiveItem.getMediaFile() == null) { + logger.warn(marker, "Az archiváló folyamat nem talált új anyagot."); + throw new Exception("No media to archive"); + } else { + String mediaFile = archiveItem.getMediaFile(); + String name = new File(mediaFile).getName(); + int extPos = name.toLowerCase().lastIndexOf(LXFEXT); + targetFileName = String.format("20%s-%s", Paths.get(mediaFile).getParent().getFileName(), + name.substring(0, extPos)); + if (targetFileName.length() > 32) { + targetFileName = targetFileName.substring(0, 28) + "_PGM"; + } + logger.info(marker, "Az archiváló folyamat az '{}' anyagot archiválja.", mediaFile); + } + + return new Object[] { archiveItem, targetFileName }; + } + + private Date getScheduledStart(String clipName, Date recordDate) { + + if (StringUtils.isBlank(clipName)) { + logger.warn(marker, "A fájlnak nincs neve, ezért nem archiválható."); + return null; + } + if (recordDate == null) { + logger.warn(marker, "Az '{}' fájl rögzítésének ideje nem meghatározható, ezért nem archiválható.", + clipName); + return null; + } + + Date timePart = null; + try { + String clipNameTime = clipName.split("_")[0]; + timePart = startTimeformat.parse(clipNameTime); + } catch (ParseException e) { + logger.warn(marker, "A '{}' fájl neve nem időbélyeggel kezdődik, ezért nem archiválható.", clipName); + return null; + } + return CalendarUtils.createCalendar(CalendarUtils.createCalendar(recordDate), timePart).getTime(); + } + + private ArchiveItem processPathItem(Path mediaFilePath) throws IOException { + File mediaFile = mediaFilePath.toFile(); + + Path dotStorePath = Paths.get(mediaFilePath.getParent().toString(), STATUSFOLDER); + Path catchedFilePath = Paths.get(dotStorePath.toString(), mediaFile.getName() + CATCHEDEXT); + File catchedFile = catchedFilePath.toFile(); + if (catchedFile.exists()) { + // logger.info("'{}' file is already catched", mediaFilePath); + return null; + } + createCatchedFile(catchedFilePath); + if (!catchedFile.exists()) { + logger.warn("'{}' catchfile does not exist.", catchedFilePath); + return null; + } + + if (mediaFile.isDirectory() || !mediaFile.getName().toLowerCase().endsWith(LXFEXT.toLowerCase()) + || mediaFilePath.getParent().toFile().getName().length() != 6 + || !mediaFile.getName().toLowerCase().contains("_pgm_")) { + logger.info("Skipping '{}'", mediaFilePath); + return null; + } + + ArchiveItem archiveItem = createArchiveItem(mediaFilePath, catchedFilePath); + + if (archiveItem == null) { + logger.warn("'{}' has no metadata specified.", mediaFilePath); + return null; + } + + DBObject existingRecording = existingRecordings + .findOne(new BasicDBObject("title", archiveItem.getItemTitle().substring(0, 16))); + + if (existingRecording != null) { + logger.warn("'{}' already archived, skipping.", archiveItem.getItemTitle()); + return null; + } + + if (StringUtils.isBlank(archiveItem.getItemHouseId())) { + logger.warn("'{}' has no Item HouseID specified in metadata.", mediaFilePath); + return null; + } + + if (StringUtils.isBlank(archiveItem.getItemTitle())) { + logger.warn("'{}' has no Item Title specified in metadata.", mediaFilePath); + return null; + } + + if (StringUtils.isBlank(archiveItem.getMediaHouseId())) { + logger.warn("'{}' has no Media HouseID specified in metadata.", mediaFilePath); + return null; + } + + if (StringUtils.isBlank(archiveItem.getMediaTitle())) { + logger.warn("'{}' has no Media Title specified in metadata.", mediaFilePath); + return null; + } + return archiveItem; + } + + private ArchiveItem processRundow(IOctopusAPI octopusAPI, DBObject r) throws Exception { + BasicDBObject rundown = (BasicDBObject) r; + long rundownID = rundown.getLong(IOctopusAPI.ID); + logger.info("Processing rundown {} {}", rundownID, rundown.getString(IOctopusAPI.NAME)); + + List stories = octopusAPI.getRundownFullStories(rundownID); + if (stories == null) + return null; + + String name = NoSQLUtils.asString(NoSQLUtils.asDBObject(rundown, IOctopusAPI.RUNDOWN_TYPE), IOctopusAPI.NAME); + if (StringUtils.isBlank(name)) + return null; + String channel = NoSQLUtils.asString(NoSQLUtils.asDBObject(rundown, IOctopusAPI.CHANNEL), IOctopusAPI.NAME); + Date scheduledStart = rundown.getDate(IOctopusAPI.SCHEDULED_START); + if (scheduledStart == null) + return null; + + ArchiveItem result = new ArchiveItem(); + result.setItemHouseId(String.valueOf(rundownID)); + String start = CalendarUtils.toString(CalendarUtils.createCalendar(scheduledStart), SCHEDULED_FORMAT); + result.setItemTitle(String.format("%s %s %s", start, name, channel)); + + StringBuilder sb = new StringBuilder(); + for (DBObject s : stories) { + BasicDBObject story = (BasicDBObject) s; + + sb.append("*** "); + sb.append(story.getString(IOctopusAPI.PARENT_STORY_ID)); + sb.append(" [" + story.getString(IOctopusAPI.FORMAT) + "] "); + sb.append(story.getString(IOctopusAPI.NAME)); + sb.append(" ***"); + sb.append("\r\n"); + String content = story.getString(IOctopusAPI.SCRIPT_CONTENT); + if (content != null) { + content = content.replace("\r\n\r\n\r\n\r\n", "\r\n"); + content = content.replace("\r\n\r\n\r\n", "\r\n"); + content = content.replace("\r\n\r\n", "\r\n"); + sb.append(content); + sb.append("\r\n"); + } + } + result.setMediaHouseId(result.getItemHouseId()); + result.setMediaDescription(sb.toString()); + return result; + } +} diff --git a/server/-product/production/HIRTV/jobs/steps/SyncOCTOPUSDataStep.java b/server/-product/production/HIRTV/jobs/steps/SyncOCTOPUSDataStep.java new file mode 100644 index 00000000..98136041 --- /dev/null +++ b/server/-product/production/HIRTV/jobs/steps/SyncOCTOPUSDataStep.java @@ -0,0 +1,32 @@ +package user.jobengine.server.steps; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + +import user.commons.remotestore.IProgressEventListener; +import user.jobengine.server.steps.shared.OctopusDataMiner; + +public class SyncOCTOPUSDataStep extends JobStep { + private static final Logger logger = LogManager.getLogger(); + private IProgressEventListener progressListener; + + @StepEntry + public Object[] execute(boolean includeArchived, int maxPastDays, String address, String user, String pwd) throws Exception { + OctopusDataMiner dataMiner = null; + try { + dataMiner = new OctopusDataMiner(address, user, pwd, maxPastDays); + dataMiner.addProgressListener(e -> { + setProgress(e.getProgress()); + }); + dataMiner.execute(includeArchived); + } catch (Exception e) { + logger.error(getMarker(), "Általános folyamat hiba. A rendszer hibaüzenete: {}", e.getMessage()); + throw e; + } finally { + if (dataMiner != null) + dataMiner.removeProgressListener(progressListener); + } + return null; + } + +} diff --git a/server/-product/production/HIRTV/jobs/steps/TSMBackupStep.java b/server/-product/production/HIRTV/jobs/steps/TSMBackupStep.java new file mode 100644 index 00000000..a71ce5de --- /dev/null +++ b/server/-product/production/HIRTV/jobs/steps/TSMBackupStep.java @@ -0,0 +1,231 @@ +package user.jobengine.server.steps; + +import java.io.File; +import java.io.IOException; +import java.sql.Timestamp; +import java.util.Date; +import java.util.List; + +import org.apache.commons.lang.RandomStringUtils; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.apache.logging.log4j.Marker; +import org.apache.logging.log4j.message.Message; +import org.apache.logging.log4j.message.ParameterizedMessage; + +import user.commons.RemoteFile; +import user.commons.StoreUri; +import user.commons.configuration.SystemConfiguration; +import user.commons.mediatool.Timecode; +import user.commons.mediatool.Timecode.Type; +import user.commons.remotestore.IProgressEventListener; +import user.commons.remotestore.IStatusEventListener; +import user.commons.remotestore.ProgressEvent; +import user.commons.remotestore.RemoteStoreProtocol; +import user.commons.remotestore.StatusEvent; +import user.jobengine.db.FileType; +import user.jobengine.db.IItemManager; +import user.jobengine.db.Media; +import user.jobengine.db.MediaFile; +import user.jobengine.db.Store; +import user.jobengine.server.IJobEngine; +import user.jobengine.server.IJobRuntime; +import user.jobengine.server.JobEngineException; +import user.jobengine.server.steps.shared.EscortFiles; +import user.jobengine.server.steps.shared.ItemManagerExtensions; +import user.jobengine.server.steps.shared.MetadataSaver; + +public class TSMBackupStep extends JobStep { + private static final String MXFEXT = ".MXF"; + private static final Logger logger = LogManager.getLogger(); + private static boolean RANDOMIZE_ARCHIVES = SystemConfiguration.getInstance().value("tsm.randomize-archives"); + private IItemManager manager; + private File sourceMediaFile; + private Store tsmStore; + private StoreUri targetUri; + private FileType fileType; + private Marker marker; + + @StepEntry + public Object[] execute(ArchiveItem archiveItem, Media mediaCubeMedia, int killDateDays) throws Exception { + marker = getSessionMarker(); + + File sourceMediaFile = new File(archiveItem.getMediaFile()); + String sourceFileName = sourceMediaFile.getName(); + long fileSize = sourceMediaFile.length(); + + try { + Timecode timecode = new Timecode(mediaCubeMedia.getLength(), Type.PAL); + String details = String.format("%s (%s, %d bytes)", sourceFileName, timecode.toString(), fileSize); + logger.info(marker, details); + getJobRuntime().setDescription(details); + } catch (Exception e) { + String details = String.format("%s (%d bytes)", sourceFileName, fileSize); + getJobRuntime().setDescription(details); + } + Timecode timecode = new Timecode(mediaCubeMedia.getLength(), Type.PAL); + String details = String.format("%s (%s, %d bytes)", sourceFileName, timecode.toString(), fileSize); + getJobRuntime().setDescription(details); + try { + + setAndCheck(archiveItem, mediaCubeMedia, getEngine()); + + // TODO mxf helyett az osszes kiterjesztest!!!!! + // A dupla ellenorzes a napon beluli ismetlesek miatt kell + long existingMediaId = archiveItem.getExistingMediaId(); + + if (fileSize == 0 && existingMediaId == 0) { + existingMediaId = ItemManagerExtensions.getExistingRundownMedia(manager, sourceFileName.replace(MXFEXT, "")); + if (existingMediaId == 0) + existingMediaId = -1; + } + + if (existingMediaId == 0) + existingMediaId = ItemManagerExtensions.getExistingRundownMedia(manager, sourceFileName.replace(MXFEXT, "")); + String targetFileName; + if (RANDOMIZE_ARCHIVES) { + // a-z, A-Z, 0-9. For example: WRMcpIk7, s57JwCVA + // veletlenszeru neveket adunk! + targetFileName = String.format("%s-%s", RandomStringUtils.randomAlphanumeric(8), sourceFileName); + } else + targetFileName = sourceFileName; + + if (existingMediaId == 0) { + StoreUri sourceUri = manager.createStoreUri(RemoteStoreProtocol.LOCAL, sourceMediaFile.getParent().toString()); + + final IJobRuntime runtime = getJobRuntime(); + sourceUri.addProgressListener(new IProgressEventListener() { + @Override + public void progressChanged(ProgressEvent evt) { + runtime.incrementProgress(evt.getProgress()); + } + }); + sourceUri.addStatusListener(new IStatusEventListener() { + @Override + public void statusChanged(StatusEvent evt) { + evt.setCancel(!canContinue()); + } + }); + + RemoteFile remoteFile = sourceUri.transferFrom(targetUri, sourceFileName, targetFileName); + + if (RemoteStoreProtocol.LOCAL.equals(sourceUri.getProtocol())) { + MetadataSaver.saveMetadata(archiveItem.getMediaFile()); + } + + } + + if (existingMediaId > 0) + logger.info(marker, "Az '{}' TSM mentése nem szükséges, mert már megtalálható az archívumban.", sourceFileName); + + // Fel kell szabadítani, hogy a kovetkezo archivalaskor is nekifusson + if (existingMediaId == -1) { + logger.info(marker, "Az '{}' mentése jelenleg nem lehetséges, mert a szükséges metaadat még nem található meg az archívumban.", sourceFileName); + if (!archiveItem.removeCatchedFile()) + logger.error(marker, + "Az '{}' állomány .catched jelző állománya nem törölhető. Az újabb archiválási kísérlethez annak kézi eltávolítása szükséges!", + sourceMediaFile.getName()); + mediaCubeMedia.remove(); + } else { + saveMetadata(mediaCubeMedia, sourceMediaFile, targetFileName, existingMediaId, fileSize, archiveItem.isDisableProxy()); + logger.info(marker, "Az '{}' archiválása sikeres.", sourceFileName); + if (killDateDays != 0) + EscortFiles.createUNCKillDate(sourceMediaFile.getParent(), sourceFileName, killDateDays, marker); + } + + } catch (Exception e) { + logger.catching(e); + Message m = new ParameterizedMessage("Az '{}' állomány archiválása sikertelen. A rendszer hibaüzenete: {}", details, e.getMessage()); + logger.error(marker, m); + if (!archiveItem.removeCatchedFile()) + logger.error(marker, + "Az '{}' állomány .catched jelző állománya nem törölhető. Az újabb archiválási kísérlethez annak kézi eltávolítása szükséges!", + sourceMediaFile.getName()); + throw new Exception(m.getFormattedMessage()); + } + return null; + } + + private void saveMetadata(Media mediaCubeMedia, File sourceFile, String targetFileName, long existingMediaId, long fileSize, boolean disableProxy) { + + if (existingMediaId == 0) { + MediaFile mf = manager.createMediaFile(targetFileName, fileType, tsmStore, mediaCubeMedia); + mf.setHouseId(sourceFile.getName()); + mf.setFileSize(fileSize); + // 210617 proxy keszites tiltasa + mf.setDisableProxy(disableProxy); + mf.add(); + } else { + Media existingMedia = manager.getMedia(existingMediaId); + List mediaFiles = existingMedia.getMediaFiles(); + if (mediaFiles != null) { + for (MediaFile mf : mediaFiles) { + mf.setPersister(manager); + mf.setId(0); + mf.setMedia(mediaCubeMedia); + // mivel itt masolat keszul, nem allitunk at semmit + // mf.setFileSize(fileSize); + // mf.setDisableProxy(disableProxy); + mf.add(); + } + } + } + mediaCubeMedia.setPersister(manager); + + // 210614 megis maradjon az aktualis idopont + mediaCubeMedia.setArchived(new Timestamp(new Date().getTime())); + + /* + * try { BasicFileAttributes attr = Files.readAttributes(sourceFile.toPath(), + * BasicFileAttributes.class); mediaCubeMedia.setArchived(new + * Timestamp(attr.creationTime().toMillis())); } catch (IOException e) { + * logger.catching(e); } + */ + mediaCubeMedia.modify(); + } + + private void setAndCheck(ArchiveItem archiveItem, Media mediaCubeMedia, IJobEngine jobEngine) throws JobEngineException, IOException { + if (jobEngine == null) { + logger.error(marker, "Az folyamatkezelő réteg nem elérhető."); + throw new NullPointerException("Internal error, missing JobEngine reference."); + } + manager = jobEngine.getItemManager(); + if (manager == null) { + logger.error(marker, "Az adatbáziskezelő réteg nem elérhető."); + throw new NullPointerException("Internal error, missing ItemManager reference."); + } + if (archiveItem == null) { + logger.error(marker, "A folyamat 'archiveItem' bemeneti paramétere üres."); + throw new NullPointerException("System is not configured properly, missing 'archiveItem' input parameter."); + } + sourceMediaFile = new File(archiveItem.getMediaFile()); + if (sourceMediaFile == null) { + logger.error(marker, "A folyamat 'archiveItem' bemeneti paraméter 'mediaFile' értéke üres."); + throw new NullPointerException("System is not configured properly, missing 'mediaFile' value in 'archiveItem' input parameter."); + } + if (!sourceMediaFile.exists()) { + logger.error(marker, "A(z) {} állomány nem létezik vagy nem érhető el.", sourceMediaFile.getName()); + throw new IOException(String.format("Input file {} does not exist or unreachable.", sourceMediaFile.getName())); + } + tsmStore = manager.getSystemStore(false); + if (tsmStore == null) { + logger.error(marker, "A TSM rendszer beállítás nem elérhető."); + throw new NullPointerException("System is not configured properly, missing TSM Store."); + } + targetUri = tsmStore.getSourceStoreUri(RemoteStoreProtocol.TSM); + if (targetUri == null) { + logger.error(marker, "A TSM rendszer beállítás paraméterei nem elérhetőek."); + throw new NullPointerException("System is not configured properly, missing TSM StoreUri."); + } + fileType = manager.getFileType("High-res"); + if (fileType == null) { + logger.error(marker, "Adatbázis bejegyzés hiba, a 'High-res' FileType nem található."); + throw new NullPointerException("System is not configured properly, missing 'High-res' FileType."); + } + if (mediaCubeMedia == null) { + logger.error(marker, "A folyamat 'mediaCubeMedia' bemeneti paramétere üres."); + throw new NullPointerException("System is not configured properly, 'mediaCubeMedia' input parameter missing."); + } + + } +} diff --git a/server/-product/production/HIRTV/jobs/steps/TSMExtendedRetrieveStep.java b/server/-product/production/HIRTV/jobs/steps/TSMExtendedRetrieveStep.java new file mode 100644 index 00000000..8f81dde8 --- /dev/null +++ b/server/-product/production/HIRTV/jobs/steps/TSMExtendedRetrieveStep.java @@ -0,0 +1,92 @@ +package user.jobengine.server.steps; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + +import user.commons.StoreUri; +import user.commons.remotestore.RemoteStoreProtocol; +import user.jobengine.db.ArchivedMedia; +import user.jobengine.db.IItemManager; +import user.jobengine.server.IJobEngine; +import user.jobengine.server.IJobRuntime; +import user.jobengine.server.steps.shared.EscortFiles; + +public class TSMExtendedRetrieveStep extends TSMRestoreStep { + private static String NEXIO_HOST = System.getProperty("nexio.host"); + // private static String NEXIO_HOST = SystemConfiguration.getInstance().value("services.nexio.host"); + private static final Logger logger = LogManager.getLogger(); + + private boolean useNexioTarget; + private int nexioPort; + private String nexioUserName, nexioPassword; + private String nexioAgency; + + @Override + protected void afterRestore(StoreUri targetUri, String targetPath, int killDateDays, String targetFileName) throws Exception { + if (useNexioTarget) { + EscortFiles.setNEXIOKillDate(killDateDays, targetFileName, nexioAgency, targetUri); + } else { + super.afterRestore(targetUri, targetPath, killDateDays, targetFileName); + } + } + + @Override + protected void beforeRestore(StoreUri targetURI, String targetName) throws Exception { + String newTargetName = targetName; + if (targetName.contains(".")) + newTargetName = targetName.substring(0, targetName.lastIndexOf('.')); + if (useNexioTarget) + if (targetURI.fileExists(newTargetName + ".mxf")) + throw new Exception(String.format("%s-The newly created file name is existed.", getClass().getSimpleName())); + } + + @Override + protected void checkTargetPath(String targetPath) { + if (!useNexioTarget) + super.checkTargetPath(targetPath); + } + + @Override + protected StoreUri createTargetUri(IItemManager manager, String targetPath) throws NullPointerException { + StoreUri result = null; + logger.info(getSessionMarker(), "Create target uri {}", targetPath); + if (useNexioTarget) { + if (NEXIO_HOST == null) { + throw new NullPointerException("Missing system property on 'nexio.host' name"); + } + result = manager.createStoreUri(RemoteStoreProtocol.FTP, NEXIO_HOST); + result.setPortNumber(nexioPort); + result.setUserName(nexioUserName); + result.setPassword(nexioPassword); + } else + result = super.createTargetUri(manager, targetPath); + return result; + } + + @StepEntry + public Object[] execute(ArchivedMedia archivedMedia, String targetPath, String targetNamePattern, String successRecipient, int killDateDays, + String localRetrievePath, String globalRetrievePath, boolean useNexioTarget, String nexioAgency, int nexioPort, String nexioUserName, + String nexioPassword, IJobEngine jobEngine, IJobRuntime jobRuntime) throws Exception { + this.useNexioTarget = useNexioTarget; + this.nexioAgency = nexioAgency; + this.nexioPort = nexioPort; + this.nexioUserName = nexioUserName; + this.nexioPassword = nexioPassword; + if (nexioPort == 0) { + throw new NullPointerException("System is not configured properly, 'nexioPort' input parameter missing."); + } + if (nexioUserName == null) { + throw new NullPointerException("System is not configured properly, 'nexioUserName' input parameter missing."); + } + if (nexioPassword == null) { + throw new NullPointerException("System is not configured properly, 'nexioPassword' input parameter missing."); + } + if (nexioAgency == null) { + throw new NullPointerException("System is not configured properly, 'nexioAgency' input parameter missing."); + } + + return super.execute(archivedMedia.getMedia(), targetPath, targetNamePattern, successRecipient, killDateDays, localRetrievePath, globalRetrievePath, + jobEngine, jobRuntime); + } + +} diff --git a/server/-product/production/HIRTV/jobs/steps/TSMRestoreStep.java b/server/-product/production/HIRTV/jobs/steps/TSMRestoreStep.java new file mode 100644 index 00000000..af216f42 --- /dev/null +++ b/server/-product/production/HIRTV/jobs/steps/TSMRestoreStep.java @@ -0,0 +1,193 @@ +package user.jobengine.server.steps; + +import java.io.IOException; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.text.Normalizer; +import java.util.List; +import java.util.regex.Pattern; + +import org.apache.commons.lang.StringUtils; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.apache.logging.log4j.Marker; +import org.apache.logging.log4j.message.Message; + +import user.commons.LogUtils; +import user.commons.RemoteFile; +import user.commons.StoreUri; +import user.commons.mediatool.Timecode; +import user.commons.mediatool.Timecode.Type; +import user.commons.remotestore.IProgressEventListener; +import user.commons.remotestore.IStatusEventListener; +import user.commons.remotestore.ProgressEvent; +import user.commons.remotestore.RemoteStoreProtocol; +import user.commons.remotestore.StatusEvent; +import user.jobengine.db.IItemManager; +import user.jobengine.db.Media; +import user.jobengine.db.MediaFile; +import user.jobengine.db.Store; +import user.jobengine.server.IJobEngine; +import user.jobengine.server.IJobRuntime; +import user.jobengine.server.steps.shared.EscortFiles; +import user.jobengine.server.steps.shared.MetadataSaver; + +public class TSMRestoreStep extends JobStep { + private static final String DOT = "."; + public static final Pattern DIACRITICS_AND_FRIENDS = Pattern.compile("[\\p{InCombiningDiacriticalMarks}\\p{IsLm}\\p{IsSk}]+"); + private static final Logger logger = LogManager.getLogger(); + private IItemManager manager; + private StoreUri targetUri; + private StoreUri sourceUri; + private String sourceFileName; + private Marker marker; + + protected void afterRestore(StoreUri targetUri, String targetPath, int killDateDays, String targetFileName) throws IOException, Exception { + if (killDateDays != 0) + EscortFiles.createUNCKillDate(targetPath, targetFileName, killDateDays, marker); + + if (RemoteStoreProtocol.LOCAL.equals(targetUri.getProtocol())) { + Path path = Paths.get(targetPath, targetFileName); + MetadataSaver.saveMetadata(path.toString()); + } + + } + + protected void beforeRestore(StoreUri targetURI, String targetFileName) throws Exception { + } + + protected void checkTargetPath(String targetPath) { + if (StringUtils.isBlank(targetPath)) { + logger.error(marker, "A folyamat 'targetPath' bemeneti paramétere üres."); + throw new NullPointerException("System is not configured properly, 'targetPath' input parameter missing."); + } + } + + protected StoreUri createTargetUri(IItemManager manager, String targetPath) { + return manager.createStoreUri(RemoteStoreProtocol.LOCAL, targetPath); + } + + @StepEntry + public Object[] execute(Media mediaCubeMedia, String targetPath, String targetNamePattern, String successRecipient, int killDateDays, + String localRetrievePath, String globalRetrievePath, IJobEngine jobEngine, IJobRuntime jobRuntime) throws Exception { + marker = jobRuntime.getSessionMarker(); + setAndCheck(mediaCubeMedia, targetPath, targetNamePattern, localRetrievePath, globalRetrievePath, jobEngine); + String targetFileName = String.format(targetNamePattern, sourceFileName); + // 20210129 + // targetFileName = getMaximizedFileName(mediaCubeMedia, targetFileName, 120); + + Timecode timecode = new Timecode(mediaCubeMedia.getLength(), Type.PAL); + try { + String details = String.format("%s (%s)", sourceFileName, timecode.toString()); + jobRuntime.setDescription(details); + beforeRestore(targetUri, targetNamePattern); + final IJobRuntime runtime = jobRuntime; + sourceUri.addProgressListener(new IProgressEventListener() { + @Override + public void progressChanged(ProgressEvent evt) { + runtime.incrementProgress(evt.getProgress()); + } + }); + sourceUri.addStatusListener(new IStatusEventListener() { + @Override + public void statusChanged(StatusEvent evt) { + evt.setCancel(!canContinue()); + } + }); + RemoteFile result = sourceUri.transferFrom(targetUri, sourceFileName, targetFileName); + + String globalTargetPath = Paths.get(targetPath, targetFileName).getParent().toString().replace(Paths.get(localRetrievePath).toString(), + globalRetrievePath); + + logger.info(marker, + "Az '{}' állomány visszatöltése sikeres volt '{}' néven. A célmappa a ide kattintva nyitható meg.", + sourceFileName, targetFileName, globalTargetPath); + afterRestore(targetUri, targetPath, killDateDays, targetFileName); + + } catch (Exception e) { + Message msg = LogUtils.format("Az '{}' állomány visszatöltése sikertelen. A rendszer hibaüzenete: {}", sourceFileName, e.getMessage()); + logger.error(marker, msg); + // logger.error(jobRuntime.marker, msg); + logger.catching(e); + throw e; + } + + return null; + } + + private String getMaximizedFileName(Media mediaCubeMedia, String targetFileName, int limit) { + String name = targetFileName; + String extension = ""; + if (name.contains(DOT)) { + extension = DOT + name.substring(name.lastIndexOf(DOT) + 1); + name = name.substring(0, name.lastIndexOf(DOT)); + } + String typeName = Normalizer.normalize(mediaCubeMedia.getItemType().getName(), Normalizer.Form.NFD); + typeName = DIACRITICS_AND_FRIENDS.matcher(typeName).replaceAll(""); + typeName = typeName.replace(" ", "_"); + + int allowedSize = limit - typeName.length() - 1 - extension.length(); + if (name.length() > allowedSize) + name = name.substring(0, allowedSize); + + return String.format("%s_%s%s", name, typeName, extension); + } + + private String getSourceFileName(Media mediaCubeMedia, Store store) { + List mediaFiles = mediaCubeMedia.getMediaFiles(); + if (mediaFiles == null) + return null; + for (MediaFile mediaFile : mediaFiles) { + if (mediaFile.getStore().getId() == store.getId()) + return mediaFile.getRelativePath(); + } + return null; + } + + private void setAndCheck(Media mediaCubeMedia, String targetPath, String targetNamePattern, String localRetrievePath, String globalRetrievePath, + IJobEngine jobEngine) { + if (jobEngine == null) { + logger.error(marker, "Az folyamatkezelő réteg nem elérhető."); + throw new NullPointerException("Internal error, missing JobEngine reference."); + } + manager = jobEngine.getItemManager(); + if (manager == null) { + logger.error(marker, "Az adatbáziskezelő réteg nem elérhető."); + throw new NullPointerException("Internal error, missing ItemManager reference."); + } + if (mediaCubeMedia == null) { + logger.error(marker, "A folyamat 'mediaCubeMedia' bemeneti paramétere üres."); + throw new NullPointerException("System is not configured properly, 'mediaCubeMedia' input parameter missing."); + } + checkTargetPath(targetPath); + if (StringUtils.isBlank(targetNamePattern)) { + logger.error(marker, "A folyamat 'targetNamePattern' bemeneti paramétere üres."); + throw new NullPointerException("System is not configured properly, 'targetNamePattern' input parameter missing."); + } + Store tsmStore = manager.getSystemStore(false); + if (tsmStore == null) { + logger.error(marker, "A TSM rendszer beállítás nem elérhető."); + throw new NullPointerException("System is not configured properly, missing TSM Store."); + } + sourceUri = tsmStore.getSourceStoreUri(RemoteStoreProtocol.TSM); + if (sourceUri == null) { + logger.error(marker, "A TSM rendszer beállítás paraméterei nem elérhetőek."); + throw new NullPointerException("System is not configured properly, missing TSM StoreUri."); + } + targetUri = createTargetUri(manager, targetPath); + sourceFileName = getSourceFileName(mediaCubeMedia, tsmStore); + if (sourceFileName == null) { + logger.error(marker, "Adatbázis bejegyzés hiba, a visszatöltendő fájl neve nem található."); + throw new NullPointerException("Database error, missing MediaFile 'relativePath'."); + } + + if (StringUtils.isBlank(localRetrievePath)) { + logger.error(marker, "A folyamat 'localRetrievePath' bemeneti paramétere üres."); + throw new NullPointerException("System is not configured properly, 'localRetrievePath' input parameter missing."); + } + if (StringUtils.isBlank(globalRetrievePath)) { + logger.error(marker, "A folyamat 'globalRetrievePath' bemeneti paramétere üres."); + throw new NullPointerException("System is not configured properly, 'globalRetrievePath' input parameter missing."); + } + } +} diff --git a/server/-product/production/HIRTV/jobs/steps/TSMRetrieveMissingMaterialStep.java b/server/-product/production/HIRTV/jobs/steps/TSMRetrieveMissingMaterialStep.java new file mode 100644 index 00000000..4f6e6dc6 --- /dev/null +++ b/server/-product/production/HIRTV/jobs/steps/TSMRetrieveMissingMaterialStep.java @@ -0,0 +1,45 @@ +package user.jobengine.server.steps; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + +import com.ibm.nosql.json.api.BasicDBObject; +import com.ibm.nosql.json.api.DB; +import com.ibm.nosql.json.api.DBCollection; + +import user.commons.morpheus.MorpheusStrings; +import user.commons.nosql.NoSQLUtils; +import user.jobengine.db.IItemManager; +import user.jobengine.db.Media; +import user.jobengine.server.IJobEngine; +import user.jobengine.server.IJobRuntime; + +public class TSMRetrieveMissingMaterialStep extends TSMRestoreStep { + private static final String TARGETNAMEPATTERN = "%s"; + private static final Logger logger = LogManager.getLogger(); + + @StepEntry + public Object[] execute(BasicDBObject material, String targetPath, String globalRetrievePath, IJobEngine jobEngine, IJobRuntime jobRuntime) + throws Exception { + try { + IItemManager itemManager = jobEngine.getItemManager(); + Media mediaCubeMedia = itemManager.getMedia(material.getLong(MorpheusStrings.MEDIAID)); + + super.execute(mediaCubeMedia, targetPath, TARGETNAMEPATTERN, null, 0, targetPath, globalRetrievePath, jobEngine, jobRuntime); + saveMaterial(material, MorpheusStrings.STATUS_DONE); + } catch (Exception e) { + saveMaterial(material, MorpheusStrings.STATUS_ERROR); + logger.catching(e); + throw e; + } + return null; + } + + private void saveMaterial(BasicDBObject material, String status) throws Exception { + DB db = NoSQLUtils.getNoSQLDB(); + DBCollection collection = db.getCollection(MorpheusStrings.COLLECTION_NAME); + material.put(MorpheusStrings.STATUS, MorpheusStrings.STATUS_DONE); + collection.save(material); + } + +} diff --git a/server/-product/production/HIRTV/jobs/steps/TSMSystemRestoreStep.java b/server/-product/production/HIRTV/jobs/steps/TSMSystemRestoreStep.java new file mode 100644 index 00000000..e85b9272 --- /dev/null +++ b/server/-product/production/HIRTV/jobs/steps/TSMSystemRestoreStep.java @@ -0,0 +1,151 @@ +package user.jobengine.server.steps; + +import java.io.IOException; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.List; + +import org.apache.commons.lang.StringUtils; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.apache.logging.log4j.Marker; +import org.apache.logging.log4j.message.Message; + +import user.commons.LogUtils; +import user.commons.RemoteFile; +import user.commons.StoreUri; +import user.commons.log4j2.marker.MediaCubeMarker; +import user.commons.remotestore.IProgressEventListener; +import user.commons.remotestore.IStatusEventListener; +import user.commons.remotestore.ProgressEvent; +import user.commons.remotestore.RemoteStoreProtocol; +import user.commons.remotestore.StatusEvent; +import user.jobengine.db.IItemManager; +import user.jobengine.db.Media; +import user.jobengine.db.MediaFile; +import user.jobengine.db.Store; +import user.jobengine.server.IJobEngine; +import user.jobengine.server.IJobRuntime; +import user.jobengine.server.steps.shared.EscortFiles; +import user.jobengine.server.steps.shared.MetadataSaver; + +public class TSMSystemRestoreStep extends JobStep { + private static final Logger logger = LogManager.getLogger(); + private IItemManager manager; + private StoreUri targetUri; + private StoreUri sourceUri; + private String sourceFileName; + private Marker marker; + + protected void afterRestore(StoreUri targetUri, String targetPath, int killDateDays, String targetFileName) throws IOException, Exception { + if (killDateDays > 0) + EscortFiles.createUNCKillDate(targetPath, targetFileName, killDateDays, marker); + if (RemoteStoreProtocol.LOCAL.equals(targetUri.getProtocol())) { + Path path = Paths.get(targetPath, targetFileName); + MetadataSaver.saveMetadata(path.toString()); + } + } + + protected void beforeRestore(StoreUri targetURI, String targetFileName) throws Exception { + } + + protected void checkTargetPath(String targetPath) { + if (StringUtils.isBlank(targetPath)) { + logger.error(marker, "A folyamat 'targetPath' bemeneti paramétere üres."); + throw new NullPointerException("System is not configured properly, 'targetPath' input parameter missing."); + } + } + + protected StoreUri createTargetUri(IItemManager manager, String targetPath) { + return manager.createStoreUri(RemoteStoreProtocol.LOCAL, targetPath); + } + + @StepEntry + public Object[] execute(Media mediaCubeMedia, String targetPath, String targetNamePattern, String successRecipient, int killDateDays, IJobEngine jobEngine, + IJobRuntime jobRuntime) throws Exception { + marker = jobRuntime.getSessionMarker(); + setAndCheck(mediaCubeMedia, targetPath, targetNamePattern, jobEngine); + String targetFileName = String.format(targetNamePattern, sourceFileName); + try { + beforeRestore(targetUri, targetNamePattern); + final IJobRuntime runtime = jobRuntime; + sourceUri.addProgressListener(new IProgressEventListener() { + @Override + public void progressChanged(ProgressEvent evt) { + runtime.incrementProgress(evt.getProgress()); + } + }); + sourceUri.addStatusListener(new IStatusEventListener() { + @Override + public void statusChanged(StatusEvent evt) { + evt.setCancel(!canContinue()); + } + }); + RemoteFile result = sourceUri.transferFrom(targetUri, sourceFileName, targetFileName); + + Message msg = LogUtils.format("Az '{}' állomány visszatöltése sikeres volt '{}' néven. ", sourceFileName, targetFileName); + if (StringUtils.isNotBlank(successRecipient)) + logger.info(new MediaCubeMarker(successRecipient), msg); + logger.info(marker, msg); + afterRestore(targetUri, targetPath, killDateDays, targetFileName); + + } catch (Exception e) { + Message msg = LogUtils.format("Az '{}' állomány visszatöltése sikertelen. A rendszer hibaüzenete: {}", sourceFileName, e.getMessage()); + logger.error(marker, msg); + // logger.error(jobRuntime.marker, msg); + logger.catching(e); + throw e; + } + + return null; + } + + private String getSourceFileName(Media mediaCubeMedia, Store store) { + List mediaFiles = mediaCubeMedia.getMediaFiles(); + if (mediaFiles == null) + return null; + for (MediaFile mediaFile : mediaFiles) { + if (mediaFile.getStore().getId() == store.getId()) + return mediaFile.getRelativePath(); + } + return null; + } + + private void setAndCheck(Media mediaCubeMedia, String targetPath, String targetNamePattern, IJobEngine jobEngine) { + if (jobEngine == null) { + logger.error(marker, "Az folyamatkezelő réteg nem elérhető."); + throw new NullPointerException("Internal error, missing JobEngine reference."); + } + manager = jobEngine.getItemManager(); + if (manager == null) { + logger.error(marker, "Az adatbáziskezelő réteg nem elérhető."); + throw new NullPointerException("Internal error, missing ItemManager reference."); + } + if (mediaCubeMedia == null) { + logger.error(marker, "A folyamat 'mediaCubeMedia' bemeneti paramétere üres."); + throw new NullPointerException("System is not configured properly, 'mediaCubeMedia' input parameter missing."); + } + checkTargetPath(targetPath); + if (StringUtils.isBlank(targetNamePattern)) { + logger.error(marker, "A folyamat 'targetNamePattern' bemeneti paramétere üres."); + throw new NullPointerException("System is not configured properly, 'targetNamePattern' input parameter missing."); + } + Store tsmStore = manager.getSystemStore(false); + if (tsmStore == null) { + logger.error(marker, "A TSM rendszer beállítás nem elérhető."); + throw new NullPointerException("System is not configured properly, missing TSM Store."); + } + sourceUri = tsmStore.getSourceStoreUri(RemoteStoreProtocol.TSM); + if (sourceUri == null) { + logger.error(marker, "A TSM rendszer beállítás paraméterei nem elérhetőek."); + throw new NullPointerException("System is not configured properly, missing TSM StoreUri."); + } + targetUri = createTargetUri(manager, targetPath); + sourceFileName = getSourceFileName(mediaCubeMedia, tsmStore); + if (sourceFileName == null) { + logger.error(marker, "Adatbázis bejegyzés hiba, a visszatöltendő fájl neve nem található."); + throw new NullPointerException("Database error, missing MediaFile 'relativePath'."); + } + + } +} diff --git a/server/-product/production/HIRTV/jobs/steps/TranscodeFFAStranStep.java b/server/-product/production/HIRTV/jobs/steps/TranscodeFFAStranStep.java new file mode 100644 index 00000000..99ca1744 --- /dev/null +++ b/server/-product/production/HIRTV/jobs/steps/TranscodeFFAStranStep.java @@ -0,0 +1,157 @@ +package user.jobengine.server.steps; + +import java.io.File; +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; + +import org.apache.commons.io.FilenameUtils; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.apache.logging.log4j.Marker; +import org.apache.logging.log4j.message.Message; +import org.apache.logging.log4j.message.ParameterizedMessage; + +import user.commons.FFAStransAPI; +import user.commons.IFFAStransAPI; +import user.commons.StoreUri; +import user.commons.mediatool.Timecode; +import user.commons.mediatool.Timecode.Type; +import user.commons.remotestore.RemoteStoreProtocol; +import user.jobengine.db.FileType; +import user.jobengine.db.IItemManager; +import user.jobengine.db.Media; +import user.jobengine.db.Store; +import user.jobengine.server.IJobEngine; +import user.jobengine.server.IJobRuntime; +import user.jobengine.server.steps.shared.EscortFiles; + +public class TranscodeFFAStranStep extends JobStep { + private static final int POLL_INTERVALL = 3000; + private static final String MP4EXT = ".MP4"; + private static final String MXFEXT = ".MXF"; + private static final String LOWRES_FILETYPE = "Low-res"; + private static final Logger logger = LogManager.getLogger("TranscodeFFAStranStep"); + private IItemManager manager; + private Store store; + private FileType fileType; + private Media mediaCubeMedia; + private Marker marker; + + @StepEntry + public Object[] execute(ArchiveItem archiveItem, Media mediaCubeMedia, String transcoderAddress, + String transcoderTemplateName, String globalHiresSourcePath, String localLowresTargetPath, + boolean deleteSource, IJobEngine jobEngine, IJobRuntime jobRuntime) throws Exception { + + this.marker = jobRuntime.getSessionMarker(); + this.manager = jobEngine.getItemManager(); + this.store = check(manager.getCurrentLowresStore(), "lowres Store"); + this.fileType = check(manager.getFileType(LOWRES_FILETYPE), "lowres FileType"); + this.mediaCubeMedia = check(mediaCubeMedia, "mediaCubeMedia"); + check(archiveItem, "archiveItem"); + check(transcoderAddress, "transcoderAddress"); + check(transcoderTemplateName, "transcoderTemplateName"); + check(globalHiresSourcePath, "globalHiresSourcePath"); + check(localLowresTargetPath, "localLowresTargetPath"); + + File sourceMediaFile = new File(archiveItem.getMediaFile()); + logger.info("Transcoding {}", archiveItem.getMediaFile()); + String sourceFileName = sourceMediaFile.getName(); + Timecode timecode = new Timecode(mediaCubeMedia.getLength(), Type.PAL); + + String details = String.format("%s (%s, %d bytes)", sourceFileName, timecode.toString(), + sourceMediaFile.length()); + + StoreUri storeUri = store.getTargetStoreUri(RemoteStoreProtocol.LOCAL); + if (storeUri == null) + throw new Exception("Can not detect proxy folder."); + + String webPath = storeUri.toString(true); + + Path targetPath = null; + try { + String targetFileName = FilenameUtils.removeExtension(sourceFileName) + MP4EXT; + targetPath = Paths.get(localLowresTargetPath, targetFileName); + if (!targetPath.toFile().exists()) { + // jobRuntime.setDescription(String.format("%s: %s", + // jobRuntime.getDescription(), details)); + jobRuntime.setDescription(String.format("%s transzkódolása", details)); + String sourceFile = Paths.get(globalHiresSourcePath, sourceFileName).toString(); + IFFAStransAPI api = new FFAStransAPI(transcoderAddress, p -> { + if (p <= 100) + jobRuntime.incrementProgress(p); + }); + + api.submit(transcoderTemplateName, sourceFile); + api.monitor(POLL_INTERVALL); + } + + // a sikeres transzkod utan nem mindig van ott egybol a fajl + long started = System.currentTimeMillis(); + while (!targetPath.toFile().exists()) { + long current = System.currentTimeMillis(); + // max 5 perc varakozas + if (current - started > 5 * 60 * 1000) + throw new Exception("Transcode job target file access timed out"); + Thread.sleep(POLL_INTERVALL); + } + + postprocess(targetPath, webPath); + + } catch (Exception e) { + logger.catching(e); + Message m = new ParameterizedMessage("{} átkódolás hiba: {}", sourceFileName, e.getMessage()); + logger.error(marker, m); + throw new Exception(m.getFormattedMessage()); + } finally { + try { + if (deleteSource && sourceMediaFile != null && sourceMediaFile.exists()) + sourceMediaFile.delete(); + } catch (Exception e) { + logger.catching(e); + } + try { + if (deleteSource && targetPath != null && targetPath.toFile().exists()) + Files.delete(targetPath); + } catch (Exception e) { + logger.catching(e); + } + } + return null; + } + + private void postprocess(Path transcodedFilePath, String webPath) throws IOException { + Path lowresPath = null; + try { + String transcodedFileName = transcodedFilePath.getFileName().toString(); + String targetPath = null; + if (transcodedFileName.indexOf(".") > 2) { + Path subdir = Paths.get(transcodedFileName.substring(0, 1), transcodedFileName.substring(1, 2), + transcodedFileName.substring(2, 3)); + EscortFiles.ensureUNCFolder(webPath, subdir.toString()); + targetPath = Paths.get(subdir.toString(), transcodedFileName).toString(); + } else { + targetPath = transcodedFileName; + } + lowresPath = Paths.get(webPath, targetPath); + int version = 1; + while (lowresPath.toFile().exists()) { + String fileName = transcodedFileName + version + MP4EXT; + lowresPath = Paths.get(lowresPath.toString().replace(transcodedFileName, fileName)); + targetPath = targetPath.replace(transcodedFileName, fileName); + transcodedFileName = fileName; + version++; + } + + Files.move(transcodedFilePath, lowresPath); + manager.createMediaFile(targetPath, fileType, store, mediaCubeMedia).add(); + } catch (IOException e) { + logger.catching(e); + logger.error(marker, "A(z) '{}' állomány mozgatása a '{}' helyre nem sikerült.", transcodedFilePath, + lowresPath); + throw e; + } + } + +} diff --git a/server/-product/production/HIRTV/jobs/steps/TranscodeSELENIOStep.java b/server/-product/production/HIRTV/jobs/steps/TranscodeSELENIOStep.java new file mode 100644 index 00000000..ebe2ce9c --- /dev/null +++ b/server/-product/production/HIRTV/jobs/steps/TranscodeSELENIOStep.java @@ -0,0 +1,243 @@ +package user.jobengine.server.steps; + +import java.io.File; +import java.net.URL; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.nio.file.StandardCopyOption; +import java.util.Arrays; +import java.util.List; + +import javax.xml.namespace.QName; + +import org.apache.commons.lang.StringUtils; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.apache.logging.log4j.Marker; +import org.apache.logging.log4j.message.Message; +import org.apache.logging.log4j.message.ParameterizedMessage; + +import user.commons.StoreUri; +import user.commons.configuration.SystemConfiguration; +import user.commons.remotestore.RemoteStoreProtocol; +import user.commons.selenio.wsclient.AudioSource; +import user.commons.selenio.wsclient.Clip; +import user.commons.selenio.wsclient.ClipList; +import user.commons.selenio.wsclient.Exception_Exception; +import user.commons.selenio.wsclient.FileSnapshot; +import user.commons.selenio.wsclient.MediaFile; +import user.commons.selenio.wsclient.State; +import user.commons.selenio.wsclient.TranscodeMgrWS; +import user.commons.selenio.wsclient.TranscodeMgrWSService; +import user.commons.selenio.wsclient.TranscodeRequest; +import user.commons.selenio.wsclient.TranscodeSource; +import user.commons.selenio.wsclient.TranscodeTask; +import user.commons.selenio.wsclient.TranscodeTask.OutputFiles; +import user.commons.selenio.wsclient.VideoSource; +import user.jobengine.db.FileType; +import user.jobengine.db.IItemManager; +import user.jobengine.db.Media; +import user.jobengine.db.Store; +import user.jobengine.server.IJobEngine; +import user.jobengine.server.IJobRuntime; +import user.jobengine.server.steps.shared.EscortFiles; +import user.jobengine.server.steps.shared.ItemManagerExtensions; + +public class TranscodeSELENIOStep extends JobStep { + + private static final String MXFEXT = ".MXF"; + private static final String LOWRES_FILETYPE = "Low-res"; + private static final Logger logger = LogManager.getLogger(); + private static String SELENIO_API_ADDRESS = SystemConfiguration.getInstance().value("services.selnio.api.address", + "http://10.10.1.71:44000/TranscodeMgrWS?wsdl"); + private static String SELENIO_API_PROJECTFILE = SystemConfiguration.getInstance().value("services.selnio.api.projevt-file", + "\\\\10.10.1.71\\Data\\Blueprints\\MP4_H264_AAC.zenium"); + private final List showStoppers = Arrays.asList(State.CANCELLED, State.COMPLETE, State.FAILED); + private TranscodeMgrWS transcoder = null; + private IItemManager manager; + private Store store; + private FileType fileType; + private String transcoderTargetPath; + private Marker marker; + private StoreUri storeUri; + + private TranscodeRequest buildTranscodeRequest(String projectFilePath, String sourceFilePath) throws java.lang.Exception { + Clip clip = new Clip(); + MediaFile mediaFile = new MediaFile(); + mediaFile.setFile(sourceFilePath); + AudioSource audioSource = new AudioSource(); + audioSource.setMediaFile(mediaFile); + clip.getAudioSource().add(audioSource); + + VideoSource videoSource = new VideoSource(); + videoSource.setMediaFile(mediaFile); + clip.setVideoSource(videoSource); + + ClipList clipList = new ClipList(); + clipList.getClip().add(clip); + + TranscodeSource transcodeSource = new TranscodeSource(); + transcodeSource.setClipList(clipList); + + TranscodeRequest transcodeRequest = new TranscodeRequest(); + FileSnapshot project = new FileSnapshot(); + project.setFullPath(projectFilePath); + transcodeRequest.setProject(project); + transcodeRequest.setTranscodeSource(transcodeSource); + + // TranscodeDestination transcodeDestination = new TranscodeDestination(); + + //transcodeDestination.setOutputWriteDirectory(outputPath); + // transcodeDestination.setOutputPostMoveDirectory(targetPath); + // transcodeRequest.setTranscodeDestination(transcodeDestination); + return transcodeRequest; + } + + @StepEntry + public Object[] execute(String globalSourcePath, ArchiveItem archiveItem, Media mediaCubeMedia, String transcoderTargetPath, IJobEngine jobEngine, + IJobRuntime jobRuntime) throws Exception { + marker = jobRuntime.getSessionMarker(); + String sourceFileName = null; + + try { + setAndCheck(globalSourcePath, transcoderTargetPath, jobEngine); + + File sourceMediaFile = new File(archiveItem.getMediaFile()); + sourceFileName = sourceMediaFile.getName(); + + //Nincs mit transzkodolni, a TSMBackupStep csinal masolatot a mediafileokrol + //A dupla ellenorzes a napon beluli ismetlesek miatt kell + long existingMediaId = archiveItem.getExistingMediaId(); + if (existingMediaId == 0) + existingMediaId = ItemManagerExtensions.getExistingRundownMedia(manager, sourceFileName.replace(MXFEXT, "")); + if (existingMediaId != 0 || sourceMediaFile.length() == 0) + return null; + + String details = String.format("%s (%d bytes)", sourceFileName, sourceMediaFile.length()); + + Path inputPath = Paths.get(globalSourcePath, sourceFileName); + String sourceFilePath = inputPath.toString(); + TranscodeRequest transcodeRequest = buildTranscodeRequest(SELENIO_API_PROJECTFILE, sourceFilePath); + TranscodeTask transcodeTask = transcoder.submitTranscodeTask(transcodeRequest); + + if (transcodeTask == null) + throw new NullPointerException("Unable to submit transcode task, server response is empty for transcode input: " + sourceFilePath); + + jobRuntime.setDescription(String.format("%s: %s", jobRuntime.getDescription(), details)); + + transcodeTask = monitor(jobRuntime, sourceFilePath, transcodeTask); + jobRuntime.incrementProgress(100); + processState(transcodeTask, jobEngine, mediaCubeMedia); + } catch (Exception e) { + logger.catching(e); + Message m = new ParameterizedMessage("Az '{}' állomány átkódolása sikertelen. A rendszer hibaüzenete: {}", sourceFileName, e.getMessage()); + logger.error(marker, m); + throw new Exception(m.getFormattedMessage()); + } + return null; + } + + private TranscodeTask monitor(IJobRuntime jobRuntime, String sourceFile, TranscodeTask transcodeTask) throws Exception_Exception, InterruptedException { + while (true) { + transcodeTask = transcoder.getTranscodeTask(transcodeTask.getId()); + if (transcodeTask == null) + throw new NullPointerException("Unable to query transcode task, server response is empty for transcode input: " + sourceFile); + + // Integer estimate = transcodeTask.getEstimateSecondsRemaining(); + // if (estimate != null) + // logger.info("Estimate {}", estimate); + Double progress = transcodeTask.getProgress(); + if (progress != null) + jobRuntime.incrementProgress((int) Math.round(progress * 100)); + + if (showStoppers.contains(transcodeTask.getState())) + break; + + Thread.sleep(2000); + } + return transcodeTask; + } + + private void onTranscodeComplete(TranscodeTask transcodeTask, IJobEngine jobEngine, Media mediaCubeMedia) { + OutputFiles of = transcodeTask.getOutputFiles(); + List outputs = of.getOutputFiles(); + + String outFile = null; + if (outputs.size() == 0) { + throw new IndexOutOfBoundsException("There are 0 file in the response: " + transcodeTask.getId()); + } else { + for (int i = 0; i < outputs.size(); i++) { + user.commons.selenio.wsclient.MediaFile selenioMediaFile = outputs.get(0); + if (outFile != null && !outFile.equals(selenioMediaFile.getFile())) + throw new IndexOutOfBoundsException("There are different files in the response: " + transcodeTask.getId()); + outFile = selenioMediaFile.getFile(); + } + } + + String webPath = null; + try { + webPath = storeUri.toString(true); + outFile = outFile.substring(outFile.lastIndexOf("\\") + 1); + if (outFile.indexOf(".") > 2) { + Path subdir = Paths.get(outFile.substring(0, 1), outFile.substring(1, 2), outFile.substring(2, 3)); + manager.createMediaFile(Paths.get(subdir.toString(), outFile).toString(), fileType, store, mediaCubeMedia).add(); + EscortFiles.ensureUNCFolder(webPath, subdir.toString()); + subdir = Paths.get(webPath, subdir.toString()); + //subdir.toFile().mkdirs(); + Files.move(Paths.get(transcoderTargetPath, outFile), Paths.get(subdir.toString(), outFile), StandardCopyOption.REPLACE_EXISTING); + } else { + manager.createMediaFile(outFile, fileType, store, mediaCubeMedia).add(); + Files.move(Paths.get(transcoderTargetPath, outFile), Paths.get(webPath, outFile), StandardCopyOption.REPLACE_EXISTING); + } + } catch (Exception e) { + logger.catching(e); + logger.error(marker, "Az '{}' állomány mozgatása a '{}' mappába nem sikerült.", outFile, webPath); + } + } + + @SuppressWarnings("incomplete-switch") + private void processState(TranscodeTask transcodeTask, IJobEngine jobEngine, Media mediaCubeMedia) throws Exception { + switch (transcodeTask.getState()) { + case CANCELLED: + throw new IllegalStateException("Transcode task was CANCELLED."); + case FAILED: + throw new IllegalStateException("Transcode task has FAILED."); + case COMPLETE: + onTranscodeComplete(transcodeTask, jobEngine, mediaCubeMedia); + break; + } + } + + private void setAndCheck(String globalSourcePath, String transcoderTargetPath, IJobEngine jobEngine) throws Exception { + if (StringUtils.isBlank(SELENIO_API_ADDRESS)) + throw new NullPointerException("System is not configured properly, 'jobengine.selenio.address' startup parameter missing."); + + if (StringUtils.isBlank(SELENIO_API_PROJECTFILE)) + throw new NullPointerException("System is not configured properly, 'jobengine.selenio.projectfilepath' startup parameter missing."); + TranscodeMgrWSService service = new TranscodeMgrWSService(new URL(SELENIO_API_ADDRESS), new QName("http://ws.server.mediamanager.digitalrapids.ca/", "TranscodeMgrWSService")); + transcoder = service.getTranscodeMgrWSPort(); + + if (StringUtils.isBlank(globalSourcePath)) + throw new NullPointerException("System is not configured properly, 'globalInputFolder' parameter missing."); + manager = jobEngine.getItemManager(); + + store = manager.getCurrentLowresStore(); + if (store == null) + throw new NullPointerException("System is not configured properly, low-res system store definition missing."); + + storeUri = store.getTargetStoreUri(RemoteStoreProtocol.LOCAL); + if (storeUri == null) + throw new Exception("Can not detect proxy folder."); + + fileType = manager.getFileType(LOWRES_FILETYPE); + if (fileType == null) + throw new NullPointerException("System is not configured properly, low-res file type definition missing."); + + if (StringUtils.isBlank(transcoderTargetPath)) + throw new NullPointerException("System is not configured properly, 'transcoderTargetPath' parameter missing."); + this.transcoderTargetPath = transcoderTargetPath; + + } + +} diff --git a/server/-product/production/HIRTV/jobs/steps/TranscodeStep.java b/server/-product/production/HIRTV/jobs/steps/TranscodeStep.java new file mode 100644 index 00000000..dd1246ea --- /dev/null +++ b/server/-product/production/HIRTV/jobs/steps/TranscodeStep.java @@ -0,0 +1,32 @@ +package user.jobengine.server.steps; + +import user.commons.configuration.SystemConfiguration; +import user.jobengine.db.Media; +import user.jobengine.server.IJobEngine; +import user.jobengine.server.IJobRuntime; + +public class TranscodeStep extends JobStep { + private static int TRANSCODER_VERSION = SystemConfiguration.getInstance().value("services.transcoder.version", 0); + private static String TRANSCODER_API_ADDRESS = SystemConfiguration.getInstance().value("services.transcoder.api.address", ""); + private static String TRANSCODER_API_TEMPLATE = SystemConfiguration.getInstance().value("services.transcoder.api.template", ""); + + @StepEntry + public Object[] execute(String globalSourcePath, ArchiveItem archiveItem, Media mediaCubeMedia, String transcoderTargetPath, IJobEngine jobEngine, + IJobRuntime jobRuntime) throws Exception { + switch (TRANSCODER_VERSION) { + case 0: { + TranscodeSELENIOStep selenioStep = new TranscodeSELENIOStep(); + selenioStep.execute(globalSourcePath, archiveItem, mediaCubeMedia, transcoderTargetPath, jobEngine, jobRuntime); + break; + } + case 1: { + TranscodeFFAStranStep ffaStransStep = new TranscodeFFAStranStep(); + ffaStransStep.execute(archiveItem, mediaCubeMedia, TRANSCODER_API_ADDRESS, TRANSCODER_API_TEMPLATE, globalSourcePath, transcoderTargetPath, false, + jobEngine, jobRuntime); + break; + } + } + return null; + } + +} diff --git a/server/-product/production/HIRTV/jobs/steps/UpdateGhostMediaDataStep.java b/server/-product/production/HIRTV/jobs/steps/UpdateGhostMediaDataStep.java new file mode 100644 index 00000000..39cb7b30 --- /dev/null +++ b/server/-product/production/HIRTV/jobs/steps/UpdateGhostMediaDataStep.java @@ -0,0 +1,89 @@ +package user.jobengine.server.steps; + +import java.util.List; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.apache.logging.log4j.Marker; + +import user.commons.remotestore.RemoteStoreProtocol; +import user.jobengine.db.IItemManager; +import user.jobengine.db.Media; +import user.jobengine.db.MediaFile; +import user.jobengine.db.Store; +import user.jobengine.server.IJobEngine; +import user.jobengine.server.IJobRuntime; +import user.jobengine.server.steps.shared.MetadataType; +import user.jobengine.server.steps.shared.MetadataTypeDetector; + +public class UpdateGhostMediaDataStep extends JobStep { + private static final Logger logger = LogManager.getLogger(); + private Marker marker; + + @StepEntry + public Object[] execute(Media mediaCubeMedia, IJobEngine jobEngine, IJobRuntime jobRuntime) throws Exception { + marker = jobRuntime.getSessionMarker(); + + IItemManager manager = jobEngine.getItemManager(); + //Refresh from db + List mediaFiles = manager.getMedia(mediaCubeMedia.getId()).getMediaFiles(); + if (mediaFiles != null && mediaFiles.size() == 2) { + MediaFile lowres = null; + MediaFile highres = null; + + for (MediaFile mf : mediaFiles) { + if (mf.getStore().getSourceStoreUri(RemoteStoreProtocol.HTTP) != null) + lowres = mf; + else + highres = mf; + } + + if (highres == null) { + logger.info(marker, "Nincs highres mediaId: {}", mediaCubeMedia.getId()); + return null; + } + if (lowres == null) { + logger.info(marker, "Nincs lowres mediaId: {}", mediaCubeMedia.getId()); + return null; + } + + String id = MetadataTypeDetector.truncateExtension(highres.getRelativePath()); + id = MetadataTypeDetector.truncateVersion(id); + boolean detect = MetadataTypeDetector.GuessMetadataType(id) == MetadataType.OctopusPlaceholder + || MetadataTypeDetector.GuessMetadataType(id) == MetadataType.OctopusStory; + if (!detect) { + logger.info(marker, "Nem bejátszó mediaId: {}, file: {}", mediaCubeMedia.getId(), highres.getRelativePath()); + return null; + } + + Store highresStore = manager.getSystemStore(false); + final long sourceMediaId = lowres.getId(); + final long highresMediaFileId = highres.getId(); + final String highresRealtivePath = highres.getRelativePath(); + + manager.executeQuery("SELECT mediaid FROM mediafile WHERE relativepath=? and storeid=? and id!=?", rs -> { + long mediaId = rs.getLong(1); + Media media = manager.getMedia(mediaId); + if (media.getMediaFilesCount() == 1) { + logger.info(marker, "Hiányzó szellem lowres hozzáadása {} alapján", media.getId()); + + MediaFile mf = (MediaFile) manager.get(MediaFile.class, sourceMediaId); + mf.setMedia(media); + mf.setId(0); + manager.add(mf); + media.setLength(mediaCubeMedia.getLength()); + manager.modify(media); + } + return true; + }, st -> { + st.setString(1, highresRealtivePath); + st.setLong(2, highresStore.getId()); + st.setLong(3, highresMediaFileId); + }); + + } + + return null; + } + +} diff --git a/server/-product/production/HIRTV/jobs/steps/UploadRecordingToNexioStep.java b/server/-product/production/HIRTV/jobs/steps/UploadRecordingToNexioStep.java new file mode 100644 index 00000000..b1e91baf --- /dev/null +++ b/server/-product/production/HIRTV/jobs/steps/UploadRecordingToNexioStep.java @@ -0,0 +1,147 @@ +package user.jobengine.server.steps; + +import java.io.File; +import java.nio.file.Paths; + +import org.apache.commons.lang.StringUtils; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.apache.logging.log4j.Marker; +import org.apache.logging.log4j.message.Message; +import org.apache.logging.log4j.message.ParameterizedMessage; + +import com.ibm.nosql.json.api.DB; + +import user.commons.RemoteFile; +import user.commons.StoreUri; +import user.commons.configuration.SystemConfiguration; +import user.commons.nosql.NoSQLUtils; +import user.commons.remotestore.IProgressEventListener; +import user.commons.remotestore.IStatusEventListener; +import user.commons.remotestore.ProgressEvent; +import user.commons.remotestore.RemoteStoreProtocol; +import user.commons.remotestore.StatusEvent; +import user.jobengine.db.IItemManager; +import user.jobengine.server.IJobEngine; +import user.jobengine.server.IJobRuntime; + +public class UploadRecordingToNexioStep extends JobStep { + private static final Logger logger = LogManager.getLogger(); + private static String NEXIO_HOST = SystemConfiguration.getInstance().value("services.nexio.host"); + + private IItemManager manager; + private DB db; + private StoreUri sourceUri; + private StoreUri targetUri; + private Marker marker; + + private int check(int value, String name) { + if (value == 0) { + logger.error(marker, "A folyamat '{}' bemeneti paramétere 0.", name); + throw new NullPointerException(String.format("System is not configured properly, missing '%s' input parameter.", name)); + } + return value; + } + + private String check(String value, String name) { + if (StringUtils.isBlank(value)) { + logger.error(marker, "A folyamat '{}' bemeneti paramétere üres.", name); + throw new NullPointerException(String.format("System is not configured properly, missing '%s' input parameter.", name)); + } + return value; + } + + @StepEntry + public Object[] execute(ArchiveItem archiveItem, String targetFileName, int nexioPort, String nexioUserName, String nexioPassword, IJobEngine jobEngine, + IJobRuntime jobRuntime) throws Exception { + + marker = jobRuntime.getSessionMarker(); + manager = jobEngine.getItemManager(); + setAndCheck(archiveItem, targetFileName, nexioPort, nexioUserName, nexioPassword); + File sourceFile = new File(archiveItem.getMediaFile()); + String sourceFileName = sourceFile.getName(); + try { + final IJobRuntime runtime = jobRuntime; + sourceUri.addProgressListener(new IProgressEventListener() { + @Override + public void progressChanged(ProgressEvent evt) { + runtime.incrementProgress(evt.getProgress()); + } + }); + sourceUri.addStatusListener(new IStatusEventListener() { + @Override + public void statusChanged(StatusEvent evt) { + evt.setCancel(!canContinue()); + } + }); + + RemoteFile targetFile = null; + try { + targetFile = targetUri.getRemoteFile(targetFileName); + } catch (Exception e) { + logger.catching(e); + } + if (targetFile != null) + throw new Exception("Exists!"); + + RemoteFile remoteFile = sourceUri.transferFrom(targetUri, sourceFileName, targetFileName); + + logger.info(marker, "Az '{}' állomány feltöltése sikeres volt '{}' néven.", sourceFile, targetFileName); + } catch (Exception e) { + logger.catching(e); + if (!archiveItem.removeCatchedFile()) + logger.error(getMarker(), "A {} állomány .catched jelző állománya nem törölhető.", new File(archiveItem.getMediaFile()).getName()); + Message m = new ParameterizedMessage("Az '{}' állomány feltöltése '{}' néven sikertelen. A rendszer hibaüzenete: {}", sourceFile, targetFileName, e + .getMessage()); + logger.error(marker, m); + throw new Exception(m.getFormattedMessage()); + } + return new Object[] {}; + } + + private void setAndCheck(ArchiveItem archiveItem, String targetFileName, int nexioPort, String nexioUserName, String nexioPassword) throws Exception { + db = NoSQLUtils.getNoSQLDB(); + if (db == null) { + logger.error(marker, "Az NoSQL adatkezelő réteg nem elérhető."); + throw new NullPointerException("Internal error, missing NoSQL DB reference."); + } + + if (archiveItem == null) { + logger.error(marker, "A folyamat 'archiveItem' bemeneti paramétere üres."); + throw new NullPointerException("Internal error, missing 'archiveItem'."); + } + + if (archiveItem.getMediaFile() == null) { + logger.error(marker, "A folyamat 'archiveItem.mediaFile' paramétere üres."); + throw new NullPointerException("Internal error, missing 'archiveItem.mediaFile'."); + } + + check(targetFileName, "targetFileName"); + + if (StringUtils.isBlank(NEXIO_HOST)) { + logger.error(marker, "A 'nexio.host' rendszer paraméter nem található."); + throw new NullPointerException("System is not configured properly, 'jobengine.selenio.address' startup parameter missing."); + } + check(nexioPort, "nexioPort"); + check(nexioUserName, "nexioUserName"); + check(nexioPassword, "nexioPassword"); + + targetUri = manager.createStoreUri(RemoteStoreProtocol.FTP, NEXIO_HOST); + targetUri.setRootPath("LXF"); + targetUri.setPortNumber(nexioPort); + targetUri.setUserName(nexioUserName); + targetUri.setPassword(nexioPassword); + if (targetUri == null) { + logger.error(marker, "A forrás nem elérhető."); + throw new NullPointerException("Internal error, missing 'sourceUri'."); + } + + sourceUri = manager.createStoreUri(RemoteStoreProtocol.LOCAL, Paths.get(archiveItem.getMediaFile()).getParent().toString()); + if (sourceUri == null) { + logger.error(marker, "A cél nem elérhető."); + throw new NullPointerException("Internal error, missing 'targetUri'."); + } + + } + +} diff --git a/server/-product/production/HIRTV/jobs/steps/shared/Cmd.java b/server/-product/production/HIRTV/jobs/steps/shared/Cmd.java new file mode 100644 index 00000000..6aaff0e9 --- /dev/null +++ b/server/-product/production/HIRTV/jobs/steps/shared/Cmd.java @@ -0,0 +1,82 @@ +package user.jobengine.server.steps.shared; + +import java.io.BufferedReader; +import java.io.InputStreamReader; +import java.util.ArrayList; +import java.util.List; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + +import user.commons.configuration.SystemConfiguration; + +public class Cmd { + public interface IResponseCallback { + void onResponse(String line); + } + + private static final String FFMPEG_EXECUTABLE = SystemConfiguration.getInstance().value("services.ffmpeg.executable-location"); + + private static final Logger log = LogManager.getLogger(); + + public static ProcessBuilder create(String... args) { + List chunks = new ArrayList<>(); + for (String arg : args) + chunks.add(arg); + + ProcessBuilder processBuilder = new ProcessBuilder(); + processBuilder.command(chunks).redirectErrorStream(true); + return processBuilder; + } + + public static ProcessBuilder create(String command, StringBuilder args) { + args.insert(0, command + "\r\n"); + String[] chunks = args.toString().replace("\r\n", " ").split(" "); + return create(chunks); + } + + public static String execute(ProcessBuilder processBuilder) { + return execute(processBuilder, true); + } + + public static String execute(ProcessBuilder processBuilder, boolean firstResponse) { + return execute(processBuilder, firstResponse, null); + } + + public static String execute(ProcessBuilder processBuilder, boolean firstResponse, IResponseCallback responseCallBack) { + String result = null; + try { + log.debug("Executing : {}", processBuilder.command().toString().replace("[", "").replace("]", "").replace(",", "")); + Process process = processBuilder.start(); + + try (BufferedReader reader = new BufferedReader(new InputStreamReader(process.getInputStream()))) { + String line = null; + while ((line = reader.readLine()) != null) { + log.debug("Process response: {}", line); + if (responseCallBack != null) + responseCallBack.onResponse(line); + //System.out.println(line); + if (line != null && line.length() > 0) { + result = line; + if (firstResponse) + break; + } + } + int exitCode = process.waitFor(); + if (exitCode != 0) + log.error("Exited with error code : " + exitCode); + } catch (Exception e) { + throw e; + } + } catch (Exception e) { + log.error(e); + } + + return result; + } + + public static String getFFMpegExecutable() { + return FFMPEG_EXECUTABLE; + } + +} diff --git a/server/-product/production/HIRTV/jobs/steps/shared/EscortFiles.java b/server/-product/production/HIRTV/jobs/steps/shared/EscortFiles.java new file mode 100644 index 00000000..bc673f95 --- /dev/null +++ b/server/-product/production/HIRTV/jobs/steps/shared/EscortFiles.java @@ -0,0 +1,378 @@ +package user.jobengine.server.steps.shared; + +import java.io.ByteArrayInputStream; +import java.io.File; +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; +import java.io.StringWriter; +import java.io.UnsupportedEncodingException; +import java.nio.file.DirectoryStream; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.nio.file.attribute.FileAttribute; +import java.nio.file.attribute.PosixFilePermission; +import java.nio.file.attribute.PosixFilePermissions; +import java.text.SimpleDateFormat; +import java.util.ArrayList; +import java.util.Calendar; +import java.util.Collections; +import java.util.Date; +import java.util.List; +import java.util.Set; + +import javax.xml.parsers.DocumentBuilder; +import javax.xml.parsers.DocumentBuilderFactory; +import javax.xml.transform.OutputKeys; +import javax.xml.transform.Transformer; +import javax.xml.transform.TransformerConfigurationException; +import javax.xml.transform.TransformerException; +import javax.xml.transform.TransformerFactory; +import javax.xml.transform.TransformerFactoryConfigurationError; +import javax.xml.transform.dom.DOMSource; +import javax.xml.transform.stream.StreamResult; + +import org.apache.commons.lang.StringUtils; +import org.apache.commons.net.ftp.FTPClient; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.apache.logging.log4j.Marker; +import org.apache.logging.log4j.message.Message; +import org.w3c.dom.DOMImplementation; +import org.w3c.dom.Document; +import org.w3c.dom.Element; + +import com.ibm.nosql.json.api.BasicDBObject; +import com.ibm.nosql.json.util.JSON; + +import user.commons.CalendarUtils; +import user.commons.StoreUri; +import user.commons.log4j2.marker.MediaCubeMarker; +import user.commons.remotestore.FtpDirectoryLister; + +public class EscortFiles { + private static final String RECORDTIMESTAMP = "RecordTimeStamp"; + private static final String MODIFIEDTIMESTAMP = "ModifiedTimeStamp"; + public static final String DOT_CATCHED = ".catched"; + public static final String DOT_JSON = ".json"; + private static final Logger logger = LogManager.getLogger(); + private static final String EXTENDEDAGENCY = "ExtendedAgency"; + private static final String EXTENDEDDESCRIPTION = "ExtendedDescription"; + private static final String KILLDATE = "KillDate"; + private static final String FORMAT_KILLDATE = "MM-dd-yyyy"; + private static final String EXTENDEDID = "extendedId"; + private static final String ID = "ID"; + private static final String KILLDATE_FILENAME = "%s.%s.killdate"; + private static final String KILLDATE_EXT = ".killdate"; + private static final String FORMAT_KILLDATENAME = "yyyyMMdd"; + public static final String STATUSFOLDER = ".STATUS"; + public static final String CONFLICTFOLDER = ".CONFLICT"; + + public static String composeKillDate(int days) { + Calendar killDate = Calendar.getInstance(); + killDate.add(Calendar.DAY_OF_YEAR, days); + SimpleDateFormat dateFormat = new SimpleDateFormat(FORMAT_KILLDATENAME); + return dateFormat.format(killDate.getTime()); + } + /* + * + * 02-02-2018 TEST + * TEST AGENT AGENT + */ + + private static String composeKillDateFileName(String fileName, int days) { + return String.format(KILLDATE_FILENAME, fileName, composeKillDate(days)); + } + + public static void createCatchedFile(Path escortFile) throws IOException { + String catchedFileName = escortFile.toString() + DOT_CATCHED; + Path catchedFilePath = Paths.get(catchedFileName); + Files.createFile(catchedFilePath); + } + + public static void createFellow(String escortFile, String extension) throws IOException { + Files.copy(Paths.get(escortFile), Paths.get(escortFile + "." + extension)); + } + + /*** + * A media elérési útján alapján a .STATUS almappában létrehozza a .catch fajlt. + * + * @param mediaFile + * @throws IOException + */ + public static void createMediaCatch(Path mediaFile) throws IOException { + Path catchedFile = createMediaCatchFilePath(mediaFile); + ensureUNCFolder(catchedFile.getParent()); + Files.createFile(catchedFile); + } + + public static Path createMediaCatchFilePath(Path mediaFile) { + String catchFileName = mediaFile.getFileName().toString() + DOT_CATCHED; + return Paths.get(mediaFile.getParent().toString(), STATUSFOLDER, catchFileName); + } + + public static void createMetadata(String filePath, String fileName, String metadata) throws IOException { + ensureUNCFolder(filePath, STATUSFOLDER); + String metadataFileName = fileName + DOT_JSON; + Path metadataPath = Paths.get(filePath, STATUSFOLDER, metadataFileName); + Files.write(metadataPath, metadata.getBytes()); + } + + public static Path constructMetadataPath(Path filePath) throws IOException { + String metadataFileName = filePath.getFileName().toString() + DOT_JSON; + return Paths.get(filePath.getParent().toString(), STATUSFOLDER, metadataFileName); + } + + public static boolean createMetadataIfNotExists(String filePath, String fileName, String metadata) throws IOException { + boolean result = false; + if (!EscortFiles.isMetadataExists(filePath, fileName)) { + EscortFiles.createMetadata(filePath, fileName, metadata); + result = true; + } + return result; + } + + public static void createMorpheusXML(String filePath, String fileName, String content) throws IOException { + ensureUNCFolder(filePath, STATUSFOLDER); + Path xmlPath = Paths.get(filePath, fileName); + if (Files.exists(xmlPath)) + throw new IOException(String.format("Az '%s' állomány már létezik.", xmlPath)); + Files.write(xmlPath, content.getBytes()); + } + + public static byte[] createNEXIODatesMeta(String fileName, Date recorded, Date modified) throws Exception { + DocumentBuilderFactory dbf = DocumentBuilderFactory.newInstance(); + DocumentBuilder db = dbf.newDocumentBuilder(); + DOMImplementation impl = db.getDOMImplementation(); + Document xmlDocument = impl.createDocument(null, null, null); + + Element root = xmlDocument.createElement(ID); + root.setAttribute(EXTENDEDID, fileName); + // 07-13-2020 (19:36:52) + // 05-18-2013 (18:52:24) + SimpleDateFormat df = new SimpleDateFormat("MM-dd-yyyy (HH:mm:ss)"); + root.appendChild(xmlDocument.createElement(MODIFIEDTIMESTAMP)).appendChild(xmlDocument.createTextNode(df.format(modified))); + root.appendChild(xmlDocument.createElement(RECORDTIMESTAMP)).appendChild(xmlDocument.createTextNode(df.format(recorded))); + xmlDocument.appendChild(root); + + return xmlDocumentToString(xmlDocument); + } + + public static byte[] createNEXIOKillDateFile(String fileName, Date killDate, String description, String agency) throws Exception { + DocumentBuilderFactory dbf = DocumentBuilderFactory.newInstance(); + DocumentBuilder db = dbf.newDocumentBuilder(); + DOMImplementation impl = db.getDOMImplementation(); + Document xmlDocument = impl.createDocument(null, null, null); + + Element root = xmlDocument.createElement(ID); + root.setAttribute(EXTENDEDID, fileName); + if (killDate != null) { + String sKillDate = CalendarUtils.toString(CalendarUtils.createCalendar(killDate), FORMAT_KILLDATE); + root.appendChild(xmlDocument.createElement(KILLDATE)).appendChild(xmlDocument.createTextNode(sKillDate)); + } + + if (StringUtils.isNotBlank(description)) + root.appendChild(xmlDocument.createElement(EXTENDEDDESCRIPTION)).appendChild(xmlDocument.createTextNode(description)); + if (StringUtils.isNotBlank(agency)) + root.appendChild(xmlDocument.createElement(EXTENDEDAGENCY)).appendChild(xmlDocument.createTextNode(agency)); + xmlDocument.appendChild(root); + + return xmlDocumentToString(xmlDocument); + } + + public static Document createNEXIOMeta(byte[] content) throws Exception { + DocumentBuilderFactory dbf = DocumentBuilderFactory.newInstance(); + DocumentBuilder db = dbf.newDocumentBuilder(); + DOMImplementation impl = db.getDOMImplementation(); + Document xmlDocument = null; + + try (InputStream is = new ByteArrayInputStream(content)) { + xmlDocument = db.parse(is); + } catch (Exception e) { + logger.catching(e); + } + + return xmlDocument; + } + + public static void createUNCKillDate(String filePath, String fileName, int days, Marker marker) throws IOException { + ensureUNCFolder(filePath, STATUSFOLDER); + String killDateFileName = composeKillDateFileName(fileName, days); + Path killDatePath = Paths.get(filePath, STATUSFOLDER, killDateFileName); + if (Files.exists(killDatePath)) + logger.warn(marker, "Az '{}' állomány már létezik.", killDatePath); + else + Files.createFile(killDatePath); + } + + @SuppressWarnings("unchecked") + public static T decode(Path escortFile) { + T result = null; + try { + byte[] bytes = Files.readAllBytes(escortFile); + String content = new String(bytes); + result = (T) JSON.parse(content); + } catch (Exception e) { + logger.error("Decode error. System message is: ", e.getMessage()); + } + return result; + } + + public static void ensureUNCFolder(Path filePath) throws IOException { + File folder = filePath.toFile(); + if (!folder.exists() || !folder.isDirectory()) { + try { + Set perms = PosixFilePermissions.fromString("rwxrwxrwx"); + FileAttribute> attr = PosixFilePermissions.asFileAttribute(perms); + Files.createDirectories(filePath, attr); + } catch (Exception e) { + // logger.catching(e); + try { + Files.createDirectories(filePath); + } catch (Exception e1) { + logger.catching(e1); + throw e1; + } + } + } + } + + public static void ensureUNCFolder(String filePath, String folderName) throws IOException { + Path statusPath = Paths.get(filePath, folderName); + ensureUNCFolder(statusPath); + } + + public static boolean isCatchedFileExists(Path escortFile) { + String catchedFileName = escortFile.toString() + DOT_CATCHED; + Path catchedFilePath = Paths.get(catchedFileName); + return Files.exists(catchedFilePath); + } + + /*** + * A media eleresi utjan alapjan a .STATUS almappaban vizsgalja .catch fajl + * letezeset. + * + * @param mediaFile + * @return + */ + public static boolean isMediaCatched(Path mediaFile) { + Path catchedFile = createMediaCatchFilePath(mediaFile); + return Files.exists(catchedFile); + } + + public static boolean isMetadataExists(String filePath, String fileName) throws IOException { + boolean result = false; + String metadataFileName = fileName + DOT_JSON; + Path metadataPath = Paths.get(filePath, STATUSFOLDER, metadataFileName); + result = Files.exists(metadataPath); + return result; + } + + public static void notifyRecipient(Path escortFile, Logger logger, Message msg) { + if (Files.exists(escortFile)) { + try { + BasicDBObject downloadable = EscortFiles.decode(escortFile); + String recipientKey = "recipient"; + if (downloadable.containsKey(recipientKey)) { + String recipient = downloadable.getString(recipientKey); + logger.info(new MediaCubeMarker(recipient, "MediaCube rendszerüzenet"), msg); + } + } catch (Exception e) { + logger.catching(e); + } + + } + } + + public static void remove(Path file) { + try { + file.toFile().delete(); + } catch (Exception e) { + logger.error("Unable to delete {}", file.toAbsolutePath().toString()); + } + } + + public static void remove(List files) { + if (files != null) + files.forEach(f -> remove(f)); + } + + public static void removeCatchedFile(Path escortFile) { + remove(Paths.get(escortFile.toString() + DOT_CATCHED)); + } + + public static List getKillDateFiles(Path filePath) { + String killDateFilePattern = String.format("%s.*%s", filePath.getFileName().toString(), KILLDATE_EXT); + List result = new ArrayList<>(); + try { + Path statusPath = Paths.get(filePath.getParent().toString(), STATUSFOLDER); + File statusPathFile = statusPath.toFile(); + if (statusPathFile.exists() && statusPathFile.isDirectory()) { + try (DirectoryStream stream = Files.newDirectoryStream(statusPath, killDateFilePattern)) { + stream.forEach(p -> result.add(p)); + } catch (Exception e) { + logger.catching(e); + } + } + Collections.sort(result); + } catch (Exception e) { + logger.catching(e); + } + return result; + } + + /*** + * A media eleresi utjan alapjan a .STATUS almappabol torli a .catch fajlt. + * + * @param mediaFile + * @throws IOException + */ + public static void removeMediaCatch(Path mediaFile) { + Path catchedFile = createMediaCatchFilePath(mediaFile); + remove(catchedFile); + } + + public static void setNEXIOKillDate(int killDateDays, String targetFileName, String nexioAgency, StoreUri targetUri) throws Exception { + OutputStream outStream = null; + try { + FTPClient targetFTP = ((FtpDirectoryLister) targetUri.getLister()).connect(); + Calendar killDate = CalendarUtils.createCalendar(new Date()); + killDate.add(Calendar.DAY_OF_YEAR, killDateDays); + if (targetFileName.toLowerCase().contains(".mxf")) + targetFileName = targetFileName.substring(0, targetFileName.lastIndexOf('.')); + byte[] killDateFile = EscortFiles.createNEXIOKillDateFile(targetFileName, killDate.getTime(), null, nexioAgency); + String xml = targetFileName + ".xml"; + outStream = targetFTP.storeFileStream(xml); + if (outStream == null) { + throw new NullPointerException( + "Can not open: " + targetFileName.substring(0, targetFileName.lastIndexOf('.')) + ".xml" + " Reply:" + targetFTP.getReplyString()); + } + outStream.write(killDateFile); + outStream.flush(); + } catch (Exception e) { + throw e; + } finally { + if (outStream != null) + outStream.close(); + targetUri.cleanUp(); + } + } + + private static byte[] xmlDocumentToString(Document xmlDocument) + throws TransformerFactoryConfigurationError, TransformerConfigurationException, TransformerException, IOException, UnsupportedEncodingException { + DOMSource domSource = new DOMSource(xmlDocument); + TransformerFactory tf = TransformerFactory.newInstance(); + Transformer transformer = tf.newTransformer(); + transformer.setOutputProperty(OutputKeys.METHOD, "xml"); + transformer.setOutputProperty(OutputKeys.ENCODING, "UTF-16"); + transformer.setOutputProperty(OutputKeys.INDENT, "yes"); + StringWriter sw = new StringWriter(); + StreamResult sr = new StreamResult(sw); + transformer.transform(domSource, sr); + String result = sw.toString(); + sw.close(); + return result.getBytes("UTF-16"); + } + +} diff --git a/server/-product/production/HIRTV/jobs/steps/shared/ExternalCommand.java b/server/-product/production/HIRTV/jobs/steps/shared/ExternalCommand.java new file mode 100644 index 00000000..307f36d0 --- /dev/null +++ b/server/-product/production/HIRTV/jobs/steps/shared/ExternalCommand.java @@ -0,0 +1,79 @@ +package user.jobengine.server.steps.shared; + +import java.io.BufferedReader; +import java.io.InputStreamReader; +import java.util.ArrayList; +import java.util.List; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + +public class ExternalCommand { + private static final Logger logger = LogManager.getLogger(); + private ExternalProfile profile; + + public ExternalCommand(ExternalProfile profile) { + this.profile = profile; + } + + public String execute(String input, String output, boolean firstResponse, IExternalCallback responseCallBack) throws Exception { + List arguments = getArguments(input, output); + List command = new ArrayList<>(); + command.add(profile.getExecutable()); + command.addAll(arguments); + + ProcessBuilder processBuilder = new ProcessBuilder(); + processBuilder.command(command); + + String result = null; + try { + logger.info("Executing : {}", processBuilder.command()); + + Process process = processBuilder.start(); + try (BufferedReader reader = new BufferedReader(new InputStreamReader(process.getInputStream()))) { + String line = null; + while ((line = reader.readLine()) != null) { + logger.debug("Process response: {}", line); + if (responseCallBack != null) + responseCallBack.onResponse(line); + //System.out.println(line); + if (line != null && line.length() > 0) { + result = line; + if (firstResponse) + break; + } + } + int exitCode = process.waitFor(); + if (exitCode != 0) { + StringBuilder msg = new StringBuilder(); + try (BufferedReader errReader = new BufferedReader(new InputStreamReader(process.getErrorStream()))) { + String errline = null; + while ((errline = errReader.readLine()) != null) { + msg.append(errline); + } + } catch (Exception ex) { + } + + throw new Exception("Exited with error code : " + exitCode + ". " + msg); + } + } catch (Exception e) { + throw e; + } + } catch (Exception e) { + logger.error(e); + throw e; + } + + return result; + } + + private List getArguments(String input, String output) { + List result = new ArrayList<>(); + + profile.getArguments().forEach(i -> { + result.add(i.replace("%i", input).replace("%o", output)); + }); + return result; + } + +} \ No newline at end of file diff --git a/server/-product/production/HIRTV/jobs/steps/shared/ExternalCommandExecutor.java b/server/-product/production/HIRTV/jobs/steps/shared/ExternalCommandExecutor.java new file mode 100644 index 00000000..a34ff7f0 --- /dev/null +++ b/server/-product/production/HIRTV/jobs/steps/shared/ExternalCommandExecutor.java @@ -0,0 +1,32 @@ +package user.jobengine.server.steps.shared; + +import user.commons.configuration.SystemConfiguration; + +public class ExternalCommandExecutor { + + public void execute(String profileName, String input, String output, IExternalCallback responseCallBack) throws Exception { + ExternalCommand externalCommand = getExternalCommand(profileName); + externalCommand.execute(input, output, false, responseCallBack); + } + + private ExternalCommand getExternalCommand(String profileName) throws Exception { + ExternalProfilesConfig config = SystemConfiguration.getInstance().load("settings/external-commands.yaml", ExternalProfilesConfig.class); + + if (config == null) + throw new Exception("Missing external-commands.yaml configuration"); + + ExternalProfile selectedProfile = null; + for (ExternalProfile profile : config.getProfiles()) { + if (profileName.equals(profile.getName())) { + selectedProfile = profile; + break; + } + } + + if (selectedProfile == null) + throw new Exception("Missing profile " + profileName + " in external-commands.yaml configuration"); + + return new ExternalCommand(selectedProfile); + } + +} diff --git a/server/-product/production/HIRTV/jobs/steps/shared/ExternalProfile.java b/server/-product/production/HIRTV/jobs/steps/shared/ExternalProfile.java new file mode 100644 index 00000000..68e22f4c --- /dev/null +++ b/server/-product/production/HIRTV/jobs/steps/shared/ExternalProfile.java @@ -0,0 +1,33 @@ +package user.jobengine.server.steps.shared; + +import java.util.List; + +public class ExternalProfile { + private String executable; + private String name; + private List arguments; + + public List getArguments() { + return arguments; + } + + public String getExecutable() { + return executable; + } + + public String getName() { + return name; + } + + public void setArguments(List arguments) { + this.arguments = arguments; + } + + public void setExecutable(String executable) { + this.executable = executable; + } + + public void setName(String name) { + this.name = name; + } +} \ No newline at end of file diff --git a/server/-product/production/HIRTV/jobs/steps/shared/ExternalProfilesConfig.java b/server/-product/production/HIRTV/jobs/steps/shared/ExternalProfilesConfig.java new file mode 100644 index 00000000..7ef77291 --- /dev/null +++ b/server/-product/production/HIRTV/jobs/steps/shared/ExternalProfilesConfig.java @@ -0,0 +1,15 @@ +package user.jobengine.server.steps.shared; + +import java.util.List; + +public class ExternalProfilesConfig { + private List profiles; + + public List getProfiles() { + return profiles; + } + + public void setProfiles(List profiles) { + this.profiles = profiles; + } +} \ No newline at end of file diff --git a/server/-product/production/HIRTV/jobs/steps/shared/FFMpeg.java b/server/-product/production/HIRTV/jobs/steps/shared/FFMpeg.java new file mode 100644 index 00000000..e4c433bd --- /dev/null +++ b/server/-product/production/HIRTV/jobs/steps/shared/FFMpeg.java @@ -0,0 +1,225 @@ +package user.jobengine.server.steps.shared; + +import java.io.FileNotFoundException; +import java.nio.file.Files; +import java.nio.file.Paths; +import java.util.ArrayList; +import java.util.List; + +import org.apache.commons.lang.StringUtils; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + +import user.commons.mediaarea.MediaArea; + +public class FFMpeg { + + public interface IProgressChanged { + void onProgressChanged(long progress); + } + + private static final String FPS = "fps="; + private static final String FRAME = "frame="; + + private static final Logger logger = LogManager.getLogger(); + + //@formatter:off + public static void concatenate(String chunklist, String output) { + ProcessBuilder processBuilder = Cmd.create( + Cmd.getFFMpegExecutable(), + "-f", "concat", + "-safe", "0", + "-i", chunklist, + "-vcodec", "copy", "-acodec", "libmp3lame", + "-y", "-v", "error", "-stats", output); + + Cmd.execute(processBuilder, false); + } + + //@formatter:off + public static void encode(String input, String output, double from, double length) { + ProcessBuilder processBuilder = Cmd.create( + Cmd.getFFMpegExecutable(), + "-ss", String.valueOf(from), + "-t", String.valueOf(length), + "-i", input, + "-vcodec", "libx264", "-acodec", "ac3", + "-movflags", "+faststart", + "-vf", "scale=-2:480,format=yuv420p", + "-y", "-v", "error", "-stats", "-f", "mp4", output); + + Cmd.execute(processBuilder, false); + } + + //@formatter:off + static public void hls_audio4ch(String input, String output, IProgressChanged progressCallback) throws Exception { + StringBuilder sb = new StringBuilder(); + sb.append("-v panic -stats -y") + .append("\r\n") + .append(String.format("-i %s", input)) + .append("\r\n") + .append("-map 0:0 -c:v h264 -an -hls_time 100000000000 -hls_list_size 0") + .append("\r\n") + .append(String.format("-f hls %s/video.m3u8", output)) + .append("\r\n"); + + for (int i = 0; i < 4; i++) { + sb.append(String.format("-map 0:%d -f segment -segment_time 100000000000 -segment_list_size 0", i+1)) + .append("\r\n") + .append(String.format("-segment_list %s/audio%d.m3u8 -segment_format mpegts %s/audio%d%%d.aac", output, i, output, i)) + .append("\r\n"); + } + System.out.println(sb); + ProcessBuilder processBuilder = Cmd.create(Cmd.getFFMpegExecutable(), sb); + long[] allFrames = new long[]{0}; + + try { + MediaArea mi = new MediaArea(Paths.get(input)); + mi.process(); + allFrames[0] = mi.getFrameCount(); + } catch (Exception e ){ + System.err.println(e); + + } + + Cmd.execute(processBuilder, false, l -> { + if (allFrames[0] == 0) { + logger.debug(l); + System.out.println(l); + return; + } + + if (progressCallback == null) + return; + if (l.contains(FRAME) && l.contains(FPS)) { + String p = StringUtils.substringBetween(l, FRAME, FPS); + if (p != null) { + p = p.trim(); + try { + int currentFrames = Integer.parseInt(p); + progressCallback.onProgressChanged((long)currentFrames * 100 / allFrames[0]); + } catch (Exception e){} + } + } + }); + +// #EXTM3U +// #EXT-X-MEDIA:TYPE=AUDIO,GROUP-ID="audio",NAME="CH1",URI="audio0.m3u8" +// #EXT-X-MEDIA:TYPE=AUDIO,GROUP-ID="audio",NAME="CH2",URI="audio1.m3u8" +// #EXT-X-MEDIA:TYPE=AUDIO,GROUP-ID="audio",NAME="CH3",URI="audio2.m3u8" +// #EXT-X-MEDIA:TYPE=AUDIO,GROUP-ID="audio",NAME="CH4",URI="audio3.m3u8" +// #EXT-X-STREAM-INF:PROGRAM-ID=1,AUDIO="audio" +// video.m3u8 + + if (!Paths.get(output, "video.m3u8").toFile().exists()) + throw new FileNotFoundException("video.m3u8"); + if (!Paths.get(output, "video0.ts").toFile().exists()) + throw new FileNotFoundException("video0.ts"); + for (int i = 0; i < 4; i++) { + String file = String.format("audio%d.m3u8", i); + if (!Paths.get(output, file).toFile().exists()) + throw new FileNotFoundException(file); + file = String.format("audio%d0.aac", i); + if (!Paths.get(output, file).toFile().exists()) + throw new FileNotFoundException(file); + } + + List indexLines = new ArrayList<>(); + indexLines.add("#EXTM3U"); + for (int i = 0; i < 4; i++) { + indexLines.add(String.format("#EXT-X-MEDIA:TYPE=AUDIO,GROUP-ID=\"audio\",NAME=\"CH%d\",URI=\"audio%d.m3u8\"", i+1, i)); + } + indexLines.add("#EXT-X-STREAM-INF:PROGRAM-ID=1,AUDIO=\"audio\""); + indexLines.add("video.m3u8"); + + Files.write(Paths.get(output, "index.m3u8"), indexLines); + } + + + + static public void withProfile(String input, String output, String profile, IProgressChanged progressCallback) throws Exception { + + + StringBuilder sb = new StringBuilder(); + sb.append("-v panic -stats -y") + .append("\r\n") + .append(String.format("-i %s", input)) + .append("\r\n") + .append("-map 0:0 -c:v h264 -an -hls_time 100000000000 -hls_list_size 0") + .append("\r\n") + .append(String.format("-f hls %s/video.m3u8", output)) + .append("\r\n"); + + for (int i = 0; i < 4; i++) { + sb.append(String.format("-map 0:%d -f segment -segment_time 100000000000 -segment_list_size 0", i+1)) + .append("\r\n") + .append(String.format("-segment_list %s/audio%d.m3u8 -segment_format mpegts %s/audio%d%%d.aac", output, i, output, i)) + .append("\r\n"); + } + System.out.println(sb); + ProcessBuilder processBuilder = Cmd.create(Cmd.getFFMpegExecutable(), sb); + long[] allFrames = new long[]{0}; + + try { + MediaArea mi = new MediaArea(Paths.get(input)); + mi.process(); + allFrames[0] = mi.getFrameCount(); + } catch (Exception e ){ + System.err.println(e); + + } + + Cmd.execute(processBuilder, false, l -> { + if (allFrames[0] == 0) { + logger.debug(l); + System.out.println(l); + return; + } + + if (progressCallback == null) + return; + if (l.contains(FRAME) && l.contains(FPS)) { + String p = StringUtils.substringBetween(l, FRAME, FPS); + if (p != null) { + p = p.trim(); + try { + int currentFrames = Integer.parseInt(p); + progressCallback.onProgressChanged((long)currentFrames * 100 / allFrames[0]); + } catch (Exception e){} + } + } + }); + +// #EXTM3U +// #EXT-X-MEDIA:TYPE=AUDIO,GROUP-ID="audio",NAME="CH1",URI="audio0.m3u8" +// #EXT-X-MEDIA:TYPE=AUDIO,GROUP-ID="audio",NAME="CH2",URI="audio1.m3u8" +// #EXT-X-MEDIA:TYPE=AUDIO,GROUP-ID="audio",NAME="CH3",URI="audio2.m3u8" +// #EXT-X-MEDIA:TYPE=AUDIO,GROUP-ID="audio",NAME="CH4",URI="audio3.m3u8" +// #EXT-X-STREAM-INF:PROGRAM-ID=1,AUDIO="audio" +// video.m3u8 + + if (!Paths.get(output, "video.m3u8").toFile().exists()) + throw new FileNotFoundException("video.m3u8"); + if (!Paths.get(output, "video0.ts").toFile().exists()) + throw new FileNotFoundException("video0.ts"); + for (int i = 0; i < 4; i++) { + String file = String.format("audio%d.m3u8", i); + if (!Paths.get(output, file).toFile().exists()) + throw new FileNotFoundException(file); + file = String.format("audio%d0.aac", i); + if (!Paths.get(output, file).toFile().exists()) + throw new FileNotFoundException(file); + } + + List indexLines = new ArrayList<>(); + indexLines.add("#EXTM3U"); + for (int i = 0; i < 4; i++) { + indexLines.add(String.format("#EXT-X-MEDIA:TYPE=AUDIO,GROUP-ID=\"audio\",NAME=\"CH%d\",URI=\"audio%d.m3u8\"", i+1, i)); + } + indexLines.add("#EXT-X-STREAM-INF:PROGRAM-ID=1,AUDIO=\"audio\""); + indexLines.add("video.m3u8"); + + Files.write(Paths.get(output, "index.m3u8"), indexLines); + } + +} diff --git a/server/-product/production/HIRTV/jobs/steps/shared/FileSearchFilterOptions.java b/server/-product/production/HIRTV/jobs/steps/shared/FileSearchFilterOptions.java new file mode 100644 index 00000000..b5a8d0e8 --- /dev/null +++ b/server/-product/production/HIRTV/jobs/steps/shared/FileSearchFilterOptions.java @@ -0,0 +1,41 @@ +package user.jobengine.server.steps.shared; + +import java.nio.file.Path; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +import com.ibm.nosql.json.api.BasicDBObject; + +public class FileSearchFilterOptions { + + private BasicDBObject filter; + + public FileSearchFilterOptions(BasicDBObject filter) { + this.filter = filter; + } + + public boolean acceptFile(Path file) { + if (filter == null) + return true; + + if (filter.containsKey("fileName")) { + //.*\.(sh|ini|conf|vhost|xml|php)$ + String fileNamePattern = filter.getString("fileName"); + if (fileNamePattern == null || fileNamePattern.trim().length() == 0) + return true; + + Pattern pattern = Pattern.compile(fileNamePattern, Pattern.CASE_INSENSITIVE); + + Matcher matcher = pattern.matcher(file.getFileName().toString()); + if (matcher.find()) + return true; + + } + + return false; + } + + public boolean preAcceptDirectory(Path file) { + return true; + } +} diff --git a/server/-product/production/HIRTV/jobs/steps/shared/IExternalCallback.java b/server/-product/production/HIRTV/jobs/steps/shared/IExternalCallback.java new file mode 100644 index 00000000..1e071913 --- /dev/null +++ b/server/-product/production/HIRTV/jobs/steps/shared/IExternalCallback.java @@ -0,0 +1,5 @@ +package user.jobengine.server.steps.shared; + +public interface IExternalCallback { + void onResponse(String data); +} \ No newline at end of file diff --git a/server/-product/production/HIRTV/jobs/steps/shared/ItemManagerExtensions.java b/server/-product/production/HIRTV/jobs/steps/shared/ItemManagerExtensions.java new file mode 100644 index 00000000..70ec476b --- /dev/null +++ b/server/-product/production/HIRTV/jobs/steps/shared/ItemManagerExtensions.java @@ -0,0 +1,95 @@ +package user.jobengine.server.steps.shared; + +import java.nio.file.Path; + +import com.ibm.nosql.json.api.BasicDBObject; + +import user.commons.RemoteFile; +import user.commons.StoreUri; +import user.commons.remotestore.RemoteStoreProtocol; +import user.jobengine.db.IItemManager; +import user.jobengine.db.IResultSetConsumer; +import user.jobengine.db.Store; + +public class ItemManagerExtensions { + + public static BasicDBObject getArchiveInfo(IItemManager manager, long houseid) { + final BasicDBObject[] result = new BasicDBObject[] { null }; + StringBuilder query = new StringBuilder(); + query.append("select count(*) as count, sum(length) as duration FROM media"); + query.append(" "); + query.append(String.format("where houseid='%d' and itemtypeid = 82", houseid)); + query.append(" "); + query.append("group by houseid"); + IResultSetConsumer consumer = rs -> { + BasicDBObject o = new BasicDBObject(); + o.put("count", rs.getLong("count")); + o.put("duration", rs.getLong("duration")); + result[0] = o; + return false; + }; + manager.executeQuery(query.toString(), consumer, null); + return result[0]; + } + + public static long getExistingRundownMedia(IItemManager manager, String houseid) { + final long[] result = new long[] { 0 }; + final String[] idToCheck = new String[] { houseid }; + int pos = houseid.lastIndexOf("-"); + // a hivas a CopyForArchiveNEXIOMaterialsStep-bol is johet, ott meg nincs + // idobelyegezve a nev! + if (pos > 0 && houseid.length() - pos > 4) + idToCheck[0] = houseid.substring(0, pos); + MetadataType metadataType = MetadataTypeDetector.GuessMetadataType(idToCheck[0]); + if (metadataType == MetadataType.OctopusPlaceholder) { + StringBuilder query = new StringBuilder(); + query.append("select mediaid, mediafilehouseid, filename"); + query.append(" "); + query.append(String.format("from vw_rundown_items where mediafilehouseid like '%s%%'", idToCheck[0])); + query.append(" "); + query.append("order by filename, mediaid"); + IResultSetConsumer consumer = rs -> { + String fileName = rs.getString("filename"); + if (idToCheck[0].equals(fileName)) { + result[0] = rs.getLong("mediaid"); + return false; + } else + return true; + }; + manager.executeQuery(query.toString(), consumer, null); + } + + return result[0]; + } + + static public boolean isArchived(IItemManager manager, Path filePath) { + boolean result = false; + String name = filePath.getFileName().toString(); + String[] tsmName = new String[] { null }; + String query = String.format("SELECT relativepath FROM MEDIAFILE WHERE houseid = '%s'", name); + manager.executeQuery(query, rs -> { + tsmName[0] = rs.getString("relativepath"); + return false; + }, null); + + Store tsmStore = manager.getSystemStore(false); + if (tsmStore == null) + throw new NullPointerException("A TSM bejegyzés nem található!"); + + StoreUri tsmStoreUri = tsmStore.getSourceStoreUri(RemoteStoreProtocol.TSM); + if (tsmStoreUri == null) + throw new NullPointerException("A TSM forrás elérése nem található!"); + + if (tsmName[0] != null) { + try { + RemoteFile remoteFile = tsmStoreUri.getRemoteFile(tsmName[0]); + result = remoteFile != null; + } catch (Exception e) { + result = false; + } finally { + tsmStoreUri.cleanUp(); + } + } + return result; + } +} diff --git a/server/-product/production/HIRTV/jobs/steps/shared/MediaCubeClient.java b/server/-product/production/HIRTV/jobs/steps/shared/MediaCubeClient.java new file mode 100644 index 00000000..0915cc33 --- /dev/null +++ b/server/-product/production/HIRTV/jobs/steps/shared/MediaCubeClient.java @@ -0,0 +1,77 @@ +package user.jobengine.server.steps.shared; + +import javax.ws.rs.client.Entity; +import javax.ws.rs.client.Invocation.Builder; +import javax.ws.rs.core.MediaType; +import javax.ws.rs.core.MultivaluedMap; +import javax.ws.rs.core.Response; +import javax.ws.rs.core.Response.Status; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.jboss.resteasy.client.jaxrs.ResteasyClient; +import org.jboss.resteasy.client.jaxrs.ResteasyClientBuilder; +import org.jboss.resteasy.client.jaxrs.ResteasyWebTarget; +import org.jboss.resteasy.specimpl.MultivaluedMapImpl; + +import com.ibm.nosql.json.JSONUtil; +import com.ibm.nosql.json.api.BasicDBObject; + +public class MediaCubeClient { + private static Logger logger = LogManager.getLogger(); + private ResteasyWebTarget webTarget; + + public MediaCubeClient(String address) { + ResteasyClient client = new ResteasyClientBuilder().build(); + webTarget = client.target(address); + } + + BasicDBObject getDbObject(String json) { + BasicDBObject result = (BasicDBObject) JSONUtil.jsonToDbObject(json); + + if (result == null) + throw new NullPointerException("API Result is null!"); + + if (result.containsKey("exception")) { + BasicDBObject e = (BasicDBObject) result.get("exception"); + throw new RuntimeException(e.getString("message")); + } + //{"exception":{"message":"Invalid credentials.","publicName":"AuthenticationFailedException"}} + return result; + } + + public BasicDBObject getStatus(long jobId) { + MultivaluedMap vars = new MultivaluedMapImpl<>(); + vars.add("jobId", jobId); + Response response = query("services/rest/jobengine/jobstatus", vars).get(); + if (response.getStatus() != Status.OK.getStatusCode()) { + logger.error(response.readEntity(String.class)); + System.out.println(response.readEntity(String.class)); + return null; + } + String result = response.readEntity(String.class); + return getDbObject(result); + } + + private Builder query(String path, MultivaluedMap vars) { + ResteasyWebTarget target = webTarget.path(path).queryParams(vars); + Builder result = target.request(); + return result; + } + + public long startjob(String template, String name, BasicDBObject jobParams) throws Exception { + MultivaluedMap vars = new MultivaluedMapImpl<>(); + vars.add("template", template); + vars.add("name", name); + Response response = query("services/rest/jobengine/startjob", vars).post(Entity.entity(jobParams.toString(), MediaType.APPLICATION_JSON)); + + if (response.getStatus() != Status.OK.getStatusCode()) { + logger.error(response.readEntity(String.class)); + return 0; + } + + String resultObject = response.readEntity(String.class); + return Long.parseLong(resultObject); + } + +} diff --git a/server/-product/production/HIRTV/jobs/steps/shared/MetadataSaver.java b/server/-product/production/HIRTV/jobs/steps/shared/MetadataSaver.java new file mode 100644 index 00000000..fca71538 --- /dev/null +++ b/server/-product/production/HIRTV/jobs/steps/shared/MetadataSaver.java @@ -0,0 +1,44 @@ +package user.jobengine.server.steps.shared; + +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + +import com.ibm.nosql.json.api.BasicDBObject; +import com.ibm.nosql.json.api.DB; +import com.ibm.nosql.json.api.DBCollection; + +import user.commons.mediaarea.MediaArea; +import user.commons.nosql.NoSQLUtils; + +public class MetadataSaver { + + private static final Logger logger = LogManager.getLogger(); + + public static void saveMetadata(String file) { + try { + Path filePath = Paths.get(file); + logger.info("Analize: {}", filePath); + if (!Files.exists(filePath)) + return; + + String fileName = filePath.getFileName().toString(); + DB db = NoSQLUtils.getNoSQLDB(); + DBCollection collection = db.getCollection("metadata"); + long existing = collection.find(new BasicDBObject("fileName", fileName)).count(); + if (existing > 0) + return; + + MediaArea mediaArea = new MediaArea(filePath); + mediaArea.process(); + BasicDBObject metadata = mediaArea.getInform(); + collection.save(metadata); + logger.info("Metadata saved for: {}", filePath); + } catch (Exception e) { + logger.error("Can't analyze input {}, skipping. System message is: {}", e.getMessage()); + } + } +} diff --git a/server/-product/production/HIRTV/jobs/steps/shared/MetadataType.java b/server/-product/production/HIRTV/jobs/steps/shared/MetadataType.java new file mode 100644 index 00000000..0e27bf34 --- /dev/null +++ b/server/-product/production/HIRTV/jobs/steps/shared/MetadataType.java @@ -0,0 +1,5 @@ +package user.jobengine.server.steps.shared; + +public enum MetadataType { + TrafficMaterial, TrafficPromo, TrafficAD, OctopusStory, OctopusPlaceholder, Generic +} diff --git a/server/-product/production/HIRTV/jobs/steps/shared/MetadataTypeDetector.java b/server/-product/production/HIRTV/jobs/steps/shared/MetadataTypeDetector.java new file mode 100644 index 00000000..9e9e6be9 --- /dev/null +++ b/server/-product/production/HIRTV/jobs/steps/shared/MetadataTypeDetector.java @@ -0,0 +1,49 @@ +package user.jobengine.server.steps.shared; + +import org.apache.commons.lang.StringUtils; + +public class MetadataTypeDetector { + + private static final String HYPHEN = "-"; + private static final String DOT = "."; + + private static final String REGEXP_TRAFFICMATERIALID = "^M{1}[0-9]{6}[A-Z]{1}"; + private static final String REGEXP_TRAFFICADID = "^R{1}[0-9]{6}[A-Z]{1}"; + private static final String REGEXP_TRAFFICPROMOID = "^P{1}[0-9]{6}[A-Z]{1}"; + private static final String REGEXP_OCTOPUSSTORYID = "^[0-9]+"; + private static final String REGEXP_OCTOPUSPLACEHOLDERID = "^[0-9]+_[0-9]+"; + private static final String REGEXP_OCTOPUSPLACEHOLDERVERSIONEDID = "^[0-9]+_[0-9]+-[0-9]{3}"; + + public static MetadataType GuessMetadataType(String id) { + if (StringUtils.isBlank(id)) + return MetadataType.Generic; + if (id.matches(REGEXP_TRAFFICMATERIALID)) + return MetadataType.TrafficMaterial; + if (id.matches(REGEXP_TRAFFICPROMOID)) + return MetadataType.TrafficPromo; + if (id.matches(REGEXP_TRAFFICADID)) + return MetadataType.TrafficAD; + if (id.matches(REGEXP_OCTOPUSSTORYID)) + return MetadataType.OctopusStory; + if (id.matches(REGEXP_OCTOPUSPLACEHOLDERID)) + return MetadataType.OctopusPlaceholder; + if (id.matches(REGEXP_OCTOPUSPLACEHOLDERVERSIONEDID)) + return MetadataType.OctopusPlaceholder; + return MetadataType.Generic; + } + + public static String truncateExtension(String name) { + String result = name; + if (result != null && result.contains(DOT)) + result = result.substring(0, result.lastIndexOf(DOT)); + return result; + } + + public static String truncateVersion(String name) { + String result = name; + if (result != null && result.contains(HYPHEN)) + result = result.split(HYPHEN)[0]; + return result; + } + +} diff --git a/server/-product/production/HIRTV/jobs/steps/shared/OctopusDataMiner.java b/server/-product/production/HIRTV/jobs/steps/shared/OctopusDataMiner.java new file mode 100644 index 00000000..368fff69 --- /dev/null +++ b/server/-product/production/HIRTV/jobs/steps/shared/OctopusDataMiner.java @@ -0,0 +1,759 @@ +package user.jobengine.server.steps.shared; + +import java.time.LocalDate; +import java.time.Period; +import java.time.format.DateTimeFormatter; +import java.util.Calendar; +import java.util.Date; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; + +import javax.swing.event.EventListenerList; +import javax.ws.rs.client.Entity; +import javax.ws.rs.client.Invocation.Builder; +import javax.ws.rs.core.MediaType; +import javax.ws.rs.core.Response; + +import org.apache.commons.lang.StringUtils; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.jboss.resteasy.client.jaxrs.BasicAuthentication; +import org.jboss.resteasy.client.jaxrs.ResteasyClient; +import org.jboss.resteasy.client.jaxrs.ResteasyClientBuilder; +import org.jboss.resteasy.client.jaxrs.ResteasyWebTarget; +import org.joda.time.DateTime; + +import com.ibm.nosql.json.JSONUtil; +import com.ibm.nosql.json.api.BasicDBList; +import com.ibm.nosql.json.api.BasicDBObject; +import com.ibm.nosql.json.api.DB; +import com.ibm.nosql.json.api.DBCollection; +import com.ibm.nosql.json.api.DBCursor; + +import user.commons.CalendarUtils; +import user.commons.ListUtils; +import user.commons.nosql.NoSQLUtils; +import user.commons.octopus.IOctopusAPI; +import user.commons.remotestore.IProgressEventListener; +import user.commons.remotestore.ProgressEvent; + +public class OctopusDataMiner { + private static final String MOS_ABSTRACT = ""; + private static final String MOS_ABSTRACT_END = ""; + private static final Logger logger = LogManager.getLogger(); + private static final String ARCHIVED = "archived"; + private static final String SCHEDULEDSTART = "scheduledStart"; + private static final String FILTER = "filter"; + private static final String LINEFEED = "\r\n"; + private static final String SIMPLE_LINEFEED = "\n"; + private static final String SAVING_STORY_ID = "Saving story {}"; + private static final String FIELDS_STORIES = "id,name,modified,type,format,mosObjects,script,scheduleFrom,scheduleTo,customColumns,CustomColumn.label,CustomColumn.value,reporters,User.longName,User.userName,descriptions,StoryDescription.text"; + private static final String FIELDS_RUNDOWN = "id,name,modified,scheduledStart,channel,Channel.name,rundownType,RundownType.name"; + public static final String FIELDS_RUNDOWN_STORYIDS = "id,slugs,Slug.storyId,Slug.position"; + public static final String SLUGS = "slugs"; + private static final String FIELDS_STORYFOLDER = "id,name,modified"; + public static final String FIELDS_STORYFOLDER_STORYIDS = "id,stories,Story.id"; + private static final String RUNDOWN = "Rundown"; + private static final String FIELDS = "fields"; + private static final String EXIT = "Exit"; + private static final String RESULT = "result"; + private static final String STORY_FOLDER = "StoryFolder"; + private static final String ENTER = "Enter"; + private static final String FINISHED = "Finished"; + private static final String STARTING = "Starting"; + private static final String MOSLABEL = "MOS: "; + private static final Object STORY = "Story"; + + private DB db; + private ResteasyWebTarget webTarget; + private HashSet storyIDs = new HashSet<>(); + private HashSet folderIDs = new HashSet<>(); + private HashSet rundownIDs = new HashSet<>(); + private EventListenerList progressListenerList; + private ProgressEvent progressEvent = new ProgressEvent(this, 0); + private Map storyRundowns; + private Map storyStoryFolders; + private Map currentRundowns; + private Map currentFolder; + private Map currentStories; + private boolean includeArchived; + private Calendar zeroDate = CalendarUtils.createCalendar(2017, 11, 4); + private int objectCount; + private int currentObjectIndex; + private boolean disableWrite = false; + private int maxPastDays; + + public OctopusDataMiner(String address, String user, String pwd) { + db = NoSQLUtils.getNoSQLDB(); + ResteasyClient client = new ResteasyClientBuilder().build(); + webTarget = client.target(address).register(new BasicAuthentication(user, pwd)); + } + + public OctopusDataMiner(String address, String user, String pwd, int maxPastDays) { + this(address, user, pwd); + this.maxPastDays = maxPastDays; + } + + public void addProgressListener(IProgressEventListener listener) { + if (progressListenerList == null) + progressListenerList = new EventListenerList(); + progressListenerList.add(IProgressEventListener.class, listener); + } + + private Map buildFolderReferences(BasicDBList storyFolders) { + Map result = new HashMap<>(); + List storyFolderList = NoSQLUtils.asList(storyFolders); + for (BasicDBObject storyFolder : storyFolderList) { + if (storyFolder == null || !storyFolder.containsKey(IOctopusAPI.ID)) + continue; + BasicDBObject storyFolderWithStoryIds = queryStoryFolder(storyFolder, FIELDS_STORYFOLDER_STORYIDS); + long storyFolderId = storyFolderWithStoryIds.getLong(IOctopusAPI.ID); + List stories = NoSQLUtils.asList(storyFolderWithStoryIds, IOctopusAPI.STORIES); + if (stories == null) + continue; + + folderIDs.add(storyFolderId); + + long position = 1; + for (BasicDBObject story : stories) { + long storyId = story.getLong(IOctopusAPI.ID); + + storyIDs.add(storyId); + + BasicDBList references = result.get(storyId); + if (references == null) { + references = new BasicDBList(); + result.put(storyId, references); + } + references.add(new BasicDBObject(IOctopusAPI.ID, storyFolderId).append(IOctopusAPI.POSITION, position++)); + } + } + return result; + } + + private Map buildRundownReferences(BasicDBList rundowns) { + Map result = new HashMap<>(); + List rundownsList = NoSQLUtils.asList(rundowns); + for (BasicDBObject rundown : rundownsList) { + if (rundown == null || !rundown.containsKey(IOctopusAPI.ID)) + continue; + + logger.info("Processing rundown {}", rundown.get(IOctopusAPI.NAME)); + BasicDBObject rundownWithStoryids = queryRundown(rundown, FIELDS_RUNDOWN_STORYIDS); + // TODO ez neha null? + long rundownId = rundownWithStoryids.getLong(IOctopusAPI.ID); + List slugs = NoSQLUtils.asList(rundownWithStoryids, IOctopusAPI.SLUGS); + if (slugs == null) + continue; + + rundownIDs.add(rundownId); + + for (BasicDBObject slug : slugs) { + if (!slug.containsKey(IOctopusAPI.STORYID)) + continue; + long storyId = slug.getLong(IOctopusAPI.STORYID); + + storyIDs.add(storyId); + + BasicDBList references = result.get(storyId); + if (references == null) { + references = new BasicDBList(); + result.put(storyId, references); + } + long position = slug.getLong(IOctopusAPI.POSITION); + if (slug.containsKey(IOctopusAPI.POSITION)) + position = slug.getLong(IOctopusAPI.POSITION); + references.add(new BasicDBObject(IOctopusAPI.ID, rundownId).append(IOctopusAPI.POSITION, position)); + } + logger.info("Actual story amount {}", storyIDs.size()); + } + return result; + } + + public void clear() { + db.getCollection(IOctopusAPI.RUNDOWN_COLLECTION).remove(); + db.getCollection(IOctopusAPI.FOLDER_COLLECTION).remove(); + db.getCollection(IOctopusAPI.STORY_COLLECTION).remove(); + } + + private void deleteDiff(Set oldIDs, Set newIDs, String collectionName) { + if (oldIDs == null || oldIDs.size() == 0) + return; + if (newIDs != null && newIDs.size() > 0) + oldIDs.removeAll(newIDs); + + if (disableWrite) + return; + + DBCollection collection = db.getCollection(collectionName); + for (long id : oldIDs) { + logger.info("Deleting {} from {}", id, collectionName); + collection.remove(new BasicDBObject(IOctopusAPI.ID, id)); + } + } + + public void execute(boolean includeArchived) throws Exception { + this.includeArchived = includeArchived; + logger.trace(STARTING); + // current = korábbi szinkronizálás + currentRundowns = getCurrentIDs(IOctopusAPI.RUNDOWN_COLLECTION); + currentFolder = getCurrentIDs(IOctopusAPI.FOLDER_COLLECTION); + currentStories = getCurrentIDs(IOctopusAPI.STORY_COLLECTION); + + BasicDBList rundowns = null; + BasicDBList storyFolders = null; + try { + rundowns = queryBuildRefRundowns(); + storyFolders = queryBuildRefFolders(); + } catch (Exception e) { + logger.catching(e); + throw e; + } + + objectCount = rundownIDs.size() + folderIDs.size() + storyIDs.size(); + + storeStories(); + storeRundowns(rundowns); + storeStoryFolders(storyFolders); + + deleteDiff(currentRundowns.keySet(), rundownIDs, IOctopusAPI.RUNDOWN_COLLECTION); + deleteDiff(currentFolder.keySet(), folderIDs, IOctopusAPI.FOLDER_COLLECTION); + deleteDiff(currentStories.keySet(), storyIDs, IOctopusAPI.STORY_COLLECTION); + logger.trace(FINISHED); + } + + private String extractContent(BasicDBObject content) { + String scriptContent = ""; + if (!content.containsKey(IOctopusAPI.TYPE)) + return scriptContent; + String type = content.getString(IOctopusAPI.TYPE); + switch (type) { + case IOctopusAPI.TEXT: { + if (content.containsKey(IOctopusAPI.TEXT)) { + String text = content.getString(IOctopusAPI.TEXT); + if (text != null) + scriptContent += String.format("%s%s", text.replaceAll(SIMPLE_LINEFEED, LINEFEED), LINEFEED); + } + break; + } + case IOctopusAPI.MOS: { + BasicDBObject mosObject = NoSQLUtils.asDBObject(content, IOctopusAPI.OBJECT); + if (mosObject != null && !mosObject.isEmpty()) { + String xml = NoSQLUtils.asString(mosObject, IOctopusAPI.XML); + if (xml != null) { + int pos1 = xml.indexOf(MOS_ABSTRACT) + MOS_ABSTRACT.length(); + int pos2 = xml.indexOf(MOS_ABSTRACT_END); + + if (pos1 > -1 && pos2 > -1) { + scriptContent += String.format("%s %s%s", MOSLABEL, xml.substring(pos1, pos2), LINEFEED); + } + } + } + break; + } + default: { + if (content.containsKey(IOctopusAPI.CONTENT)) { + List innerContents = NoSQLUtils.asList(content, IOctopusAPI.CONTENT); + if (innerContents != null) { + for (BasicDBObject actualInnerContent : innerContents) { + if (actualInnerContent != null && actualInnerContent.isEmpty()) + scriptContent += extractContent(actualInnerContent); + } + } + } + break; + } + } + return scriptContent; + } + + /*** + * Get all custom columns label/non-null-value pair as map + * + * @param story + * @return + */ + private BasicDBObject getCustomColumns(BasicDBObject story) { + BasicDBObject result = new BasicDBObject(); + List customColumns = NoSQLUtils.asList(story, IOctopusAPI.CUSTOM_COLUMNS); + if (customColumns == null) + return null; + for (BasicDBObject customColumn : customColumns) { + String currentName = NoSQLUtils.asString(customColumn, IOctopusAPI.LABEL); + if (currentName == null) + continue; + String currentValue = NoSQLUtils.asString(customColumn, IOctopusAPI.VALUE); + if (currentValue == null) + continue; + result.put(currentName, currentValue); + } + return result; + } + + private String getDescriptions(BasicDBObject story) { + StringBuilder result = new StringBuilder(); + List descriptions = NoSQLUtils.asList(story, IOctopusAPI.DESCRIPTIONS); + if (descriptions == null) + return null; + for (BasicDBObject description : descriptions) { + String currentDesc = NoSQLUtils.asString(description, IOctopusAPI.TEXT); + if (currentDesc == null) + continue; + result.append(currentDesc); + } + return result.toString(); + } + + private BasicDBList extractRelevantMOSObjects(BasicDBObject story) { + List mosObjects = NoSQLUtils.asList(story, IOctopusAPI.MOS_OBJECTS); + if (mosObjects == null) + return null; + + Map mosLabels = extractScriptMosObjectIDs(story); + // logger.info(Arrays.deepToString(new ArrayList<>(mosLabels.keySet()))); + BasicDBList result = null; + for (BasicDBObject mosObject : mosObjects) { + if (!mosObject.containsKey(IOctopusAPI.MOS_ID)) + continue; + String mosId = mosObject.getString(IOctopusAPI.MOS_ID); + if (!IOctopusAPI.NEXIO_MOS.equals(mosId)) + continue; + String objId = mosObject.getString(IOctopusAPI.OBJ_ID); + if (objId == null) + continue; + + // logger.info("MOS ID: {}", objId); + + MetadataType metadataType = MetadataTypeDetector.GuessMetadataType(objId); + if (!MetadataType.OctopusPlaceholder.equals(metadataType) && !MetadataType.OctopusStory.equals(metadataType)) + continue; + if (result == null) + result = new BasicDBList(); + + String label = mosLabels.get(objId); + if (StringUtils.isNotBlank(label)) + mosObject.append(IOctopusAPI.LABEL, label); + result.add(mosObject); + } + return result; + } + + private String extractScriptContent(BasicDBObject story) { + BasicDBObject script = NoSQLUtils.asDBObject(story, IOctopusAPI.SCRIPT); + if (script == null || script.isEmpty()) + return null; + + List body = NoSQLUtils.asList(script, IOctopusAPI.BODY); + if (body == null || body.size() == 0) + return null; + + StringBuilder sb = new StringBuilder(); + + for (BasicDBObject bodyItem : body) { + if (bodyItem.containsKey(IOctopusAPI.TYPE)) { + sb.append(bodyItem.getString(IOctopusAPI.TYPE)); + sb.append(LINEFEED); + } + if (bodyItem.containsKey(IOctopusAPI.LABEL)) { + sb.append(bodyItem.getString(IOctopusAPI.LABEL)); + sb.append(LINEFEED); + } + List contents = NoSQLUtils.asList(bodyItem, IOctopusAPI.CONTENT); + if (contents == null) + continue; + for (BasicDBObject content : contents) { + sb.append(extractContent(content)); + } + } + return sb.length() == 0 ? null : sb.toString(); + } + + private Map extractScriptMosObjectIDs(BasicDBObject story) { + BasicDBObject script = NoSQLUtils.asDBObject(story, IOctopusAPI.SCRIPT); + if (script == null || script.isEmpty()) + return null; + + List body = NoSQLUtils.asList(script, IOctopusAPI.BODY); + if (body == null || body.size() == 0) + return null; + + Map result = new HashMap<>(); + + for (BasicDBObject bodyItem : body) { + String label = bodyItem.getString(IOctopusAPI.LABEL); + if (StringUtils.isBlank(label)) + continue; + BasicDBObject clip = NoSQLUtils.asDBObject(bodyItem, IOctopusAPI.CLIP); + if (clip == null) + continue; + BasicDBObject obj = NoSQLUtils.asDBObject(clip, IOctopusAPI.OBJECT); + if (obj == null) + continue; + String mosId = obj.getString(IOctopusAPI.MOS_ID); + if (!IOctopusAPI.NEXIO_MOS.equals(mosId)) + continue; + String objId = obj.getString(IOctopusAPI.OBJ_ID); + if (objId == null) + continue; + result.put(objId, label); + + } + return result; + } + + private void fireProgressEvent() { + currentObjectIndex++; + // logger.info("currentObjectIndex {}", currentObjectIndex); + int progress = currentObjectIndex * 100 / objectCount; + if (progress != progressEvent.getProgress()) { + progressEvent.setProgress(progress); + fireProgressEvent(progressEvent); + } + } + + private void fireProgressEvent(ProgressEvent evt) { + logger.debug("Progress changed to " + evt.getProgress() + "%"); + if (progressListenerList == null) + return; + Object[] listeners = progressListenerList.getListenerList(); + for (int i = 0; i < listeners.length; i += 2) { + if (listeners[i] == IProgressEventListener.class) + ((IProgressEventListener) listeners[i + 1]).progressChanged(evt); + } + } + + public Map getCurrentIDs(String name) { + Map result = new HashMap<>(); + + DBCursor cursor = db.getCollection(name).find(new BasicDBObject(), new BasicDBObject(IOctopusAPI.ID, 1)); + if (cursor.hasNext()) { + List objects = ListUtils.cast(cursor.toArray()); + for (BasicDBObject obj : objects) { + if (obj == null) + continue; + long id = NoSQLUtils.asLong(obj, IOctopusAPI.ID); + if (id == 0) + continue; + Object _id = obj.getID(); + if (_id == null) + continue; + result.put(id, _id); + } + } + return result; + } + + BasicDBObject getDbObject(String json) { + BasicDBObject result = (BasicDBObject) JSONUtil.jsonToDbObject(json); + + if (result == null) + throw new NullPointerException("API Result is null!"); + + if (result.containsKey("exception")) { + BasicDBObject e = (BasicDBObject) result.get("exception"); + // throw new RuntimeException(e.getString("message")); + logger.error("Query error: {}", e.getString("message")); + return null; + } + return result; + } + + private Builder query(String path, String fields) { + ResteasyWebTarget target = webTarget.path(path).queryParam(FIELDS, fields); + Builder result = target.request(); + return result; + } + + private BasicDBList queryBuildRefFolders() { + BasicDBList storyFolders; + logger.info("Fetch story folders"); + storyFolders = queryStoryFolders(); + logger.info("Story folders amount {}", storyFolders.size()); + + logger.info("Fetch folder story references"); + storyStoryFolders = buildFolderReferences(storyFolders); + logger.info("Actual stories amount {}", storyIDs.size()); + return storyFolders; + } + + private BasicDBList queryBuildRefRundowns() { + BasicDBList rundowns; + logger.info("Fetch rundowns"); + rundowns = queryRundowns(); + logger.info("Rundowns amount {}", rundowns.size()); + + logger.info("Fetch rundown story references"); + storyRundowns = buildRundownReferences(rundowns); + logger.info("Actual stories amount {}", storyIDs.size()); + return rundowns; + } + + public BasicDBObject queryRundown(BasicDBObject rundown, String fields) { + logger.trace(ENTER); + BasicDBObject result = null; + long id = NoSQLUtils.asLong(rundown, IOctopusAPI.ID); + Response response = query(String.format("%s/%d", RUNDOWN, id), fields).get(); + String json = response.readEntity(String.class); + BasicDBObject resultObject = getDbObject(json); + if (resultObject == null) + logger.error("Rundown {} {} is not available", id, rundown.getString(IOctopusAPI.NAME)); + else + result = NoSQLUtils.asDBObject(resultObject, RESULT); + logger.trace(EXIT); + return result; + } + + public BasicDBList queryRundowns() { + logger.trace(ENTER); + BasicDBList result = null; + Builder query = query(RUNDOWN, FIELDS_RUNDOWN); + Response response = null; + if (includeArchived) { + response = query.post(Entity.entity(new BasicDBObject(FILTER, new BasicDBObject(ARCHIVED, true)).toString(), MediaType.APPLICATION_JSON)); + } else { + response = query.get(); + } + + String json = response.readEntity(String.class); + BasicDBObject resultObject = getDbObject(json); + if (resultObject != null) + result = NoSQLUtils.asDBList(resultObject, RESULT); + + if (maxPastDays > 0 && result != null) { + logger.info("Using maxPastDays {} limit", maxPastDays); + List rundownList = NoSQLUtils.asList(result); + result = new BasicDBList(); + // "2022-12-02T19:10:00.000+01:00" + LocalDate oldestScheduleDate = LocalDate.now().minus(Period.ofDays(maxPastDays + 1)); + for (BasicDBObject rundown : rundownList) { + String scheduledStart = rundown.getString(SCHEDULEDSTART); + if (scheduledStart == null) + continue; + LocalDate scheduleDate = LocalDate.parse(scheduledStart, DateTimeFormatter.ISO_OFFSET_DATE_TIME); + if (scheduleDate.isAfter(oldestScheduleDate)) { + result.add(rundown); + } + + } + } + + logger.trace(EXIT); + return result; + } + + public BasicDBObject queryStory(long storyID) { + logger.trace(ENTER); + BasicDBObject result = null; + Response response = query(String.format("%s/%d", STORY, storyID), FIELDS_STORIES).get(); + String json = response.readEntity(String.class); + BasicDBObject resultObject = getDbObject(json); + if (resultObject == null) + logger.error("Story {} is not available", storyID); + else + result = NoSQLUtils.asDBObject(resultObject, RESULT); + logger.trace(EXIT); + return result; + } + + public BasicDBObject queryStoryFolder(BasicDBObject storyFolder, String fields) { + logger.trace(ENTER); + BasicDBObject result = null; + long id = NoSQLUtils.asLong(storyFolder, IOctopusAPI.ID); + Response response = query(String.format("%s/%d", STORY_FOLDER, id), fields).get(); + String json = response.readEntity(String.class); + BasicDBObject resultObject = getDbObject(json); + if (resultObject == null) + logger.error("StoryFolder {} {} is not available", id, storyFolder.getString(IOctopusAPI.NAME)); + else + result = NoSQLUtils.asDBObject(resultObject, RESULT); + logger.trace(EXIT); + return result; + } + + public BasicDBList queryStoryFolders() { + logger.trace(ENTER); + BasicDBList result = null; + Response response = query(STORY_FOLDER, FIELDS_STORYFOLDER).get(); + String json = response.readEntity(String.class); + BasicDBObject resultObject = getDbObject(json); + if (resultObject != null) + result = NoSQLUtils.asDBList(resultObject, RESULT); + + // /* teszt */ + // List list = NoSQLUtils.asList(result); + // for (BasicDBObject actual : list) { + // String fullName = concatParentsToStoryFolder(actual, + // actual.getString(IOctopusAPI.NAME)); + // //logger.info("Checking StoryFolder {}", fullName); + // actual.remove(IOctopusAPI.NAME); + // actual.append(IOctopusAPI.NAME, fullName); + // } + // + logger.trace(EXIT); + return result; + } + + public void removeProgressListener(IProgressEventListener listener) { + progressListenerList.remove(IProgressEventListener.class, listener); + } + + void setObjectID(Map current, long id, BasicDBObject objectToSave) { + Object _id = current.get(id); + if (_id == null) + return; + objectToSave.put(IOctopusAPI._ID, _id); + } + + private void storeRundown(BasicDBObject rundown) { + logger.trace(ENTER); + String name = rundown.containsKey(IOctopusAPI.NAME) ? rundown.getString(IOctopusAPI.NAME) : null; + logger.debug("Storing rundown {} {}", name, rundown.get(IOctopusAPI.SCHEDULED_START)); + Date scheduledStart = toDate(rundown, IOctopusAPI.SCHEDULED_START); + if (scheduledStart != null && scheduledStart.after(zeroDate.getTime())) { + rundown.put(IOctopusAPI.SCHEDULED_START, toDate(rundown, IOctopusAPI.SCHEDULED_START)); + rundown.put(IOctopusAPI.MODIFIED, toDate(rundown, IOctopusAPI.MODIFIED)); + DBCollection collection = db.getCollection(IOctopusAPI.RUNDOWN_COLLECTION); + long rundownID = NoSQLUtils.asLong(rundown, IOctopusAPI.ID); + setObjectID(currentRundowns, rundownID, rundown); + if (!disableWrite) + collection.save(rundown); + } + logger.trace(ENTER); + } + + private void storeRundowns(BasicDBList rundowns) { + logger.trace(ENTER); + List rundownsList = NoSQLUtils.asList(rundowns); + for (BasicDBObject rundown : rundownsList) { + try { + storeRundown(rundown); + } catch (Exception e) { + logger.catching(e); + throw e; + } + fireProgressEvent(); + } + logger.trace(ENTER); + } + + private void storeStories() { + logger.trace(ENTER); + for (long storyID : storyIDs) { + // logger.info("Storing story {}", storyID); + try { + BasicDBObject story = queryStory(storyID); + if (story != null) + storeStory(story); + } catch (Exception e) { + logger.catching(e); + throw e; + } + fireProgressEvent(); + } + logger.trace(ENTER); + } + + private void storeStory(BasicDBObject story) { + logger.trace(ENTER); + if (!story.containsKey(IOctopusAPI.ID)) { + logger.error("Missing id in story {}", story.toPrettyString(null)); + return; + } + + normalizeStory(story); + + long storyID = story.getLong(IOctopusAPI.ID); + BasicDBList rundownRef = storyRundowns.get(storyID); + BasicDBList storyFolderRef = storyStoryFolders.get(storyID); + DBCollection collection = db.getCollection(IOctopusAPI.STORY_COLLECTION); + if (rundownRef != null) + story.put(IOctopusAPI.REF_RUNDOWN, rundownRef); + if (storyFolderRef != null) + story.put(IOctopusAPI.REF_STORYFOLDER, storyFolderRef); + + logger.debug(SAVING_STORY_ID, storyID); + setObjectID(currentStories, storyID, story); + if (!disableWrite) + collection.save(story); + logger.trace(EXIT); + } + + public void normalizeStory(BasicDBObject story) { + story.put(IOctopusAPI.MODIFIED, toDate(story, IOctopusAPI.MODIFIED)); + story.put(IOctopusAPI.SCHEDULEFROM, toDate(story, IOctopusAPI.SCHEDULEFROM)); + story.put(IOctopusAPI.SCHEDULETO, toDate(story, IOctopusAPI.SCHEDULETO)); + + String scriptContent = extractScriptContent(story); + story.put(IOctopusAPI.SCRIPT_CONTENT, scriptContent); + + BasicDBList modifiedMOS = extractRelevantMOSObjects(story); + if (modifiedMOS == null || modifiedMOS.isEmpty()) + story.remove(IOctopusAPI.MOS_OBJECTS); + else + story.put(IOctopusAPI.MOS_OBJECTS, modifiedMOS); + + BasicDBObject customColumns = getCustomColumns(story); + // logger.info(customColumns.toPrettyString("")); + String parentStoryId = NoSQLUtils.asString(customColumns, "ParentStoryID"); + if (parentStoryId == null) { + parentStoryId = story.getString(IOctopusAPI.ID); + } + story.append(IOctopusAPI.PARENT_STORY_ID, parentStoryId); + String location = NoSQLUtils.asString(customColumns, IOctopusAPI.LOCATION_HU); + if (location != null && location.trim().length() > 0) + story.append(IOctopusAPI.LOCATION, location); + + String descriptions = getDescriptions(story); + if (descriptions != null && descriptions.trim().length() > 0) + story.append(IOctopusAPI.DESCRIPTIONS, descriptions); + + story.remove(IOctopusAPI.CUSTOM_COLUMNS); + story.remove(IOctopusAPI.SCRIPT); + // story.remove(IOctopusAPI.DESCRIPTIONS); + } + + private void storeStoryFolder(BasicDBObject storyFolder) { + logger.trace(ENTER); + storyFolder.put(IOctopusAPI.MODIFIED, toDate(storyFolder, IOctopusAPI.MODIFIED)); + DBCollection collection = db.getCollection(IOctopusAPI.FOLDER_COLLECTION); + String name = storyFolder.getString(IOctopusAPI.NAME); + logger.debug("Storing story folder {}", name); + long folderID = NoSQLUtils.asLong(storyFolder, IOctopusAPI.ID); + setObjectID(currentFolder, folderID, storyFolder); + if (!disableWrite) + collection.save(storyFolder); + logger.trace(EXIT); + } + + private void storeStoryFolders(BasicDBList storyFolders) { + logger.trace(ENTER); + List storyFolderList = NoSQLUtils.asList(storyFolders); + for (BasicDBObject storyFolder : storyFolderList) { + try { + storeStoryFolder(storyFolder); + } catch (Exception e) { + logger.catching(e); + throw e; + } + fireProgressEvent(); + } + logger.trace(EXIT); + } + + private Date toDate(BasicDBObject obj, String name) { + Date result = null; + if (obj.containsKey(name)) { + String dt = obj.getString(name); + if (dt != null) { + // create jodatime from date + DateTime jdt = new DateTime(dt); + result = jdt.toDate(); + } + } + return result; + } + +} diff --git a/server/-product/production/HIRTV/jobs/steps/shared/PlanAirExtensions.java b/server/-product/production/HIRTV/jobs/steps/shared/PlanAirExtensions.java new file mode 100644 index 00000000..ef86f0a5 --- /dev/null +++ b/server/-product/production/HIRTV/jobs/steps/shared/PlanAirExtensions.java @@ -0,0 +1,300 @@ +package user.jobengine.server.steps.shared; + +import java.io.StringWriter; +import java.sql.Connection; +import java.sql.DriverManager; +import java.sql.ResultSetMetaData; +import java.text.SimpleDateFormat; +import java.util.ArrayList; +import java.util.Date; +import java.util.List; + +import javax.xml.parsers.DocumentBuilder; +import javax.xml.parsers.DocumentBuilderFactory; +import javax.xml.parsers.ParserConfigurationException; +import javax.xml.transform.OutputKeys; +import javax.xml.transform.Transformer; +import javax.xml.transform.TransformerException; +import javax.xml.transform.TransformerFactory; +import javax.xml.transform.dom.DOMSource; +import javax.xml.transform.stream.StreamResult; + +import org.apache.commons.lang.StringUtils; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.w3c.dom.DOMImplementation; +import org.w3c.dom.Document; +import org.w3c.dom.Node; + +import user.commons.mediatool.Timecode; +import user.commons.mediatool.Timecode.Type; +import user.jobengine.db.IItemManager; +import user.jobengine.db.IResultSetConsumer; +import user.jobengine.db.IStatementDecorator; + +public class PlanAirExtensions { + + private static MetadataType TRAFFIC_MATERIAL; + private static MetadataType TRAFFIC_AD; + private static MetadataType TRAFFIC_PROMO; + + PlanAirExtensions() { + TRAFFIC_MATERIAL = MetadataType.TrafficMaterial; + TRAFFIC_AD = MetadataType.TrafficAD; + TRAFFIC_PROMO = MetadataType.TrafficPromo; + } + + private static final Logger logger = LogManager.getLogger(); + // @Operation int, @@@Options int, @@ItemID int, @@IntParam1 int=Null, + // @@IntParam2 int=Null, @@IntParam3 int=Null, + // @@StrParam1 varchar(200)=Null, @@StrParam2 varchar(max)=Null, @@DateParam1 + // datetime=Null, @@DateParam2 datetime=Null + private static final String MATERIAL_SQL = "{call dbo.clIFsp_EC_MAM(1002, Null, Null, Null, Null, Null, ?)}";// musorid + // item.v_ProgrammeID = (int)ReadInt(reader, ref f); + // item.v_Live = (bool)ReadBool(reader, ref f); + // item.v_EpisodeID = ReadString(reader, ref f); + // item.v_VariantID = ReadInt(reader, ref f); + // item.v_MediaID = ReadString(reader, ref f); + // item.v_VariantTypeID = ReadInt(reader, ref f); + // item.v_ProgTitle = ReadString(reader, ref f); + // item.v_EpTitle = ReadString(reader, ref f); + // item.v_Episode = ReadShort(reader, ref f); + // item.v_VariantType = ReadString(reader, ref f); + // item.v_VariantKeywords = ReadString(reader, ref f); + // item.v_VariantLengthTC = ReadString(reader, ref f); + // item.v_VariantLengthFrame = ReadInt(reader, ref f); + // item.v_VariantNrSegments = ReadInt(reader, ref f); + // item.v_FirstBroadcastDate = ReadDateTime(reader, ref f); + // item.v_NextBroadcastDate = ReadDateTime(reader, ref f); + // item.v_OkForAir = ReadBool(reader, ref f); + // item.v_ForTransm = ReadBool(reader, ref f); + + private static final String MATERIAL_SEGMENTS_SQL = "{call dbo.clIFsp_EC_MAM(1010, Null, ?)}";// variantid + // item.v_SegID = (int)ReadInt(reader, ref f); + // item.v_VariantID = ReadInt(reader, ref f); + // item.v_SegTitle = ReadString(reader, ref f); + // item.v_SegKeyWords = ReadString(reader, ref f); + // item.v_SegNumber = ReadInt(reader, ref f); + // item.v_TcIn = ReadInt(reader, ref f); + // item.v_TcOut = ReadInt(reader, ref f); + // item.v_Duration = ReadInt(reader, ref f); + // item.v_TcInTC = ReadString(reader, ref f); + // item.v_TcOutTC = ReadString(reader, ref f); + // item.v_DurationTC = ReadString(reader, ref f); + // item.v_Dropable = ReadBool(reader, ref f); + + private static final String PROMO_SQL = "{call dbo.clIFsp_EC_MAM(2002, Null, Null, Null, Null, Null, ?)}"; + // item.t_SpotID = (int)ReadInt(reader, ref f); + // item.t_MediaID = ReadString(reader, ref f); + // item.v_Title = ReadString(reader, ref f); + // item.v_PromoType = ReadString(reader, ref f); + // item.v_ProgTitle = ReadString(reader, ref f); + // item.v_Episode = ReadShort(reader, ref f); + // item.t_PSStart = ReadDateTime(reader, ref f); + // item.t_PSEnd = ReadDateTime(reader, ref f); + // item.v_EstimatedDuration = ReadInt(reader, ref f); + // item.t_TcIn = ReadInt(reader, ref f); + // item.t_TcOut = ReadInt(reader, ref f); + // item.t_Duration = ReadInt(reader, ref f); + // item.v_TcIn = ReadString(reader, ref f); + // item.v_TcOut = ReadString(reader, ref f); + // item.v_Duration = ReadString(reader, ref f); + // item.v_Stations = ReadString(reader, ref f); + // item.t_OkForAir = ReadBool(reader, ref f); + // item.v_OkForAirs = ReadString(reader, ref f); + // item.v_IsInactive = ReadBool(reader, ref f); + private static final String AD_SQL = "{call dbo.clIFsp_EC_MAM(3002, Null, Null, Null, Null, Null, ?)}"; + // item.t_SpotID = (int)ReadInt(reader, ref f); + // item.t_MediaID = ReadString(reader, ref f); + // item.v_Title = ReadString(reader, ref f); + // item.t_Advertiser = ReadString(reader, ref f); + // item.v_EstimatedDuration = ReadInt(reader, ref f); + // item.t_TcIn = ReadInt(reader, ref f); + // item.t_TcOut = ReadInt(reader, ref f); + // item.t_Duration = ReadInt(reader, ref f); + // item.v_TcIn = ReadString(reader, ref f); + // item.v_TcOut = ReadString(reader, ref f); + // item.v_Duration = ReadString(reader, ref f); + // item.t_OkForAir = ReadBool(reader, ref f); + // item.v_OkForAirs = ReadString(reader, ref f); + + private static final String SQLSERVER_JDBC_SQL_SERVER_DRIVER = "com.microsoft.sqlserver.jdbc.SQLServerDriver"; + private static final String HYPHEN_SPACES = " - "; + private static Connection connection; + + private static void appendInstanceData(Document doc, Node node, String name, String deviceID, Timecode tcIn, Timecode tcOut) { + node.appendChild(doc.createElement("Long_File_Id")).appendChild(doc.createTextNode(name)); + node.appendChild(doc.createElement("Device_Id")).appendChild(doc.createTextNode(deviceID)); + node.appendChild(doc.createElement("Timecode_In")).appendChild(doc.createTextNode(tcIn.toString(""))); + node.appendChild(doc.createElement("Timecode_Out")).appendChild(doc.createTextNode(tcOut.toString(""))); + node.appendChild(doc.createElement("Quality_Check")).appendChild(doc.createTextNode(new SimpleDateFormat("dd/MM/yyyy").format(new Date()))); + } + + private static void appendItemData(Document doc, Node node, String name, String title, Timecode duration, String type) { + + node.appendChild(doc.createElement("Material_Id")).appendChild(doc.createTextNode(name)); + node.appendChild(doc.createElement("Title")).appendChild(doc.createTextNode(title)); + node.appendChild(doc.createElement("On_Air_Duration")).appendChild(doc.createTextNode(duration.toString(""))); + node.appendChild(doc.createElement("Material_Type")).appendChild(doc.createTextNode(type)); + } + + private static Document createXMLDocument() throws ParserConfigurationException { + DocumentBuilderFactory dbf = DocumentBuilderFactory.newInstance(); + DocumentBuilder db = dbf.newDocumentBuilder(); + DOMImplementation impl = db.getDOMImplementation(); + Document xmlDocument = impl.createDocument(null, null, null); + return xmlDocument; + } + + public static long getExistingRundownMedia(IItemManager manager, String houseid) { + final long[] result = new long[] { 0 }; + StringBuilder query = new StringBuilder(); + query.append("select mediaid, mediafilehouseid, filename"); + query.append(" "); + query.append(String.format("from vw_rundown_items where mediafilehouseid like '%s%%'", houseid)); + query.append(" "); + query.append("order by filename, mediaid"); + IResultSetConsumer consumer = rs -> { + String fileName = rs.getString("filename"); + if (houseid.equals(fileName)) { + result[0] = rs.getLong("mediaid"); + return false; + } else + return true; + }; + manager.executeQuery(query.toString(), consumer, null); + return result[0]; + } + + /* + * + * XMLTEST011 Teszt Mozi + * 00000914 + * PROGRAMME //PROGRAMME, COMMERCIAL, JUNCTION + * ISILON + * 09/11/2017 + * XMLTEST011 + * + */ + public static String getMorpeusXML(IItemManager manager, String dbUrl, String userName, String password, String name, String deviceID) throws Exception { + String result = null; + try { + Class.forName(SQLSERVER_JDBC_SQL_SERVER_DRIVER); + + MetadataType type = MetadataTypeDetector.GuessMetadataType(name); + + Document doc = createXMLDocument(); + Node rootNode = doc.appendChild(doc.createElement("ImportItems")); + Node importItemNode = rootNode.appendChild(doc.createElement("ImportItem")); + Node itemNode = importItemNode.appendChild(doc.createElement("Item")); + Node instanceNode = importItemNode.appendChild(doc.createElement("Instance")); + + connection = DriverManager.getConnection(dbUrl, userName, password); + // TODO hiba esetén exception + if (MetadataType.TrafficMaterial.equals(type)) { + manager.executeQuery(connection, MATERIAL_SQL, rs -> { + String progTitle = rs.getString("v_ProgTitle"); + String epTitle = rs.getString("v_EpTitle"); + String title = progTitle; + if (StringUtils.isNotBlank(epTitle)) + title += HYPHEN_SPACES + epTitle; + int variantID = rs.getInt("v_VariantID"); + List segments = new ArrayList<>(); + manager.executeQuery(connection, MATERIAL_SEGMENTS_SQL, rs1 -> { + ResultSetMetaData rsmd = rs1.getMetaData(); + for (int i = 1; i <= rsmd.getColumnCount(); i++) + System.out.println(rsmd.getColumnName(i) + " " + rsmd.getColumnTypeName(i)); + + int[] segment = new int[] { rs1.getInt("v_TcIn"), rs1.getInt("v_TcOut") }; + segments.add(segment); + return true; + }, st -> { + st.setInt(1, variantID); + }); + int out = segments.get(segments.size() - 1)[1]; + int in = segments.get(0)[0]; + appendItemData(doc, itemNode, name, title, new Timecode(out - in, Type.PAL), "PROGRAMME"); + appendInstanceData(doc, instanceNode, name, deviceID, new Timecode(in, Type.PAL), new Timecode(out, Type.PAL)); + return false; + }, st -> { + st.setString(1, name); + }); + } else if (MetadataType.TrafficPromo.equals(type)) { + manager.executeQuery(connection, PROMO_SQL, rs -> { + String title = rs.getString("v_Title"); + int in = rs.getInt("t_TcIn"); + int out = rs.getInt("t_TcOut"); + appendItemData(doc, itemNode, name, title, new Timecode(out - in, Type.PAL), "COMMERCIAL"); + appendInstanceData(doc, instanceNode, name, deviceID, new Timecode(in, Type.PAL), new Timecode(out, Type.PAL)); + return false; + }, st -> { + st.setString(1, name); + }); + } else if (MetadataType.TrafficAD.equals(type)) { + manager.executeQuery(connection, AD_SQL, rs -> { + String title = rs.getString("v_Title"); + int in = rs.getInt("t_TcIn"); + int out = rs.getInt("t_TcOut"); + appendItemData(doc, itemNode, name, title, new Timecode(out - in, Type.PAL), "JUNCTION"); + appendInstanceData(doc, instanceNode, name, deviceID, new Timecode(in, Type.PAL), new Timecode(out, Type.PAL)); + return false; + }, st -> { + st.setString(1, name); + }); + } else { + return null; + } + result = XMLtoString(doc); + } catch (Exception e) { + logger.catching(e); + throw e; + } finally { + if (connection != null) + connection.close(); + } + return result; + } + + public static Connection search(Connection con, MetadataType type, IItemManager manager, String dbUrl, String userName, String password, + IResultSetConsumer consumer, IStatementDecorator decorator) throws Exception { + try { + Class.forName(SQLSERVER_JDBC_SQL_SERVER_DRIVER); + + if (con == null) + connection = DriverManager.getConnection(dbUrl, userName, password); + // TODO hiba esetén exception + + if (MetadataType.TrafficMaterial.equals(type)) { + manager.executeQuery(connection, MATERIAL_SQL, consumer, decorator); + } else if (MetadataType.TrafficPromo.equals(type)) { + manager.executeQuery(connection, PROMO_SQL, consumer, decorator); + } else if (MetadataType.TrafficAD.equals(type)) { + manager.executeQuery(connection, AD_SQL, consumer, decorator); + } + } catch (Exception e) { + logger.catching(e); + throw e; + } finally { + // if (connection != null) + // connection.close(); + } + return connection; + } + + private static String XMLtoString(Document doc) throws TransformerException { + DOMSource domSource = new DOMSource(doc); + TransformerFactory tf = TransformerFactory.newInstance(); + Transformer transformer = tf.newTransformer(); + // transformer.setOutputProperty(OutputKeys.OMIT_XML_DECLARATION, "yes"); + transformer.setOutputProperty(OutputKeys.STANDALONE, "yes"); + transformer.setOutputProperty(OutputKeys.METHOD, "xml"); + transformer.setOutputProperty(OutputKeys.ENCODING, "UTF-8"); + transformer.setOutputProperty("{http://xml.apache.org/xslt}indent-amount", "4"); + transformer.setOutputProperty(OutputKeys.INDENT, "yes"); + StringWriter sw = new StringWriter(); + StreamResult sr = new StreamResult(sw); + transformer.transform(domSource, sr); + return sw.toString(); + } + +} -- 2.54.0