From 99892db7d9b3842d00a44faa0c47176fdf91f8ad Mon Sep 17 00:00:00 2001 From: "vasary.daniel" Date: Thu, 27 May 2021 13:39:40 +0000 Subject: [PATCH] git-tfs-id: [http://tfs.userrendszerhaz.hu:8080/tfs/DefaultCollection]$/MediaCube;C32237 --- .../run-mediacube-server-hirtv.launch | 2 +- server/-product/.tpignore | 2 + .../production/AMC/service-mediacube.bat | 48 +++ .../production/HIRTV/jobs/executors.yaml | 5 - .../production/HIRTV/jobs/schedules-orig.json | 318 ++++++++++++++ .../production/HIRTV/jobs/schedules.json | 342 +-------------- .../HIRTV/jobs/templates/cancelable.xml | 2 +- ...te-materials.xml => delete-materials1.xml} | 4 + .../jobs/templates/delete-materials2.xml | 21 + .../jobs/templates/delete-materials3.xml | 21 + .../jobs/templates/delete-materials4.xml | 21 + .../jobs/templates/delete-materials5.xml | 21 + .../HIRTV/jobs/templates/sync-octopus.xml | 12 + .../jobs/steps/ArchiveListBuilderStep.java | 167 -------- .../jobs/steps/ArchiveMaterialSubmitStep.java | 53 --- .../LOCAL/jobs/steps/ArchiveRecursive.java | 291 ------------- .../jobs/steps/BatchRetrieveForkStep.java | 76 ---- .../LOCAL/jobs/steps/CancelableStep.java | 57 ++- .../steps/CleanupMountedLocationStep.java | 316 -------------- .../jobs/steps/CreateArchiveItemStep.java | 45 -- .../jobs/steps/CreateMissingLowresStep.java | 84 ---- .../LOCAL/jobs/steps/ExternalCommand.java | 79 ---- .../jobs/steps/ExternalCommandExecutor.java | 32 -- .../LOCAL/jobs/steps/ExternalProfile.java | 33 -- .../jobs/steps/ExternalProfilesConfig.java | 15 - .../LOCAL/jobs/steps/FileCopyStep.java | 95 ----- .../jobs/steps/FileSearchFilterOptions.java | 41 -- .../LOCAL/jobs/steps/HSMMigrateStep.java | 401 ------------------ .../LOCAL/jobs/steps/IExternalCallback.java | 5 - .../LOCAL/jobs/steps/MXFCutterStep.java | 129 ------ .../LOCAL/jobs/steps/MediaCubeClient.java | 77 ---- .../LOCAL/jobs/steps/MediaToolStep.java | 41 -- .../jobs/steps/MetadataTransformStep.java | 168 -------- .../steps/OutputPathAndNameSelectorStep.java | 151 ------- .../jobs/steps/PathItemsCollectorStep.java | 68 --- ...epareRemoteTranscodeJobParametersStep.java | 20 - .../steps/QueryMissingProxyMediaStep.java | 76 ---- .../LOCAL/jobs/steps/RemoteJobStep.java | 43 -- .../LOCAL/jobs/steps/TSMBackupStep.java | 207 --------- .../jobs/steps/TSMExtendedRetrieveStep.java | 91 ---- .../LOCAL/jobs/steps/TSMRestoreStep.java | 159 ------- .../jobs/steps/TSMSimpleRestoreStep.java | 57 --- .../jobs/steps/TestForkCancelableStep.java | 35 -- .../production/LOCAL/jobs/steps/TestLib.java | 8 - .../jobs/steps/TranscodeFFAStranStep.java | 151 ------- .../LOCAL/jobs/steps/TranscodeFFMpegStep.java | 52 --- .../jobs/steps/UpdateGhostMediaDataStep.java | 88 ---- .../LOCAL/settings/application-htv.yaml | 20 +- ...ication-local.yaml => application-mv.yaml} | 0 server/-product/production/config-210526.ini | 9 + .../META-INF/MANIFEST.MF | 1 + .../mediacube/executors/tests/Support.java | 17 + .../server/steps/CancelableStep.java | 2 +- .../server/steps/CreateMissingLowresStep.java | 7 +- .../steps/DeleteNEXIOMaterialsStep.java | 10 +- .../server/steps/PASAPOOLTransferToStep.java | 3 + .../server/steps/SyncOCTOPUSDataStep.java | 23 +- .../jobengine/server/steps/TransferStep.java | 82 ++-- .../steps/shared/ExternalCommandExecutor.java | 7 - .../server/steps/shared/OctopusDataMiner.java | 18 +- .../src/user/commons/StoreUri.java | 2 + .../remotestore/FtpDirectoryLister.java | 2 +- .../036_alter_BeabbleBeach_storeuri.sql | 52 +++ .../user/jobengine/search/MediaFinder.java | 6 +- .../server/IJobEngineConfiguration.java | 2 + .../src/user/jobengine/server/JobEngine.java | 3 + .../server/JobEngineConfiguration.java | 19 +- .../jobengine/server/JobStepExecutor.java | 8 +- .../server/scheduler/ScheduledJob.java | 2 +- server/user.mediacube.gui/WEB-INF/web.xml | 2 +- .../pages/searchitems/results.zul | 2 +- .../resources/i3-label_hu.properties | 2 +- .../user/jobengine/gui/ComponentBinder.java | 2 + 73 files changed, 694 insertions(+), 3839 deletions(-) create mode 100644 server/-product/production/AMC/service-mediacube.bat delete mode 100644 server/-product/production/HIRTV/jobs/executors.yaml create mode 100644 server/-product/production/HIRTV/jobs/schedules-orig.json rename server/-product/production/HIRTV/jobs/templates/{delete-materials.xml => delete-materials1.xml} (75%) create mode 100644 server/-product/production/HIRTV/jobs/templates/delete-materials2.xml create mode 100644 server/-product/production/HIRTV/jobs/templates/delete-materials3.xml create mode 100644 server/-product/production/HIRTV/jobs/templates/delete-materials4.xml create mode 100644 server/-product/production/HIRTV/jobs/templates/delete-materials5.xml delete mode 100644 server/-product/production/LOCAL/jobs/steps/ArchiveListBuilderStep.java delete mode 100644 server/-product/production/LOCAL/jobs/steps/ArchiveMaterialSubmitStep.java delete mode 100644 server/-product/production/LOCAL/jobs/steps/ArchiveRecursive.java delete mode 100644 server/-product/production/LOCAL/jobs/steps/BatchRetrieveForkStep.java delete mode 100644 server/-product/production/LOCAL/jobs/steps/CleanupMountedLocationStep.java delete mode 100644 server/-product/production/LOCAL/jobs/steps/CreateArchiveItemStep.java delete mode 100644 server/-product/production/LOCAL/jobs/steps/CreateMissingLowresStep.java delete mode 100644 server/-product/production/LOCAL/jobs/steps/ExternalCommand.java delete mode 100644 server/-product/production/LOCAL/jobs/steps/ExternalCommandExecutor.java delete mode 100644 server/-product/production/LOCAL/jobs/steps/ExternalProfile.java delete mode 100644 server/-product/production/LOCAL/jobs/steps/ExternalProfilesConfig.java delete mode 100644 server/-product/production/LOCAL/jobs/steps/FileCopyStep.java delete mode 100644 server/-product/production/LOCAL/jobs/steps/FileSearchFilterOptions.java delete mode 100644 server/-product/production/LOCAL/jobs/steps/HSMMigrateStep.java delete mode 100644 server/-product/production/LOCAL/jobs/steps/IExternalCallback.java delete mode 100644 server/-product/production/LOCAL/jobs/steps/MXFCutterStep.java delete mode 100644 server/-product/production/LOCAL/jobs/steps/MediaCubeClient.java delete mode 100644 server/-product/production/LOCAL/jobs/steps/MediaToolStep.java delete mode 100644 server/-product/production/LOCAL/jobs/steps/MetadataTransformStep.java delete mode 100644 server/-product/production/LOCAL/jobs/steps/OutputPathAndNameSelectorStep.java delete mode 100644 server/-product/production/LOCAL/jobs/steps/PathItemsCollectorStep.java delete mode 100644 server/-product/production/LOCAL/jobs/steps/PrepareRemoteTranscodeJobParametersStep.java delete mode 100644 server/-product/production/LOCAL/jobs/steps/QueryMissingProxyMediaStep.java delete mode 100644 server/-product/production/LOCAL/jobs/steps/RemoteJobStep.java delete mode 100644 server/-product/production/LOCAL/jobs/steps/TSMBackupStep.java delete mode 100644 server/-product/production/LOCAL/jobs/steps/TSMExtendedRetrieveStep.java delete mode 100644 server/-product/production/LOCAL/jobs/steps/TSMRestoreStep.java delete mode 100644 server/-product/production/LOCAL/jobs/steps/TSMSimpleRestoreStep.java delete mode 100644 server/-product/production/LOCAL/jobs/steps/TestForkCancelableStep.java delete mode 100644 server/-product/production/LOCAL/jobs/steps/TestLib.java delete mode 100644 server/-product/production/LOCAL/jobs/steps/TranscodeFFAStranStep.java delete mode 100644 server/-product/production/LOCAL/jobs/steps/TranscodeFFMpegStep.java delete mode 100644 server/-product/production/LOCAL/jobs/steps/UpdateGhostMediaDataStep.java rename server/-product/production/LOCAL/settings/{application-local.yaml => application-mv.yaml} (100%) create mode 100644 server/-product/production/config-210526.ini create mode 100644 server/user.jobengine.osgi.db/migrations/scripts/036_alter_BeabbleBeach_storeuri.sql diff --git a/server/-configuration/run-mediacube-server-hirtv.launch b/server/-configuration/run-mediacube-server-hirtv.launch index a27365a6..4f2f3d81 100644 --- a/server/-configuration/run-mediacube-server-hirtv.launch +++ b/server/-configuration/run-mediacube-server-hirtv.launch @@ -20,7 +20,7 @@ - + diff --git a/server/-product/.tpignore b/server/-product/.tpignore index 51fcdd4a..9d5bf9a8 100644 --- a/server/-product/.tpignore +++ b/server/-product/.tpignore @@ -11,3 +11,5 @@ \Q/production/AMC/tmp/\E.* \Q/production/LOCAL/log/\E.* \Q/production/LOCAL/tmp/\E.* +\Q/production/HIRTV/jobs/classes/\E.* +\Q/production/LOCAL/jobs/classes/\E.* diff --git a/server/-product/production/AMC/service-mediacube.bat b/server/-product/production/AMC/service-mediacube.bat new file mode 100644 index 00000000..29a7dd23 --- /dev/null +++ b/server/-product/production/AMC/service-mediacube.bat @@ -0,0 +1,48 @@ +@echo off +cd "F:\mediacube" + +if not exist Z:\ net use Z: \\192.168.23.160\VICC dal /user:hadmin /persistent:yes +if not exist G:\downloads mkdir G:\downloads +if not exist G:\uploads mkdir G:\uploads +if not exist F:\mediacube\uploads mklink /D F:\mediacube\uploads G:\uploads +if not exist F:\mediacube\downloads mklink /D F:\mediacube\downloads G:\downloads + +del /Q G:\downloads\.STATUS\*.catched +del /Q G:\downloads\*.part + +del /Q G:\uploads\.STATUS\*.catched +del /Q G:\uploads\*.part + +if exist tmp rmdir tmp /s /q +mkdir tmp + +set "DSMI_CONFIG=F:\mediacube\settings\dsm.opt" +set "DSM_CONFIG=F:\mediacube\settings\dsm.opt" +set "DSMI_DIR=c:\Program Files\Tivoli\TSM\baclient" +set "DSM_DIR=F:\mediacube" +set "DSMI_LOG=F:\mediacube\log" +set "LD_LIBRARY_PATH=c:\Program Files\Tivoli\TSM\baclient;c:\opt\mediacube" +set "LIBPATH=c:\Program Files\Tivoli\TSM\baclient" +set "SHLIB_PATH=c:\Program Files\Tivoli\TSM\baclient" +set "PATH=F:\jdk1.8.0_112\bin\;%PATH%" + +java ^ +-Dcom.sun.management.jmxremote ^ +-Dcom.sun.management.jmxremote.port=9010 ^ +-Dcom.sun.management.jmxremote.local.only=false ^ +-Dcom.sun.management.jmxremote.authenticate=false ^ +-Dcom.sun.management.jmxremote.ssl=false ^ +-Dorg.eclipse.epp.logging.aeri.skipReports=true ^ +-Declipse.ignoreApp=true ^ +-Dosgi.noShutdown=true ^ +-Dlog4j.configurationFile=settings/log4j2.xml ^ +-Djetty.home=settings ^ +-Djetty.etc.config.urls=jetty.xml ^ +-Djava.io.tmpdir=tmp ^ +-Dfile.encoding=UTF-8 ^ +-Dgosh.home=configuration ^ +-Djna.library.path=F:/mediacube ^ +-jar plugins/org.eclipse.equinox.launcher_1.3.201.v20161025-1711.jar ^ +-Xms512m ^ +-Xmx4096m ^ +-console diff --git a/server/-product/production/HIRTV/jobs/executors.yaml b/server/-product/production/HIRTV/jobs/executors.yaml deleted file mode 100644 index 8cba80bd..00000000 --- a/server/-product/production/HIRTV/jobs/executors.yaml +++ /dev/null @@ -1,5 +0,0 @@ -executors: - - className: user.jobengine.server.steps.ArchiveListBuilderStep - maxConcurrent: 1 - - className: user.jobengine.server.steps.ArchiveMaterialSubmitStep - maxConcurrent: 2 diff --git a/server/-product/production/HIRTV/jobs/schedules-orig.json b/server/-product/production/HIRTV/jobs/schedules-orig.json new file mode 100644 index 00000000..076b6fc0 --- /dev/null +++ b/server/-product/production/HIRTV/jobs/schedules-orig.json @@ -0,0 +1,318 @@ +{"joblist":[ + { + "template": "cancelable.xml" + }, + { + "active": true, + "executeimmediate": true, + "name" : "OCTOPUS adatok szinkronizálása", + "template": "sync-octopus.xml", + "cronexpression": "0/30 * * * * ?", + "parameters": [ + {"name": "includeArchived", "value": false, "type": "java.lang.Boolean"}, + {"name": "address", "value": "http://10.10.1.10/api/v1", "type": "java.lang.String"}, + {"name": "user", "value": "mam", "type": "java.lang.String"}, + {"name": "pwd", "value": "napocska", "type": "java.lang.String"} + ] + }, + { + "active": true, + "executeimmediate": false, + "name" : "Limitált archiválás az ISILON/ARCHIVE mappából", + "template": "archive-limited.xml", + "cronexpression": "0 0 6-22/2 * * ?", + "parameters": [ + {"name": "sourcePath", "value": "/mnt/ISILON/ARCHIVE", "type": "java.lang.String"}, + {"name": "killDateDays", "value": 1, "type": "java.lang.Integer"}, + {"name": "limit", "value": 20, "type": "java.lang.Integer"} + ] + }, + { + "active": true, + "executeimmediate": false, + "name" : "Archiválás az ISILON/ARCHIVE mappából", + "template": "archive-ondemand.xml", + "cronexpression": "0 0 1 * * ?", + "parameters": [ + {"name": "sourcePath", "value": "/mnt/ISILON/ARCHIVE", "type": "java.lang.String"}, + {"name": "killDateDays", "value": 1, "type": "java.lang.Integer"}, + {"name": "limit", "value": 0, "type": "java.lang.Integer"} + ] + }, + { + "active": true, + "executeimmediate": false, + "name" : "PROXY pótlás FFASTRANS 71", + "template": "sys-recreate-lowres-71.xml", + "cronexpression": "0 * * * * ?", + "parameters": [ + {"name": "localLowresPath", "value": "/mnt/FIXTRANSCODER-71/OUTPUT", "type": "java.lang.String" }, + {"name": "transcoderAddress", "value": "http://10.10.1.71:65445/api/json/v1/", "type": "java.lang.String"}, + {"name": "globalRetrievePath", "value": "file://10.10.1.30/transcode", "type": "java.lang.String"}, + {"name": "localRetrievePath", "value": "/mediacube/data/lowres/www/video/IFT3/transcode", "type": "java.lang.String"}, + {"name": "localHiresPath", "value": "/mediacube/data/lowres/www/video/IFT3/transcode", "type": "java.lang.String" }, + {"name": "globalHiresPath", "value": "L:\\transcode", "type": "java.lang.String" }, + {"name": "transcoderTemplateName", "value": "MAM_proxy", "type": "java.lang.String"} + ] + }, + { + "active": false, + "executeimmediate": false, + "name" : "PROXY pótlás FFASTRANS 72", + "template": "sys-recreate-lowres-72.xml", + "cronexpression": "10 * * * * ?", + "parameters": [ + {"name": "localLowresPath", "value": "/mnt/FIXTRANSCODER-72/OUTPUT", "type": "java.lang.String" }, + {"name": "transcoderAddress", "value": "http://10.10.1.72:65445/api/json/v1/", "type": "java.lang.String"}, + {"name": "globalRetrievePath", "value": "file://10.10.1.30/transcode", "type": "java.lang.String"}, + {"name": "localRetrievePath", "value": "/mediacube/data/lowres/www/video/IFT3/transcode", "type": "java.lang.String"}, + {"name": "localHiresPath", "value": "/mediacube/data/lowres/www/video/IFT3/transcode", "type": "java.lang.String" }, + {"name": "globalHiresPath", "value": "L:\\transcode", "type": "java.lang.String" }, + {"name": "transcoderTemplateName", "value": "MAM_proxy", "type": "java.lang.String"} + ] + }, + { + "active": false, + "executeimmediate": false, + "name" : "PROXY pótlás FFASTRANS 73", + "template": "sys-recreate-lowres-73.xml", + "cronexpression": "20 * * * * ?", + "parameters": [ + {"name": "localLowresPath", "value": "/mnt/FIXTRANSCODER-73/OUTPUT", "type": "java.lang.String" }, + {"name": "transcoderAddress", "value": "http://10.10.1.73:65445/api/json/v1/", "type": "java.lang.String"}, + {"name": "globalRetrievePath", "value": "file://10.10.1.30/transcode", "type": "java.lang.String"}, + {"name": "localRetrievePath", "value": "/mediacube/data/lowres/www/video/IFT3/transcode", "type": "java.lang.String"}, + {"name": "localHiresPath", "value": "/mediacube/data/lowres/www/video/IFT3/transcode", "type": "java.lang.String" }, + {"name": "globalHiresPath", "value": "L:\\transcode", "type": "java.lang.String" }, + {"name": "transcoderTemplateName", "value": "MAM_proxy", "type": "java.lang.String"} + ] + }, + { + "active": false, + "executeimmediate": false, + "name" : "PROXY pótlás FFASTRANS 74", + "template": "sys-recreate-lowres-74.xml", + "cronexpression": "30 * * * * ?", + "parameters": [ + {"name": "localLowresPath", "value": "/mnt/FIXTRANSCODER-74/OUTPUT", "type": "java.lang.String" }, + {"name": "transcoderAddress", "value": "http://10.10.1.74:65445/api/json/v1/", "type": "java.lang.String"}, + {"name": "globalRetrievePath", "value": "file://10.10.1.30/transcode", "type": "java.lang.String"}, + {"name": "localRetrievePath", "value": "/mediacube/data/lowres/www/video/IFT3/transcode", "type": "java.lang.String"}, + {"name": "localHiresPath", "value": "/mediacube/data/lowres/www/video/IFT3/transcode", "type": "java.lang.String" }, + {"name": "globalHiresPath", "value": "L:\\transcode", "type": "java.lang.String" }, + {"name": "transcoderTemplateName", "value": "MAM_proxy", "type": "java.lang.String"} + ] + }, + { + "name" : "SYS: create-lowres-ondemand", + "template": "create-lowres-ondemand.xml", + "parameters": [ + {"name": "globalRetrievePath", "value": "file://10.10.1.30/transcode", "type": "java.lang.String"}, + {"name": "localRetrievePath", "value": "/mediacube/data/lowres/www/video/IFT3/transcode", "type": "java.lang.String"}, + {"name": "localHiresPath", "value": "/mediacube/data/lowres/www/video/IFT3/transcode", "type": "java.lang.String" }, + {"name": "globalHiresPath", "value": "L:\\transcode", "type": "java.lang.String" }, + {"name": "localLowresPath", "value": "/mnt/FIXTRANSCODER/OUTPUT", "type": "java.lang.String" }, + {"name": "transcoderAddress", "value": "http://10.10.1.74:65445/api/json/v1/", "type": "java.lang.String"}, + {"name": "transcoderTemplateName", "value": "MAM_proxy", "type": "java.lang.String"} + ] + }, + { + "active": true, + "executeimmediate": false, + "name" : "Lejárt ISISLON/ARCHIVE anyagok törlése", + "template": "delete-materials1.xml", + "cronexpression": "0 0 5 * * ?", + "parameters": [ + {"name": "sourcePath", "value": "/mnt/ISILON/ARCHIVE", "type": "java.lang.String"}, + {"name": "skipArchiveCheck", "value": true, "type": "java.lang.Boolean"} + ] + }, + { + "active": true, + "executeimmediate": false, + "name" : "Lejárt ISILON/TQC/CHECK/KESZ anyagok törlése", + "template": "delete-materials2.xml", + "cronexpression": "0 0 8 * * ?", + "parameters": [ + {"name": "sourcePath", "value": "/mnt/ISILON/TQC/CHECK/KESZ", "type": "java.lang.String"}, + {"name": "skipArchiveCheck", "value": true, "type": "java.lang.Boolean"} + ] + }, + { + "active": true, + "executeimmediate": false, + "name" : "Lejárt ISILON/TQC/PROMO/KESZ anyagok törlése", + "template": "delete-materials3.xml", + "cronexpression": "0 0 6 * * ?", + "parameters": [ + {"name": "sourcePath", "value": "/mnt/ISILON/TQC/PROMO/KESZ", "type": "java.lang.String"}, + {"name": "skipArchiveCheck", "value": true, "type": "java.lang.Boolean"} + ] + }, + { + "active": true, + "executeimmediate": false, + "name" : "Lejárt ISILON/TQC/REKLAM/KESZ anyagok törlése", + "template": "delete-materials4.xml", + "cronexpression": "0 0 7 * * ?", + "parameters": [ + {"name": "sourcePath", "value": "/mnt/ISILON/TQC/REKLAM/KESZ", "type": "java.lang.String"}, + {"name": "skipArchiveCheck", "value": true, "type": "java.lang.Boolean"} + ] + }, + { + "active": true, + "executeimmediate": false, + "name" : "Lejárt ISILON/OCTOPUS/_NAPI_MEGTEKINTO anyagok törlése", + "template": "delete-materials5.xml", + "cronexpression": "0 0 22 * * ?", + "parameters": [ + {"name": "sourcePath", "value": "/mnt/ISILON/OCTOPUS/_NAPI_MEGTEKINTO", "type": "java.lang.String"}, + {"name": "skipArchiveCheck", "value": true, "type": "java.lang.Boolean"} + ] + }, + { + "active": true, + "executeimmediate": false, + "name" : "Lejárt ISISLON/PROMO_NLE mappák törlése", + "template": "delete-promo-materials.xml", + "cronexpression": "0 0 6 * * ?", + "parameters": [ + {"name": "sourcePath", "value": "/mnt/ISILON/PROMO_NLE", "type": "java.lang.String"} + ] + }, + { + "active": true, + "executeimmediate": false, + "name" : "NEXIO bejátszó anyagok másolása az ISILON/ARCHIVE mappába", + "template": "copyforarchive-nexio-materials.xml", + "cronexpression": "0 0 10 * * ?", + "parameters": [ + {"name": "nexioPort", "value": 2098, "type": "java.lang.Integer"}, + {"name": "nexioUserName", "value": "administrator", "type": "java.lang.String"}, + {"name": "nexioPassword", "value": "system", "type": "java.lang.String"}, + {"name": "archiveFtp", "value": "ftp://10.10.1.100/ARCHIVE", "type": "java.lang.String"}, + {"name": "archiveUserName", "value": "mediacube", "type": "java.lang.String"}, + {"name": "archivePassword", "value": "Broadca5T", "type": "java.lang.String"}, + {"name": "daysBeforeNow", "value": 1, "type": "java.lang.Integer"}, + {"name": "nexioKillDateDays", "value": 21, "type": "java.lang.Integer"}, + {"name": "nexioAgency", "value": "ARCHIVED", "type": "java.lang.String"} + ] + }, + { + "active": true, + "executeimmediate": false, + "name" : "NEXIO visszarögzített anyagok másolása az ISILON/ARCHIVE mappába", + "template": "copyforarchive-nexio-recordings.xml", + "cronexpression": "0 0 12 * * ? *", + "parameters": [ + {"name": "nexioPort", "value": 2098, "type": "java.lang.Integer"}, + {"name": "nexioUserName", "value": "administrator", "type": "java.lang.String"}, + {"name": "nexioPassword", "value": "system", "type": "java.lang.String"}, + {"name": "archiveFtp", "value": "ftp://10.10.1.100/ARCHIVE", "type": "java.lang.String"}, + {"name": "archiveUserName", "value": "mediacube", "type": "java.lang.String"}, + {"name": "archivePassword", "value": "Broadca5T", "type": "java.lang.String"}, + {"name": "filterAgencies", "value": "schedule-rec", "type": "java.lang.String"}, + {"name": "limit", "value": 30, "type": "java.lang.Integer"}, + {"name": "nexioKillDateDays", "value": 7, "type": "java.lang.Integer"}, + {"name": "nexioAgency", "value": "HIRADO_ARCHIVED", "type": "java.lang.String"} + ] + }, + { + "active": true, + "executeimmediate": false, + "name" : "Híranyag statisztika importálása", + "template": "import-statistics.xml", + "cronexpression": "0 0 6 * * ?", + "parameters": [ + {"name": "daysBeforeNow", "value": 1, "type": "java.lang.Integer"} + ] + }, + { + "active": false, + "executeimmediate": true, + "name" : "TRAFFIC anyagok visszatöltése", + "template": "retrieve-traffic-missing-materials.xml", + "cronexpression": "0 0 0/1 1/1 * ? *", + "parameters": [ + {"name": "dbUrl", "value": "jdbc:sqlserver://10.10.1.45\\sql16;databaseName=PA_Echo;", "type": "java.lang.String"}, + {"name": "userName", "value": "MAM", "type": "java.lang.String"}, + {"name": "password", "value": "Echotv.hu", "type": "java.lang.String"}, + {"name": "lookupDays", "value": 7, "type": "java.lang.Integer"}, + {"name": "targetPath", "value": "/mnt/ISILON/PLAYOUT/IceGateway/Input", "type": "java.lang.String"}, + {"name": "killDateDays", "value": 10, "type": "java.lang.Integer"} + ] + }, + { + "active": true, + "executeimmediate": false, + "name" : "Lejárt NEXIO anyagok törlése", + "template": "delete-nexio-materials.xml", + "cronexpression": "0 0 6 * * ?", + "parameters": [ + {"name": "port", "value": 2098, "type": "java.lang.Integer"}, + {"name": "userName", "value": "administrator", "type": "java.lang.String"}, + {"name": "password", "value": "system", "type": "java.lang.String"}, + {"name": "filterAgencies", "value": "HIRADO_23_00,HIRADO_ARCHIVED,HIRADO_CLN", "type": "java.lang.String"}, + {"name": "gracePeriodDays", "value": 1, "type": "java.lang.Integer"}, + {"name": "notificationOnly", "value": false, "type": "java.lang.Boolean"} + ] + }, + { + "name" : "SYS: MORPHEUS 'missing materials' importálása", + "template": "import-morpheus-missing-materials.xml", + "parameters": [ + {"name": "csvFilePath", "value": "/mnt/MORPHEUS", "type": "java.lang.String"}, + {"name": "processedFolder", "value": "DONE", "type": "java.lang.String"}, + {"name": "targetPath", "value": "/mnt/ISILON/ARCHIVE_RESTORE", "type": "java.lang.String"} + ] + }, + { + "name" : "SYS: batch-retrieve-ondemand", + "template": "batch-retrieve-ondemand.xml" + }, + { + "name" : "SYS: retrieve-ondemand", + "template": "retrieve-ondemand.xml", + "parameters": [ + {"name": "globalRetrievePath", "value": "file://isilon.intra.echotv.hu", "type": "java.lang.String"}, + {"name": "localRetrievePath", "value": "/mnt/ISILON", "type": "java.lang.String"}, + {"name": "materialOutputFolder", "value": "PLAYOUT_NLE", "type": "java.lang.String"}, + {"name": "promoOutputFolder", "value": "PROMO_NLE", "type": "java.lang.String"}, + {"name": "advertisementOutputFolder", "value": "REKLAM_NLE", "type": "java.lang.String"}, + {"name": "octopusOutputFolder", "value": "OCTOPUS", "type": "java.lang.String"}, + {"name": "genericOutputFolder", "value": "ARCHIVE_RESTORE", "type": "java.lang.String"}, + {"name": "onlineOutputFolder", "value": "ONLINE", "type": "java.lang.String"}, + {"name": "killDateDays", "value": 7, "type": "java.lang.Integer"}, + {"name": "nexioAgency", "value": "ARCHIVE_RESTORE", "type": "java.lang.String"}, + {"name": "nexioPort", "value": 2098, "type": "java.lang.Integer"}, + {"name": "nexioUserName", "value": "administrator", "type": "java.lang.String"}, + {"name": "nexioPassword", "value": "system", "type": "java.lang.String"} + ] + }, + { + "name" : "sys: MORPHEUS 'missing materials' importálása", + "template": "sys-import-morpheus-missing-materials.xml", + "parameters": [ + {"name": "csvFilePath", "value": "/mnt/MORPHEUS", "type": "java.lang.String"}, + {"name": "processedFolder", "value": "DONE", "type": "java.lang.String"}, + {"name": "targetPath", "value": "/mnt/ISILON/PLAYOUT/Video", "type": "java.lang.String"} + ] + }, + { + "name" : "sys: MORPHEUS 'missing material' visszatöltése", + "template": "sys-retrieve-missing-material.xml", + "parameters": [ + {"name": "targetPath", "value": "/mnt/ISILON/PLAYOUT/Video", "type": "java.lang.String"}, + {"name": "globalRetrievePath", "value": "\\\\10.10.1.100\\BRAAVOS\\PLAYOUT\\Video", "type": "java.lang.String"}, + {"name": "morpheusDeviceID", "value": "ISILON", "type": "java.lang.String"}, + {"name": "dbUrl", "value": "jdbc:sqlserver://10.10.1.45;databaseName=PA_Echo;", "type": "java.lang.String"}, + {"name": "userName", "value": "MAM", "type": "java.lang.String"}, + {"name": "password", "value": "Echotv.hu", "type": "java.lang.String"}, + {"name": "targetMetadataPath", "value": "/mnt/ISILON/PLAYOUT/MorpheusGateway/Input", "type": "java.lang.String"} + ] + } +]} + + + diff --git a/server/-product/production/HIRTV/jobs/schedules.json b/server/-product/production/HIRTV/jobs/schedules.json index b500c0c8..2f921d82 100644 --- a/server/-product/production/HIRTV/jobs/schedules.json +++ b/server/-product/production/HIRTV/jobs/schedules.json @@ -1,336 +1,8 @@ -{"joblist":[ - { - "template": "cancelable.xml" - }, - { - "active": true, - "executeimmediate": true, - "name" : "OCTOPUS adatok szinkronizálása", - "template": "sync-octopus.xml", - "cronexpression": "0/30 * * * * ?", - "parameters": [ - {"name": "includeArchived", "value": false, "type": "java.lang.Boolean"} - ] - }, - { - "active": true, - "executeimmediate": false, - "name" : "Limitált archiválás az ISILON/ARCHIVE mappából", - "template": "archive-limited.xml", - "cronexpression": "0 0 6-22/1 * * ?", - "parameters": [ - {"name": "sourcePath", "value": "/mnt/ISILON/ARCHIVE", "type": "java.lang.String"}, - {"name": "killDateDays", "value": 1, "type": "java.lang.Integer"}, - {"name": "limit", "value": 20, "type": "java.lang.Integer"} - ] - }, - { - "active": true, - "executeimmediate": false, - "name" : "Archiválás az ISILON/ARCHIVE mappából", - "template": "archive-ondemand.xml", - "cronexpression": "0 0 1 * * ?", - "parameters": [ - {"name": "sourcePath", "value": "/mnt/ISILON/ARCHIVE", "type": "java.lang.String"}, - {"name": "killDateDays", "value": 1, "type": "java.lang.Integer"}, - {"name": "limit", "value": 0, "type": "java.lang.Integer"} - ] - }, - { - "active": true, - "executeimmediate": false, - "name" : "PROXY pótlás FFASTRANS 71", - "template": "sys-recreate-lowres-71.xml", - "cronexpression": "0 * * * * ?", - "parameters": [ - {"name": "localLowresPath", "value": "/mnt/FIXTRANSCODER-71/OUTPUT", "type": "java.lang.String" }, - {"name": "transcoderAddress", "value": "http://10.10.1.71:65445/api/json/v1/", "type": "java.lang.String"}, - {"name": "globalRetrievePath", "value": "file://10.10.1.30/transcode", "type": "java.lang.String"}, - {"name": "localRetrievePath", "value": "/mediacube/data/lowres/www/video/IFT3/transcode", "type": "java.lang.String"}, - {"name": "localHiresPath", "value": "/mediacube/data/lowres/www/video/IFT3/transcode", "type": "java.lang.String" }, - {"name": "globalHiresPath", "value": "L:\\transcode", "type": "java.lang.String" }, - {"name": "transcoderTemplateName", "value": "MAM_proxy", "type": "java.lang.String"} - ] - }, - { - "active": true, - "executeimmediate": false, - "name" : "PROXY pótlás FFASTRANS 72", - "template": "sys-recreate-lowres-72.xml", - "cronexpression": "10 * * * * ?", - "parameters": [ - {"name": "localLowresPath", "value": "/mnt/FIXTRANSCODER-72/OUTPUT", "type": "java.lang.String" }, - {"name": "transcoderAddress", "value": "http://10.10.1.72:65445/api/json/v1/", "type": "java.lang.String"}, - {"name": "globalRetrievePath", "value": "file://10.10.1.30/transcode", "type": "java.lang.String"}, - {"name": "localRetrievePath", "value": "/mediacube/data/lowres/www/video/IFT3/transcode", "type": "java.lang.String"}, - {"name": "localHiresPath", "value": "/mediacube/data/lowres/www/video/IFT3/transcode", "type": "java.lang.String" }, - {"name": "globalHiresPath", "value": "L:\\transcode", "type": "java.lang.String" }, - {"name": "transcoderTemplateName", "value": "MAM_proxy", "type": "java.lang.String"} - ] - }, - { - "active": true, - "executeimmediate": false, - "name" : "PROXY pótlás FFASTRANS 73", - "template": "sys-recreate-lowres-73.xml", - "cronexpression": "20 * * * * ?", - "parameters": [ - {"name": "localLowresPath", "value": "/mnt/FIXTRANSCODER-73/OUTPUT", "type": "java.lang.String" }, - {"name": "transcoderAddress", "value": "http://10.10.1.73:65445/api/json/v1/", "type": "java.lang.String"}, - {"name": "globalRetrievePath", "value": "file://10.10.1.30/transcode", "type": "java.lang.String"}, - {"name": "localRetrievePath", "value": "/mediacube/data/lowres/www/video/IFT3/transcode", "type": "java.lang.String"}, - {"name": "localHiresPath", "value": "/mediacube/data/lowres/www/video/IFT3/transcode", "type": "java.lang.String" }, - {"name": "globalHiresPath", "value": "L:\\transcode", "type": "java.lang.String" }, - {"name": "transcoderTemplateName", "value": "MAM_proxy", "type": "java.lang.String"} - ] - }, - { - "active": true, - "executeimmediate": false, - "name" : "PROXY pótlás FFASTRANS 74", - "template": "sys-recreate-lowres-74.xml", - "cronexpression": "30 * * * * ?", - "parameters": [ - {"name": "localLowresPath", "value": "/mnt/FIXTRANSCODER-74/OUTPUT", "type": "java.lang.String" }, - {"name": "transcoderAddress", "value": "http://10.10.1.74:65445/api/json/v1/", "type": "java.lang.String"}, - {"name": "globalRetrievePath", "value": "file://10.10.1.30/transcode", "type": "java.lang.String"}, - {"name": "localRetrievePath", "value": "/mediacube/data/lowres/www/video/IFT3/transcode", "type": "java.lang.String"}, - {"name": "localHiresPath", "value": "/mediacube/data/lowres/www/video/IFT3/transcode", "type": "java.lang.String" }, - {"name": "globalHiresPath", "value": "L:\\transcode", "type": "java.lang.String" }, - {"name": "transcoderTemplateName", "value": "MAM_proxy", "type": "java.lang.String"} - ] - }, - { - "name" : "SYS: create-lowres-ondemand", - "template": "create-lowres-ondemand.xml", - "parameters": [ - {"name": "globalRetrievePath", "value": "file://10.10.1.30/transcode", "type": "java.lang.String"}, - {"name": "localRetrievePath", "value": "/mediacube/data/lowres/www/video/IFT3/transcode", "type": "java.lang.String"}, - {"name": "localHiresPath", "value": "/mediacube/data/lowres/www/video/IFT3/transcode", "type": "java.lang.String" }, - {"name": "globalHiresPath", "value": "L:\\transcode", "type": "java.lang.String" }, - {"name": "localLowresPath", "value": "/mnt/FIXTRANSCODER/OUTPUT", "type": "java.lang.String" }, - {"name": "transcoderAddress", "value": "http://10.10.1.74:65445/api/json/v1/", "type": "java.lang.String"}, - {"name": "transcoderTemplateName", "value": "MAM_proxy", "type": "java.lang.String"} - ] - }, - { - "active": true, - "executeimmediate": false, - "name" : "Lejárt ISILON/TQC/CHECK/KESZ anyagok törlése", - "template": "delete-materials.xml", - "cronexpression": "0 0 8 * * ?", - "parameters": [ - {"name": "sourcePath", "value": "/mnt/ISILON/TQC/CHECK/KESZ", "type": "java.lang.String"} - ] - }, - { - "active": true, - "executeimmediate": false, - "name" : "Lejárt ISILON/TQC/PROMO/KESZ anyagok törlése", - "template": "delete-materials.xml", - "cronexpression": "0 0 6 * * ?", - "parameters": [ - {"name": "sourcePath", "value": "/mnt/ISILON/TQC/PROMO/KESZ", "type": "java.lang.String"} - ] - }, - { - "active": true, - "executeimmediate": false, - "name" : "Lejárt ISILON/TQC/REKLAM/KESZ anyagok törlése", - "template": "delete-materials.xml", - "cronexpression": "0 0 7 * * ?", - "parameters": [ - {"name": "sourcePath", "value": "/mnt/ISILON/TQC/REKLAM/KESZ", "type": "java.lang.String"} - ] - }, - { +{ + "joblist": [{ "active": true, - "executeimmediate": false, - "name" : "Lejárt ISILON/ARCHIVE anyagok törlése", - "template": "delete-materials.xml", - "cronexpression": "0 0 5 * * ?", - "parameters": [ - {"name": "sourcePath", "value": "/mnt/ISILON/ARCHIVE", "type": "java.lang.String"} - ] - }, - { - "active": true, - "executeimmediate": false, - "name" : "Lejárt ISILON/OCTOPUS/_NAPI_MEGTEKINTO anyagok törlése", - "template": "delete-materials.xml", - "cronexpression": "0 0 22 * * ?", - "parameters": [ - {"name": "sourcePath", "value": "/mnt/ISILON/OCTOPUS/_NAPI_MEGTEKINTO", "type": "java.lang.String"} - ] - }, - { - "active": true, - "executeimmediate": false, - "name" : "Lejárt ISISLON/PROMO_NLE mappák törlése", - "template": "delete-promo-materials.xml", - "cronexpression": "0 0 6 * * ?", - "parameters": [ - {"name": "sourcePath", "value": "/mnt/ISILON/PROMO_NLE", "type": "java.lang.String"} - ] - }, - { - "active": true, - "executeimmediate": false, - "name" : "NEXIO bejátszó anyagok másolása az ISILON/ARCHIVE mappába", - "template": "copyforarchive-nexio-materials.xml", - "cronexpression": "0 0 10 * * ?", - "parameters": [ - {"name": "nexioPort", "value": 2098, "type": "java.lang.Integer"}, - {"name": "nexioUserName", "value": "administrator", "type": "java.lang.String"}, - {"name": "nexioPassword", "value": "system", "type": "java.lang.String"}, - {"name": "archiveFtp", "value": "ftp://10.10.1.100/ARCHIVE", "type": "java.lang.String"}, - {"name": "archiveUserName", "value": "mediacube", "type": "java.lang.String"}, - {"name": "archivePassword", "value": "Broadca5T", "type": "java.lang.String"}, - {"name": "daysBeforeNow", "value": 1, "type": "java.lang.Integer"}, - {"name": "nexioKillDateDays", "value": 21, "type": "java.lang.Integer"}, - {"name": "nexioAgency", "value": "ARCHIVED", "type": "java.lang.String"} - ] - }, - { - "active": true, - "executeimmediate": false, - "name" : "NEXIO visszarögzített anyagok másolása az ISILON/ARCHIVE mappába", - "template": "copyforarchive-nexio-recordings.xml", - "cronexpression": "0 0 12 * * ? *", - "parameters": [ - {"name": "nexioPort", "value": 2098, "type": "java.lang.Integer"}, - {"name": "nexioUserName", "value": "administrator", "type": "java.lang.String"}, - {"name": "nexioPassword", "value": "system", "type": "java.lang.String"}, - {"name": "archiveFtp", "value": "ftp://10.10.1.100/ARCHIVE", "type": "java.lang.String"}, - {"name": "archiveUserName", "value": "mediacube", "type": "java.lang.String"}, - {"name": "archivePassword", "value": "Broadca5T", "type": "java.lang.String"}, - {"name": "filterAgencies", "value": "schedule-rec", "type": "java.lang.String"}, - {"name": "limit", "value": 30, "type": "java.lang.Integer"}, - {"name": "nexioKillDateDays", "value": 7, "type": "java.lang.Integer"}, - {"name": "nexioAgency", "value": "HIRADO_ARCHIVED", "type": "java.lang.String"} - ] - }, - { - "active": true, - "executeimmediate": false, - "name" : "Híranyag statisztika importálása", - "template": "import-statistics.xml", - "cronexpression": "0 0 6 * * ?", - "parameters": [ - {"name": "daysBeforeNow", "value": 1, "type": "java.lang.Integer"} - ] - }, - { - "active": false, - "executeimmediate": true, - "name" : "TRAFFIC anyagok visszatöltése", - "template": "retrieve-traffic-missing-materials.xml", - "cronexpression": "0 0 0/1 1/1 * ? *", - "parameters": [ - {"name": "dbUrl", "value": "jdbc:sqlserver://10.10.1.45\\sql16;databaseName=PA_Echo;", "type": "java.lang.String"}, - {"name": "userName", "value": "MAM", "type": "java.lang.String"}, - {"name": "password", "value": "Echotv.hu", "type": "java.lang.String"}, - {"name": "lookupDays", "value": 7, "type": "java.lang.Integer"}, - {"name": "targetPath", "value": "/mnt/ISILON/PLAYOUT/IceGateway/Input", "type": "java.lang.String"}, - {"name": "killDateDays", "value": 10, "type": "java.lang.Integer"} - ] - }, - { - "active": false, - "executeimmediate": true, - "name" : "Lejárt ISISLON/ARCHIVE_TEMP anyagok törlése", - "template": "delete-materials.xml", - "cronexpression": "0 0 0/1 1/1 * ? *", - "parameters": [ - {"name": "sourcePath", "value": "/mnt/ISILON/ARCHIVE_TEMP", "type": "java.lang.String"} - ] - }, - { - "active": true, - "executeimmediate": false, - "name" : "Lejárt NEXIO anyagok törlése", - "template": "delete-nexio-materials.xml", - "cronexpression": "0 0 6 * * ?", - "parameters": [ - {"name": "port", "value": 2098, "type": "java.lang.Integer"}, - {"name": "userName", "value": "administrator", "type": "java.lang.String"}, - {"name": "password", "value": "system", "type": "java.lang.String"}, - {"name": "filterAgencies", "value": "HIRADO_23_00,HIRADO_ARCHIVED,HIRADO_CLN", "type": "java.lang.String"}, - {"name": "gracePeriodDays", "value": 1, "type": "java.lang.Integer"}, - {"name": "notificationOnly", "value": false, "type": "java.lang.Boolean"} - ] - }, - { - "active": false, - "executeimmediate": false, - "name" : "Párhuzamosított teszt folyamat", - "template": "fake-concurrent.xml", - "cronexpression": "0/10 * * ? * *", - "parameters": [ {"name": "itemID", "value": 1, "type": "java.lang.Long"} ] - }, - { - "active": false, - "executeimmediate": false, - "name" : "Teszt folyamat", - "template": "fake.xml", - "cronexpression": "0/10 * * ? * *", - "parameters": [ {"name": "itemID", "value": 1, "type": "java.lang.Long"} ] - }, - { - "name" : "SYS: MORPHEUS 'missing materials' importálása", - "template": "import-morpheus-missing-materials.xml", - "parameters": [ - {"name": "csvFilePath", "value": "/mnt/MORPHEUS", "type": "java.lang.String"}, - {"name": "processedFolder", "value": "DONE", "type": "java.lang.String"}, - {"name": "targetPath", "value": "/mnt/ISILON/ARCHIVE_RESTORE", "type": "java.lang.String"} - ] - }, - { - "name" : "SYS: batch-retrieve-ondemand", - "template": "batch-retrieve-ondemand.xml", - }, - { - "name" : "SYS: retrieve-ondemand", - "template": "retrieve-ondemand.xml", - "parameters": [ - {"name": "globalRetrievePath", "value": "file://isilon.intra.echotv.hu", "type": "java.lang.String"}, - {"name": "localRetrievePath", "value": "/mnt/ISILON", "type": "java.lang.String"}, - {"name": "materialOutputFolder", "value": "PLAYOUT_NLE", "type": "java.lang.String"}, - {"name": "promoOutputFolder", "value": "PROMO_NLE", "type": "java.lang.String"}, - {"name": "advertisementOutputFolder", "value": "REKLAM_NLE", "type": "java.lang.String"}, - {"name": "octopusOutputFolder", "value": "OCTOPUS", "type": "java.lang.String"}, - {"name": "genericOutputFolder", "value": "ARCHIVE_RESTORE", "type": "java.lang.String"}, - {"name": "onlineOutputFolder", "value": "ONLINE", "type": "java.lang.String"}, - {"name": "killDateDays", "value": 7, "type": "java.lang.Integer"}, - {"name": "nexioAgency", "value": "ARCHIVE_RESTORE", "type": "java.lang.String"}, - {"name": "nexioPort", "value": 2098, "type": "java.lang.Integer"}, - {"name": "nexioUserName", "value": "administrator", "type": "java.lang.String"}, - {"name": "nexioPassword", "value": "system", "type": "java.lang.String"} - ] - }, - { - "name" : "sys: MORPHEUS 'missing materials' importálása", - "template": "sys-import-morpheus-missing-materials.xml", - "parameters": [ - {"name": "csvFilePath", "value": "/mnt/MORPHEUS", "type": "java.lang.String"}, - {"name": "processedFolder", "value": "DONE", "type": "java.lang.String"}, - {"name": "targetPath", "value": "/mnt/ISILON/PLAYOUT/Video", "type": "java.lang.String"} - ] - }, - { - "name" : "sys: MORPHEUS 'missing material' visszatöltése", - "template": "sys-retrieve-missing-material.xml", - "parameters": [ - {"name": "targetPath", "value": "/mnt/ISILON/PLAYOUT/Video", "type": "java.lang.String"}, - {"name": "globalRetrievePath", "value": "\\\\10.10.1.100\\BRAAVOS\\PLAYOUT\\Video", "type": "java.lang.String"}, - {"name": "morpheusDeviceID", "value": "ISILON", "type": "java.lang.String"}, - {"name": "dbUrl", "value": "jdbc:sqlserver://10.10.1.45;databaseName=PA_Echo;", "type": "java.lang.String"}, - {"name": "userName", "value": "MAM", "type": "java.lang.String"}, - {"name": "password", "value": "Echotv.hu", "type": "java.lang.String"}, - {"name": "targetMetadataPath", "value": "/mnt/ISILON/PLAYOUT/MorpheusGateway/Input", "type": "java.lang.String"} - ] - } -]} - - - + "cronexpression": "0/2 * * * * ?", + "template": "cancelable.xml" + } + ] +} \ No newline at end of file diff --git a/server/-product/production/HIRTV/jobs/templates/cancelable.xml b/server/-product/production/HIRTV/jobs/templates/cancelable.xml index a400a93b..4f3b9f86 100644 --- a/server/-product/production/HIRTV/jobs/templates/cancelable.xml +++ b/server/-product/production/HIRTV/jobs/templates/cancelable.xml @@ -1,5 +1,5 @@ - + diff --git a/server/-product/production/HIRTV/jobs/templates/delete-materials.xml b/server/-product/production/HIRTV/jobs/templates/delete-materials1.xml similarity index 75% rename from server/-product/production/HIRTV/jobs/templates/delete-materials.xml rename to server/-product/production/HIRTV/jobs/templates/delete-materials1.xml index 82b30492..af64d93c 100644 --- a/server/-product/production/HIRTV/jobs/templates/delete-materials.xml +++ b/server/-product/production/HIRTV/jobs/templates/delete-materials1.xml @@ -3,6 +3,7 @@ + @@ -11,6 +12,9 @@ + + + diff --git a/server/-product/production/HIRTV/jobs/templates/delete-materials2.xml b/server/-product/production/HIRTV/jobs/templates/delete-materials2.xml new file mode 100644 index 00000000..af64d93c --- /dev/null +++ b/server/-product/production/HIRTV/jobs/templates/delete-materials2.xml @@ -0,0 +1,21 @@ + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/server/-product/production/HIRTV/jobs/templates/delete-materials3.xml b/server/-product/production/HIRTV/jobs/templates/delete-materials3.xml new file mode 100644 index 00000000..af64d93c --- /dev/null +++ b/server/-product/production/HIRTV/jobs/templates/delete-materials3.xml @@ -0,0 +1,21 @@ + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/server/-product/production/HIRTV/jobs/templates/delete-materials4.xml b/server/-product/production/HIRTV/jobs/templates/delete-materials4.xml new file mode 100644 index 00000000..af64d93c --- /dev/null +++ b/server/-product/production/HIRTV/jobs/templates/delete-materials4.xml @@ -0,0 +1,21 @@ + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/server/-product/production/HIRTV/jobs/templates/delete-materials5.xml b/server/-product/production/HIRTV/jobs/templates/delete-materials5.xml new file mode 100644 index 00000000..af64d93c --- /dev/null +++ b/server/-product/production/HIRTV/jobs/templates/delete-materials5.xml @@ -0,0 +1,21 @@ + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/server/-product/production/HIRTV/jobs/templates/sync-octopus.xml b/server/-product/production/HIRTV/jobs/templates/sync-octopus.xml index 86644526..7d870f8a 100644 --- a/server/-product/production/HIRTV/jobs/templates/sync-octopus.xml +++ b/server/-product/production/HIRTV/jobs/templates/sync-octopus.xml @@ -3,6 +3,9 @@ + + + @@ -11,6 +14,15 @@ + + + + + + + + + diff --git a/server/-product/production/LOCAL/jobs/steps/ArchiveListBuilderStep.java b/server/-product/production/LOCAL/jobs/steps/ArchiveListBuilderStep.java deleted file mode 100644 index b2321829..00000000 --- a/server/-product/production/LOCAL/jobs/steps/ArchiveListBuilderStep.java +++ /dev/null @@ -1,167 +0,0 @@ -package user.jobengine.server.steps; - -import java.io.File; -import java.io.IOException; -import java.nio.file.DirectoryStream; -import java.nio.file.Files; -import java.nio.file.Path; -import java.nio.file.Paths; -import java.util.LinkedList; -import java.util.List; - -import org.apache.commons.lang.StringUtils; -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; -import org.apache.logging.log4j.Marker; - -import user.jobengine.server.IJobEngine; -import user.jobengine.server.IJobRuntime; - -/** - * Az archivalhato mediak listazasa MediaFileWrapper objektumokban. A listazott media allomanyokat megjeloli .catched signal allomannyal, hogy a legkozelebbi - * listazas figyelmen kivul hagyja. - * - * @author robi - */ -public class ArchiveListBuilderStep extends JobStep { - private static final Logger logger = LogManager.getLogger(); - // private static final String UTF8 = "utf-8"; - private static final String STATUSFOLDER = ".STATUS"; - private static final String JSONEXT = ".json"; - private static final String CATCHEDEXT = ".catched"; - - public static final String ITEM_TITLE = "itemTitle"; - public static final String ITEM_HOUSEID = "itemHouseId"; - public static final String ITEM_DESCRIPTION = "itemDescription"; - public static final String MEDIA_HOUSEID = "mediaHouseId"; - public static final String MEDIA_TITLE = "mediaTitle"; - public static final String MEDIA_DESCRIPTION = "mediaDescription"; - public static final String MEDIA_TYPE = "mediaType"; - private static final String DURATION = "duration"; - private static final String EXISTING_MEDIAID = "existingMediaId"; - private static final String TAGS = "tags"; - - private Marker marker; - - private ArchiveItem createArchiveItem(Path jsonFilePath, Path mediaFilePath, Path catchedFilePath) { - ArchiveItem result = null; - try { - result = ArchiveItem.fromFile(jsonFilePath); - result.setMediaFile(mediaFilePath.toString()); - result.setCatchedFile(catchedFilePath.toString()); - } catch (Exception e) { - logger.catching(e); - } - - return result; - } - - private void createCatchedFile(Path catchedFilePath) { - try { - Files.createFile(catchedFilePath); - //Files.write(catchedFilePath, CATCHED.getBytes(UTF8), StandardOpenOption.CREATE); - } catch (Exception e) { - logger.catching(e); - } - } - - @StepEntry - public Object[] execute(String sourcePath, int limit, IJobEngine jobEngine, IJobRuntime jobRuntime) { - marker = jobRuntime.getSessionMarker(); - List archiveList = new LinkedList(); - DirectoryStream directoryStream = null; - try { - DirectoryStream stream = Files.newDirectoryStream(Paths.get(sourcePath)); - for (Path p : stream) { - processPathItem(p, archiveList); - } - } catch (Exception e) { - logger.catching(e); - logger.error(marker, "Az '{}' mappa elérése sikertelen. A rendszer hibaüzenete: {}", e.getMessage()); - } finally { - if (directoryStream != null) { - try { - directoryStream.close(); - } catch (IOException e) { - } - } - } - - if (limit > 0 && archiveList.size() > limit) { - archiveList = archiveList.subList(0, limit); - logger.info(marker, "A folyamat alkalmazza a beállított {} limitet.", limit); - } - - if (archiveList.size() == 0) - logger.info(marker, "Nincs archiválandó anyag."); - else - logger.info(marker, "Az archiváló folyamat {} új anyagot érzékelt.", archiveList == null ? 0 : archiveList.size()); - - for (ArchiveItem archiveItem : archiveList) { - createCatchedFile(Paths.get(archiveItem.getCatchedFile())); - } - - return new Object[] { archiveList }; - } - - private boolean processPathItem(Path mediaFilePath, final List archiveList) { - File mediaFile = mediaFilePath.toFile(); - - // if (mediaFile.length() > 0) - // return false; - - if (mediaFile.isDirectory()) { - return false; - } - - Path dotStorePath = Paths.get(mediaFilePath.getParent().toString(), STATUSFOLDER); - Path catchedFilePath = Paths.get(dotStorePath.toString(), mediaFile.getName() + CATCHEDEXT); - File catchedFile = catchedFilePath.toFile(); - if (catchedFile.exists()) { - logger.warn("{} file is already catched.", mediaFile.getName()); - return false; - } - - Path jsonFilePath = Paths.get(dotStorePath.toString(), mediaFile.getName() + JSONEXT); - File jsonFile = jsonFilePath.toFile(); - if (!jsonFile.exists()) { - logger.warn("{} has no json metadata.", mediaFile.getName()); - return false; - } - - ArchiveItem archiveItem = createArchiveItem(jsonFilePath, mediaFilePath, catchedFilePath); - - if (archiveItem == null) { - logger.warn("{} has no metadata specified.", mediaFile.getName()); - return false; - } - - if (StringUtils.isBlank(archiveItem.getItemHouseId())) { - logger.warn("{} has no Item HouseID specified in metadata.", mediaFile.getName()); - return false; - } - - if (StringUtils.isBlank(archiveItem.getItemTitle())) { - logger.warn("{} has no Item Title specified in metadata.", mediaFile.getName()); - return false; - } - - if (StringUtils.isBlank(archiveItem.getMediaHouseId())) { - logger.warn("{} has no Media HouseID specified in metadata.", mediaFile.getName()); - return false; - } - - if (StringUtils.isBlank(archiveItem.getMediaTitle())) { - logger.warn("{} has no Media Title specified in metadata.", mediaFile.getName()); - return false; - } - - //A tenyleges archivalast vesszuk elore - if (mediaFile.length() == 0) - archiveList.add(archiveItem); - else - archiveList.add(0, archiveItem); - //createCatchedFile(catchedFilePath); - return true; - } -} diff --git a/server/-product/production/LOCAL/jobs/steps/ArchiveMaterialSubmitStep.java b/server/-product/production/LOCAL/jobs/steps/ArchiveMaterialSubmitStep.java deleted file mode 100644 index f11e3a52..00000000 --- a/server/-product/production/LOCAL/jobs/steps/ArchiveMaterialSubmitStep.java +++ /dev/null @@ -1,53 +0,0 @@ -package user.jobengine.server.steps; - -import java.io.File; -import java.util.List; - -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; -import org.apache.logging.log4j.Marker; - -import user.commons.ListUtils; -import user.jobengine.server.IJobEngine; -import user.jobengine.server.IJobRuntime; - -public class ArchiveMaterialSubmitStep extends JobStep { - private static final Logger logger = LogManager.getLogger(); - private static final String JOBTEMPLATE = "archive-material.xml"; - private static final String KILL_DATE_DAYS = "killDateDays"; - private static final String ARCHIVE = "Archiválás"; - private static final String ARCHIVE_ITEM = "archiveItem"; - private Marker marker; - - @StepEntry - public Object[] execute(List archiveList, int killDateDays, IJobEngine jobEngine, IJobRuntime jobRuntime) throws Exception { - marker = jobRuntime.getSessionMarker(); - if (archiveList == null || archiveList.size() == 0) - return null; - - if (jobRuntime.forkPrepare()) { - for (int i = 0; i < archiveList.size(); i++) { - ArchiveItem archiveItem = archiveList.get(i); - try { - IJobRuntime runtime = jobEngine.submit(jobRuntime, null, JOBTEMPLATE, ARCHIVE, - ListUtils.asMap(ARCHIVE_ITEM, archiveItem, KILL_DATE_DAYS, killDateDays)); - - int progress = (i + 1) * 100 / archiveList.size(); - setProgress(progress); - if (runtime == null) - throw new Exception("Submit returned null runtime"); - } catch (Exception e) { - logger.catching(e); - String fileName = new File(archiveItem.getMediaFile()).getName(); - logger.error(marker, "Az '{}' állomány archiválási kísérlete sikertelen. A rendszer üzenete: {}", fileName, e.getMessage()); - if (!archiveItem.removeCatchedFile()) - logger.error(marker, "Az '{}' állomány .catched jelző állománya nem törölhető.", fileName); - throw e; - } - } - } - jobRuntime.forkWaitComplete(); - return null; - } - -} diff --git a/server/-product/production/LOCAL/jobs/steps/ArchiveRecursive.java b/server/-product/production/LOCAL/jobs/steps/ArchiveRecursive.java deleted file mode 100644 index 6a65d5fa..00000000 --- a/server/-product/production/LOCAL/jobs/steps/ArchiveRecursive.java +++ /dev/null @@ -1,291 +0,0 @@ -package user.jobengine.server.steps; - -import java.io.File; -import java.io.IOException; -import java.nio.file.FileVisitResult; -import java.nio.file.FileVisitor; -import java.nio.file.Files; -import java.nio.file.Path; -import java.nio.file.Paths; -import java.nio.file.attribute.BasicFileAttributes; -import java.text.SimpleDateFormat; -import java.util.Arrays; -import java.util.List; -import java.util.Map; - -import org.apache.commons.io.FilenameUtils; -import org.apache.commons.lang.StringUtils; -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; - -import com.ibm.nosql.json.api.BasicDBObject; - -import user.commons.JobStatus; -import user.commons.ListUtils; -import user.commons.mediatool.MediaInfo; -import user.jobengine.server.IJobEngine; -import user.jobengine.server.IJobRuntime; -import user.mediacube.metadata.interfaces.IMetadata; -import user.mediacube.metadata.interfaces.IMetadataProvider; -import user.mediacube.metadata.interfaces.IMetadataProviderFactory; -import user.mediacube.metadata.interfaces.MetadataProviderType; -import user.mediacube.metadata.interfaces.MetadataType; -import user.mediacube.metadata.interfaces.PlanAirMetadataListOptions; - -public class ArchiveRecursive extends JobStep implements FileVisitor { - private static final Logger logger = LogManager.getLogger(); - private static final String JOBTEMPLATE = "archive-material.xml"; - private static final String ITEM_TITLE = "itemTitle"; - private static final String ITEM_HOUSEID = "itemHouseId"; - private static final String MEDIA_HOUSEID = "mediaHouseId"; - private static final String MEDIA_TITLE = "mediaTitle"; - private static final String MEDIA_DESCRIPTION = "mediaDescription"; - private static final String MEDIA_TYPE = "mediaType"; - private static final String ARCHIVE = "Archiválás"; - private static final String ARCHIVE_ITEM = "archiveItem"; - private static final String KILL_DATE_DAYS = "killDateDays"; - private SimpleDateFormat df = new SimpleDateFormat("yyyy.MM.dd"); - - private List skipPathNames = Arrays.asList("!ARCHIVALAS_ALATT", "VASAROLT", EscortFiles.STATUSFOLDER, EscortFiles.CONFLICTFOLDER); - private int limit; - private int submitted; - private int killDateDays; - - private boolean canReadMediaInfo(Path mediaFilePath) { - boolean result = false; - try { - MediaInfo mi = new MediaInfo(mediaFilePath); - mi.process(); - result = true; - } catch (Exception e) { - logger.warn(getSessionMarker(), e.getMessage()); - } - return result; - } - - private void checkArchiveItem(ArchiveItem archiveItem, Path mediaPath) throws Exception { - if (archiveItem == null) - throw new Exception(mediaPath + " has no metadata specified."); - - if (StringUtils.isBlank(archiveItem.getItemHouseId())) - throw new Exception(mediaPath + " has no Item HouseID specified in metadata."); - - if (StringUtils.isBlank(archiveItem.getItemTitle())) - throw new Exception(mediaPath + " has no Item Title specified in metadata."); - - if (StringUtils.isBlank(archiveItem.getMediaHouseId())) - throw new Exception(mediaPath + " has no Media HouseID specified in metadata."); - - if (StringUtils.isBlank(archiveItem.getMediaTitle())) - throw new Exception(mediaPath + " has no Media Title specified in metadata."); - } - - private ArchiveItem createArchiveItem(Path filePath) throws Exception { - ArchiveItem result = null; - - String fileName = filePath.getFileName().toString(); - String mediaHouseId = FilenameUtils.removeExtension(fileName); - try { - result = getPlanAirMetadata(mediaHouseId); - } catch (Exception e) { - - logger.error("PlanAir metadata error", e); - //nem latja a drivert pl. - throw e; - } - - if (result == null) { - result = new ArchiveItem(); - BasicFileAttributes attr = Files.readAttributes(filePath, BasicFileAttributes.class); - result.setItemHouseId(df.format(attr.lastModifiedTime().toMillis())); - result.setItemTitle(filePath.getParent().toString()); - result.setMediaHouseId(mediaHouseId); - result.setMediaTitle(fileName); - //TODO kesobb kivenni - //result.setMediaDescription("/OMARCHIVE"); - result.setMediaType("Generic"); - } - - result.setMediaFile(filePath.toString()); - return result; - } - - @StepEntry - public Object[] execute(String sourcePath, int killDateDays, int limit) throws Exception { - this.killDateDays = killDateDays; - this.limit = limit; - try { - if (getJobRuntime().forkPrepare()) { - Files.walkFileTree(Paths.get(sourcePath), this); - } - } catch (Exception e) { - logger.error(getSessionMarker(), "Az '{}' mappa elérése sikertelen. A rendszer hibaüzenete: {}", sourcePath, e.getMessage()); - } finally { - if (submitted > 0) - getJobRuntime().forkWaitComplete(); - else - getJobRuntime().cancelForkPrepare(); - - } - return null; - } - - private ArchiveItem getPlanAirMetadata(String mediaHouseId) throws Exception { - PlanAirMetadataListOptions opt = new PlanAirMetadataListOptions(); - opt.setSearch(mediaHouseId); - opt.setType(MetadataType.Material); - - BasicDBObject json = null; - List data = null; - - IMetadataProviderFactory factory = getService(IMetadataProviderFactory.class); - IMetadataProvider planairProvider = factory.getProvider(MetadataProviderType.PLANAIR); - - ArchiveItem result = null; - data = planairProvider.list(opt); - if (data.size() != 0) - json = data.get(0).asJSON(); - else { - opt.setType(MetadataType.Promo); - data = planairProvider.list(opt); - if (data.size() != 0) - json = data.get(0).asJSON(); - else { - opt.setType(MetadataType.AD); - data = planairProvider.list(opt); - if (data.size() != 0) - json = data.get(0).asJSON(); - } - } - if (json != null) { - result = new ArchiveItem(); - result.setItemHouseId(json.getString(ITEM_HOUSEID)); - result.setItemTitle(json.getString(ITEM_TITLE)); - result.setMediaHouseId(json.getString(MEDIA_HOUSEID)); - result.setMediaTitle(json.getString(MEDIA_TITLE)); - result.setMediaDescription(json.getString(MEDIA_DESCRIPTION)); - result.setMediaType(json.getString(MEDIA_TYPE)); - } - return result; - } - - private boolean handleArchiveConflict(Path mediaPath) throws Exception { - boolean result = false; - String sourceFileName = mediaPath.getFileName().toString(); - if (getManager().isMediaFileExists(sourceFileName)) { - EscortFiles.createMediaCatch(mediaPath); - result = true; - - // try { - // Path parent = sourcePath.getParent(); - // Path conflictPath = Paths.get(parent.toString(), CONFLICT); - // File folder = conflictPath.toFile(); - // if (!folder.exists() || !folder.isDirectory()) { - // Set perms = PosixFilePermissions.fromString("rwxrwxrwx"); - // FileAttribute> attr = PosixFilePermissions.asFileAttribute(perms); - // try { - // Files.createDirectories(conflictPath, attr); - // } catch (Exception e) { - // try { - // Files.createDirectory(conflictPath); - // } catch (Exception e1) { - // logger.catching(e); - // throw e; - // } - // } - // } - // - // Path conflictFile = Paths.get(conflictPath.toString(), sourceFileName); - // if (conflictFile.toFile().exists()) - // conflictFile = Paths.get(conflictPath.toString(), sourceFileName + (new Date()).getTime()); - // Files.move(sourcePath, conflictFile); - // result = true; - // } catch (Exception e1) { - // logger.error(getSessionMarker(), "Hiba az '{}' állomány mappába mozgatásakor. A rendszer üzenete: {}", CONFLICT, e1.getMessage()); - // } - } - - return result; - } - - @Override - public FileVisitResult postVisitDirectory(Path paramT, IOException paramIOException) throws IOException { - return FileVisitResult.CONTINUE; - } - - @Override - public FileVisitResult preVisitDirectory(Path dir, BasicFileAttributes paramBasicFileAttributes) throws IOException { - Path dirName = dir.getFileName(); - logger.info("PreVisit {}", dir); - - if (skipPathNames.contains(dirName.toString())) - return FileVisitResult.SKIP_SUBTREE; - else - logger.info("PreVisit {}", dir); - - return FileVisitResult.CONTINUE; - } - - private boolean processPathItem(Path mediaPath) throws Exception { - if (submitted == limit) - return false; - - // if (!"Transfered".equals(mediaPath.getParent().getFileName().toString())) - // return false; - - if (mediaPath.getFileName().toString().startsWith(".")) - return false; - - File mediaFile = mediaPath.toFile(); - if (mediaFile.isDirectory()) - return false; - - if (EscortFiles.isMediaCatched(mediaPath)) - return false; - - if (handleArchiveConflict(mediaPath)) - return false; - - if (!canReadMediaInfo(mediaPath)) - return false; - - ArchiveItem archiveItem = createArchiveItem(mediaPath); - - try { - checkArchiveItem(archiveItem, mediaPath); - submitted++; - - Map parameters = ListUtils.asMap(ARCHIVE_ITEM, archiveItem, KILL_DATE_DAYS, killDateDays); - IJobRuntime runtime = getEngine().submit(getJobRuntime(), e -> { - if (e.getStatus().equals(JobStatus.CANCELED) || e.getStatus().equals(JobStatus.SUSPENDED)) - EscortFiles.removeMediaCatch(mediaPath); - }, JOBTEMPLATE, ARCHIVE, 1, IJobEngine.DEFAULT_OWNER, parameters); - if (runtime == null) - throw new Exception("Submit returned null runtime"); - runtime.setRelated(mediaPath.toString()); - EscortFiles.createMediaCatch(mediaPath); - } catch (Exception e) { - String fileName = new File(archiveItem.getMediaFile()).getName(); - logger.error(getSessionMarker(), "Az '{}' állomány archiválási kísérlete sikertelen. A rendszer üzenete: {}", fileName, e.getMessage()); - } - - return true; - } - - @Override - public FileVisitResult visitFile(Path filePath, BasicFileAttributes paramBasicFileAttributes) throws IOException { - //logger.info("Will archived {}", filePath); - try { - processPathItem(filePath); - } catch (Exception e) { - logger.catching(e); - } - return FileVisitResult.CONTINUE; - } - - @Override - public FileVisitResult visitFileFailed(Path filePath, IOException paramIOException) throws IOException { - logger.info("Error archive {}", filePath); - return FileVisitResult.CONTINUE; - } -} diff --git a/server/-product/production/LOCAL/jobs/steps/BatchRetrieveForkStep.java b/server/-product/production/LOCAL/jobs/steps/BatchRetrieveForkStep.java deleted file mode 100644 index 15cf36c1..00000000 --- a/server/-product/production/LOCAL/jobs/steps/BatchRetrieveForkStep.java +++ /dev/null @@ -1,76 +0,0 @@ -package user.jobengine.server.steps; - -import java.util.List; -import java.util.Map; - -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; - -import user.commons.MediaCubeFinishMarker; -import user.commons.MediaCubeMarker; -import user.jobengine.db.ArchivedMedia; -import user.jobengine.server.IJobEngine; -import user.jobengine.server.IJobRuntime; -import user.jobengine.server.scheduler.ScheduledJob; - -public class BatchRetrieveForkStep extends JobStep { - private static final String TARGET_PATH_TYPE = "targetPathType"; - private static final Logger logger = LogManager.getLogger(); - private static final String CHILD_TEMPLATE = "retrieve-ondemand.xml"; - private static final String ARCHIVEDMEDIA = "archivedMedia"; - private static final String RECIPIENT = "successRecipient"; - private static final String HOUSEID = "houseId"; - private MediaCubeMarker marker; - - @StepEntry - public Object[] execute(List basket, String houseId, String recipient, String targetPathType, IJobEngine jobEngine, IJobRuntime jobRuntime) - throws Exception { - marker = (MediaCubeMarker) jobRuntime.getSessionMarker(); - - //session szinten csak a finishMarker cimzettje az erdekes, es ezt a cimet pluszban hasznalja a konfigban megadott cimmel - //a finishMarker orokli a cim bellitast a sessionMarkertol - marker.setTo(recipient); - - ((MediaCubeMarker) jobRuntime.getFinishMarker()).setTo(recipient); - - if (basket == null || basket.size() == 0) - return null; - setProgress(10); - - MediaCubeMarker mailMarker = new MediaCubeMarker(recipient); - mailMarker.setSessionName("Archívum viszatöltés"); - mailMarker.setSessionID(houseId); - logger.info(mailMarker, "A visszatöltések elindultak az alábbi állományokra:"); - - if (jobRuntime.forkPrepare()) { - for (ArchivedMedia archivedMedia : basket) { - logger.info(mailMarker, archivedMedia.getMedia().getMediaFilesName()); - submit(archivedMedia, recipient, houseId, targetPathType, jobEngine, jobRuntime); - } - } - setProgress(50); - logger.info(new MediaCubeFinishMarker(mailMarker), "A visszatöltések végeztével megerősítő üzenetet küldünk."); - jobRuntime.forkWaitComplete(); - setProgress(100); - return null; - } - - public void submit(ArchivedMedia archivedMedia, String recipient, String houseId, String targetPathType, IJobEngine jobEngine, IJobRuntime jobRuntime) - throws Exception { - try { - ScheduledJob scheduledJob = jobEngine.getScheduledJob(CHILD_TEMPLATE); - Map parameters = scheduledJob.getJobParameters(); - parameters.put(ARCHIVEDMEDIA, archivedMedia); - parameters.put(HOUSEID, houseId); - parameters.put(RECIPIENT, recipient); - parameters.put(TARGET_PATH_TYPE, targetPathType); - IJobRuntime child = jobEngine.submit(jobRuntime, null, CHILD_TEMPLATE, String.format("Visszatöltés %s részére", recipient), parameters); - ((MediaCubeMarker) child.getSessionMarker()).setTo(recipient); - } catch (Exception e) { - logger.catching(e); - logger.error(marker, "Hiba a kötegelt visszatöltésben. A rendszer üzenete: {}", e.getMessage()); - } - - } - -} diff --git a/server/-product/production/LOCAL/jobs/steps/CancelableStep.java b/server/-product/production/LOCAL/jobs/steps/CancelableStep.java index 11e27c7c..75bef7c5 100644 --- a/server/-product/production/LOCAL/jobs/steps/CancelableStep.java +++ b/server/-product/production/LOCAL/jobs/steps/CancelableStep.java @@ -1,51 +1,42 @@ package user.jobengine.server.steps; -import org.apache.commons.net.ftp.FTPClient; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; -import org.apache.logging.log4j.Marker; -import org.apache.logging.log4j.MarkerManager; -import user.commons.StoreUri; -import user.commons.remotestore.FtpDirectoryLister; -import user.commons.remotestore.RemoteStoreProtocol; - - -/* DO NOT REMOVE! -import user.jobengine.server.steps.TestLib; -*/ public class CancelableStep extends JobStep { private static final Logger logger = LogManager.getLogger(); int count = 10; @StepEntry public Object[] execute(int param) throws Exception { + + getJobRuntime().setRelated("TESZT" + param); + for (int i = 0; i < count; i++) { + if (getJobRuntime().isWaitingCancel()) + break; + if (i == 1) + throw new Exception("AAAAAAAAAA"); + + Thread.sleep(1000); + int progress = (i + 1) * 100 / count; + setProgress(progress); + } + try { - TestLib t = new TestLib(); - - Marker marker = MarkerManager.getMarker("MEDIAPROFILE"); - getJobRuntime().setRelated("TESZT" + param); - - for (int i = 0; i < count; i++) { - if (getJobRuntime().isWaitingCancel()) - break; - Thread.sleep(1000); - /* - if (param==0) - throw new Exception(); - if (param==1) { - getJobRuntime().setDisableRetry(true); - throw new Exception(); - } - */ - int progress = (i + 1) * 100 / count; - setProgress(progress); - } + + // logger.warn("Ez a fo logba megy"); + // + // //marker="MEDIACUBE" + // logger.warn(getMarker(), "Ez a markered logba megy"); + // + // //marker="MEDIACUBE | folyamat_nev" + // logger.warn(getJobRuntime().getSessionMarker(), "Ez a markered logba megy es a nevesitett logba"); + } catch (Exception e) { - logger.info(e.getMessage()); + e.printStackTrace(); throw e; } - return null; + return new Object[] { param }; } } diff --git a/server/-product/production/LOCAL/jobs/steps/CleanupMountedLocationStep.java b/server/-product/production/LOCAL/jobs/steps/CleanupMountedLocationStep.java deleted file mode 100644 index 4e6f10d1..00000000 --- a/server/-product/production/LOCAL/jobs/steps/CleanupMountedLocationStep.java +++ /dev/null @@ -1,316 +0,0 @@ -package user.jobengine.server.steps; - -import java.io.File; -import java.io.IOException; -import java.nio.file.DirectoryStream; -import java.nio.file.FileVisitResult; -import java.nio.file.FileVisitor; -import java.nio.file.Files; -import java.nio.file.Path; -import java.nio.file.Paths; -import java.nio.file.SimpleFileVisitor; -import java.nio.file.attribute.BasicFileAttributes; -import java.text.ParseException; -import java.text.SimpleDateFormat; -import java.util.ArrayList; -import java.util.Collections; -import java.util.Date; -import java.util.List; - -import org.apache.commons.io.FileUtils; -import org.apache.commons.lang.StringUtils; -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; -import org.apache.logging.log4j.Marker; - -import user.commons.RemoteFile; -import user.commons.StoreUri; -import user.commons.remotestore.RemoteStoreProtocol; -import user.jobengine.db.Store; -import user.jobengine.server.IJobEngine; -import user.jobengine.server.IJobRuntime; - -public class CleanupMountedLocationStep extends JobStep implements FileVisitor { - - private static final Logger logger = LogManager.getLogger(); - private static final String PROJECTFOLDER = "PROJECT"; - private static final String DATEFORMAT = "yyyyMMdd"; - private static final String DOT = "."; - private static final String STATUSFOLDER = ".STATUS"; - private static final String EWC2EXT = ".ewc2"; - private static final String XMPEXT = ".xmp"; - private static final String CATCHEDEXT = ".catched"; - private static final String KILLDATEEXT = ".killdate"; - private static final String JSONEXT = ".json"; - - private static boolean isEmpty(final Path directory) throws IOException { - try (DirectoryStream dirStream = Files.newDirectoryStream(directory)) { - final int[] count = new int[] { 0 }; - final int[] specialCount = new int[] { 0 }; - dirStream.forEach(p -> { - count[0]++; - // if (p.getFileName().toString().toLowerCase().equals(PROJECTFOLDER.toLowerCase())) - // specialCount[0]++; - if (p.getFileName().toString().toLowerCase().equals(STATUSFOLDER.toLowerCase())) - specialCount[0]++; - - }); - if (specialCount[0] == count[0]) - return true; - } - return false; - } - - private Marker marker; - - final int[] allCount = new int[] { 0 }; - final int[] currentCount = new int[] { 0 }; - - private Path sourcePath; - private SimpleDateFormat dateFormat; - private StoreUri tsmStoreUri; - - private Date checkExpiration(List killDateFiles) { - Date killDate = null; - for (Path killDateFile : killDateFiles) { - Date currentKillDate = getKillDate(killDateFile); - if (currentKillDate == null) - continue; - if ((killDate != null && currentKillDate.after(killDate)) || killDate == null) - killDate = currentKillDate; - } - return new Date().after(killDate) ? killDate : null; - } - - @StepEntry - public Object[] execute(String sourceFolder, IJobEngine jobEngine, IJobRuntime jobRuntime) throws Exception { - marker = getSessionMarker(); - sourcePath = Paths.get(sourceFolder); - DirectoryStream directoryStream = null; - if (StringUtils.isBlank(sourcePath.toString())) { - logger.error(marker, "A folyamat 'sourcePath' bemeneti paramétere üres."); - throw new NullPointerException("System is not configured properly, 'sourceFolder' input parameter missing."); - } - - if (!sourcePath.toFile().exists() || !sourcePath.toFile().isDirectory()) { - logger.error(marker, "A {} mappa nem létezik.", sourceFolder); - throw new NullPointerException(String.format("Directory %s not exists.", sourceFolder)); - } - - Store tsmStore = getManager().getSystemStore(false); - if (tsmStore == null) - throw new NullPointerException("A TSM bejegyzés nem található!"); - - tsmStoreUri = tsmStore.getSourceStoreUri(RemoteStoreProtocol.TSM); - if (tsmStoreUri == null) - throw new NullPointerException("A TSM forrás elérése nem található!"); - - try { - setProgress(1); - dateFormat = new SimpleDateFormat(DATEFORMAT); - - Files.walkFileTree(sourcePath, new SimpleFileVisitor() { - @Override - public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) throws IOException { - allCount[0]++; - return super.visitFile(file, attrs); - } - }); - Files.walkFileTree(sourcePath, this); - } catch (Exception e) { - logger.catching(e); - logger.error(marker, "Hiba a '{}' mappa feldolgozásában. A rendszer hibaüzenete: {}", sourcePath, e.getMessage()); - throw e; - } finally { - if (directoryStream != null) { - try { - directoryStream.close(); - } catch (IOException e) { - } - } - } - return null; - } - - private Date getKillDate(Path killDateFile) { - String fileName = killDateFile.getFileName().toString(); - int end = fileName.lastIndexOf(DOT); - if (end < 1) - return null; - int start = fileName.lastIndexOf(DOT, end - 1); - if (start < 0) - return null; - String strKillDate = fileName.substring(start + 1, end); - Date result = null; - if (StringUtils.isNumeric(strKillDate)) { - try { - result = dateFormat.parse(strKillDate); - } catch (ParseException e) { - logger.error(marker, "A {} fájl 'killdate' állománya hibás formátumú, a {} karaktersorozat nem konvertálható dátummá.", fileName, strKillDate); - return null; - } - } else - logger.error(marker, "A {} fájl 'killdate' állománya hibás formátumú, az dátum helyett ez áll: '{}'.", fileName, strKillDate); - return result; - } - - private List getKillDateFiles(Path filePath) { - String killDateFilePattern = String.format("%s.*%s", filePath.getFileName().toString(), KILLDATEEXT); - List result = new ArrayList<>(); - Path statusPath = null; - try { - statusPath = Paths.get(filePath.getParent().toString(), STATUSFOLDER); - } catch (Exception e) { - logger.catching(e); - return null; - } - File statusPathFile = statusPath.toFile(); - if (statusPathFile.exists() && statusPathFile.isDirectory()) { - try (DirectoryStream stream = Files.newDirectoryStream(statusPath, killDateFilePattern)) { - stream.forEach(p -> result.add(p)); - } catch (Exception e) { - logger.catching(e); - } - } - Collections.sort(result); - return result; - } - - private boolean isArchived(Path filePath) { - boolean result = false; - String name = filePath.getFileName().toString(); - String[] tsmName = new String[] { null }; - String query = String.format("SELECT relativepath FROM MEDIAFILE WHERE houseid = '%s'", name); - getManager().executeQuery(query, rs -> { - tsmName[0] = rs.getString("relativepath"); - return false; - }, null); - - if (tsmName[0] != null) { - try { - RemoteFile remoteFile = tsmStoreUri.getRemoteFile(tsmName[0]); - result = remoteFile != null; - } catch (Exception e) { - logger.error(e.getMessage()); - } finally { - tsmStoreUri.cleanUp(); - } - } - return result; - } - - @Override - public FileVisitResult postVisitDirectory(Path dir, IOException exc) throws IOException { - if (!dir.equals(sourcePath) && isEmpty(dir)) { - if (!removeExistingSpecialDirectory(dir, PROJECTFOLDER)) - return FileVisitResult.CONTINUE; - if (!removeExistingSpecialDirectory(dir, STATUSFOLDER)) - return FileVisitResult.CONTINUE; - if (removeFile(dir)) - logger.info(marker, "A {} üres mappa törlése sikeres.", dir); - - } - return FileVisitResult.CONTINUE; - } - - @Override - public FileVisitResult preVisitDirectory(Path dir, BasicFileAttributes attrs) throws IOException { - - //A .-al kezdodo mappakat kihagyjuk - if (dir.getFileName().toString().startsWith(".")) - return FileVisitResult.SKIP_SUBTREE; - - return FileVisitResult.CONTINUE; - } - - private void processPathItem(Path filePath) { - currentCount[0]++; - - if (filePath.getFileName().toString().startsWith(".")) - return; - - int progress = currentCount[0] * 100 / allCount[0]; - setProgress(progress); - - logger.info("Checking {}", filePath); - List killDateFiles = getKillDateFiles(filePath); - if (killDateFiles == null || killDateFiles.size() == 0) { - logger.warn(marker, "A {} fájlhoz nem található 'killdate' állomány.", filePath); - return; - } - - if (killDateFiles.size() != 1) - logger.warn(marker, "A {} fájlhoz több 'killdate' állomány található, a legújabb dátum határozza meg a törlés időpontját.", filePath); - - Date killDate = checkExpiration(killDateFiles); - if (killDate == null) - return; - - if (!isArchived(filePath)) { - logger.error(marker, "A(z) {} anyag törlésre van kijelölve, de nem található az archívumban.", filePath); - return; - } - - if (removeFiles(filePath, killDateFiles)) - logger.info(marker, "A {} fájl és kapcsolódó állományai a {} killdate bejegyzés alapján sikeresen törlődtek.", filePath.getFileName(), - dateFormat.format(killDate)); - else - logger.warn(marker, "A {} fájl és kapcsolódó állományai a {} killdate bejegyzés alapján csak részlegesen vagy egyáltalán nem törlődtek.", - filePath.getFileName(), dateFormat.format(killDate)); - } - - private boolean removeExistingSpecialDirectory(Path dir, String folderName) throws IOException { - File projectPath = Paths.get(dir.toString(), folderName).toFile(); - if (projectPath.exists() && projectPath.isDirectory()) { - FileUtils.deleteDirectory(projectPath); - if (projectPath.exists()) { - logger.warn(marker, "A {} alatti {} mappa törlése nem sikerült.", dir, folderName); - return false; - } - } - return true; - } - - private boolean removeFile(Path filePath) { - boolean result = false; - try { - //logger.error("REMOVE {}", filePath); - File file = filePath.toFile(); - if (file.exists()) - result = file.delete(); - } catch (Exception e) { - logger.error(marker, "A {} fájl nem törölhető. A rendszer hibaüzenete: {}", filePath, e.getMessage()); - } - return result; - } - - private boolean removeFiles(Path filePath, List killDateFiles) { - if (!removeFile(filePath)) - return false; - - removeFile(Paths.get(filePath.toString() + EWC2EXT)); - removeFile(Paths.get(filePath.toString() + XMPEXT)); - removeFile(Paths.get(filePath.getParent().toString(), STATUSFOLDER, filePath.getFileName().toString() + CATCHEDEXT)); - removeFile(Paths.get(filePath.getParent().toString(), STATUSFOLDER, filePath.getFileName().toString() + JSONEXT)); - - boolean result = true; - for (Path killDateFile : killDateFiles) { - if (!removeFile(killDateFile)) - result = false; - } - - return result; - } - - @Override - public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) throws IOException { - processPathItem(file); - return FileVisitResult.CONTINUE; - } - - @Override - public FileVisitResult visitFileFailed(Path file, IOException e) throws IOException { - logger.error(marker, "A {} fájl nem érhető el. A rendszer hibaüzenete: {}", file.toString(), e.getMessage()); - return FileVisitResult.CONTINUE; - } -} diff --git a/server/-product/production/LOCAL/jobs/steps/CreateArchiveItemStep.java b/server/-product/production/LOCAL/jobs/steps/CreateArchiveItemStep.java deleted file mode 100644 index b169edf5..00000000 --- a/server/-product/production/LOCAL/jobs/steps/CreateArchiveItemStep.java +++ /dev/null @@ -1,45 +0,0 @@ -package user.jobengine.server.steps; - -import java.nio.file.Paths; - -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; - -import com.ibm.nosql.json.api.BasicDBObject; -import com.ibm.nosql.json.api.DB; -import com.ibm.nosql.json.api.DBCollection; - -import user.commons.nosql.NoSQLUtils; -import user.jobengine.db.Media; -import user.jobengine.db.Store; - -public class CreateArchiveItemStep extends JobStep { - private static final Logger logger = LogManager.getLogger(); - - @StepEntry - public Object[] execute(Media mediaCubeMedia, String localHiresPath) throws Exception { - DB db = NoSQLUtils.getNoSQLDB(); - DBCollection collection = db.getCollection("missing_lowres"); - Store highResStore = getManager().getSystemStore(false); - - ArchiveItem archiveItem = null; - try { - if (mediaCubeMedia.getMediaFilesCount() != 1) - throw new Exception("Expected media count is 1, found " + mediaCubeMedia.getMediaFilesCount()); - if (mediaCubeMedia.getMediaFiles().get(0).getStoreId() != highResStore.getId()) - throw new Exception("Expected media store is a high-res store"); - - String name = mediaCubeMedia.getMediaFileRealName(); - archiveItem = new ArchiveItem(); - archiveItem.setMediaFile(Paths.get(localHiresPath, name).toString()); - collection.save(new BasicDBObject("name", name)); - } catch (Exception e) { - logger.catching(e); - logger.info(getMarker(), e.getMessage()); - throw e; - } finally { - setProgress(100); - } - return new Object[] { archiveItem }; - } -} diff --git a/server/-product/production/LOCAL/jobs/steps/CreateMissingLowresStep.java b/server/-product/production/LOCAL/jobs/steps/CreateMissingLowresStep.java deleted file mode 100644 index 19bad479..00000000 --- a/server/-product/production/LOCAL/jobs/steps/CreateMissingLowresStep.java +++ /dev/null @@ -1,84 +0,0 @@ -package user.jobengine.server.steps; - -import java.nio.file.Paths; - -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; - -import com.ibm.nosql.json.api.BasicDBObject; -import com.ibm.nosql.json.api.DB; -import com.ibm.nosql.json.api.DBCollection; - -import user.commons.MediaCubeUndoMarker; -import user.commons.nosql.NoSQLUtils; -import user.jobengine.db.IItemManager; -import user.jobengine.db.Media; -import user.jobengine.server.IJobEngine; -import user.jobengine.server.IJobRuntime; - -public class CreateMissingLowresStep extends JobStep { - private static final Logger logger = LogManager.getLogger(); - - @StepEntry - public Object[] execute(String localHiresPath, IJobEngine jobEngine, IJobRuntime jobRuntime) throws Exception { - Object[] result = new Object[] { null, null, "%s", null, 0, true }; - DB db = NoSQLUtils.getNoSQLDB(); - DBCollection collection = db.getCollection("missing_lowres"); - IItemManager manager = jobEngine.getItemManager(); - Media media = getFirstUntranscodedMedia(manager, collection); - - try { - if (media == null) { - logger.info(new MediaCubeUndoMarker(getSessionMarker().getSessionID()), "Nincs feldolgozandó hiány."); - // throw new Exception("Nincs feldolgozandó hiány."); - cancel(); - return null; - } - - String name = media.getMediaFileRealName(); - result[0] = media; - ArchiveItem archiveItem = new ArchiveItem(); - archiveItem.setMediaFile(Paths.get(localHiresPath, name).toString()); - result[1] = archiveItem; - collection.save(new BasicDBObject("name", name)); - logger.info(getSessionMarker(), "Processing mediaID: {}", media.getId()); - - } catch (Exception e) { - logger.catching(e); - logger.error(getSessionMarker(), e.getMessage()); - throw e; - } finally { - setProgress(100); - } - return result; - } - - private Media getFirstUntranscodedMedia(IItemManager manager, DBCollection collection) { - Media[] result = new Media[] { null }; - //MV - String query = "SELECT mediaid FROM VW_MISSING_PROXY_IDS WHERE HOUSEID like 'M%' or HOUSEID like 'P%' or HOUSEID like 'R%' ORDER BY modified DESC"; - - //HTV - //String query = "SELECT mediaid FROM VW_MISSING_PROXY_IDS"; - manager.executeQuery(query, rs -> { - try { - long mediaId = rs.getLong(1); - Media media = manager.getMedia(mediaId); - //a nevgeneralas miatt az eredeti MediaFilesName nem jo, a pontos nev kell nekunk - String name = media.getMediaFileRealName(); - logger.info(getSessionMarker(), "Checking {}", name); - long existing = collection.find(new BasicDBObject("name", name)).count(); - if (existing > 0) { - logger.info(getSessionMarker(), "{} is on missing_lowres list", name); - return true; - } - - result[0] = media; - } catch (Exception e) { - logger.error(e); - } - return false; - }, null); - return result[0]; - } -} diff --git a/server/-product/production/LOCAL/jobs/steps/ExternalCommand.java b/server/-product/production/LOCAL/jobs/steps/ExternalCommand.java deleted file mode 100644 index 307f36d0..00000000 --- a/server/-product/production/LOCAL/jobs/steps/ExternalCommand.java +++ /dev/null @@ -1,79 +0,0 @@ -package user.jobengine.server.steps.shared; - -import java.io.BufferedReader; -import java.io.InputStreamReader; -import java.util.ArrayList; -import java.util.List; - -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; - -public class ExternalCommand { - private static final Logger logger = LogManager.getLogger(); - private ExternalProfile profile; - - public ExternalCommand(ExternalProfile profile) { - this.profile = profile; - } - - public String execute(String input, String output, boolean firstResponse, IExternalCallback responseCallBack) throws Exception { - List arguments = getArguments(input, output); - List command = new ArrayList<>(); - command.add(profile.getExecutable()); - command.addAll(arguments); - - ProcessBuilder processBuilder = new ProcessBuilder(); - processBuilder.command(command); - - String result = null; - try { - logger.info("Executing : {}", processBuilder.command()); - - Process process = processBuilder.start(); - try (BufferedReader reader = new BufferedReader(new InputStreamReader(process.getInputStream()))) { - String line = null; - while ((line = reader.readLine()) != null) { - logger.debug("Process response: {}", line); - if (responseCallBack != null) - responseCallBack.onResponse(line); - //System.out.println(line); - if (line != null && line.length() > 0) { - result = line; - if (firstResponse) - break; - } - } - int exitCode = process.waitFor(); - if (exitCode != 0) { - StringBuilder msg = new StringBuilder(); - try (BufferedReader errReader = new BufferedReader(new InputStreamReader(process.getErrorStream()))) { - String errline = null; - while ((errline = errReader.readLine()) != null) { - msg.append(errline); - } - } catch (Exception ex) { - } - - throw new Exception("Exited with error code : " + exitCode + ". " + msg); - } - } catch (Exception e) { - throw e; - } - } catch (Exception e) { - logger.error(e); - throw e; - } - - return result; - } - - private List getArguments(String input, String output) { - List result = new ArrayList<>(); - - profile.getArguments().forEach(i -> { - result.add(i.replace("%i", input).replace("%o", output)); - }); - return result; - } - -} \ No newline at end of file diff --git a/server/-product/production/LOCAL/jobs/steps/ExternalCommandExecutor.java b/server/-product/production/LOCAL/jobs/steps/ExternalCommandExecutor.java deleted file mode 100644 index a34ff7f0..00000000 --- a/server/-product/production/LOCAL/jobs/steps/ExternalCommandExecutor.java +++ /dev/null @@ -1,32 +0,0 @@ -package user.jobengine.server.steps.shared; - -import user.commons.configuration.SystemConfiguration; - -public class ExternalCommandExecutor { - - public void execute(String profileName, String input, String output, IExternalCallback responseCallBack) throws Exception { - ExternalCommand externalCommand = getExternalCommand(profileName); - externalCommand.execute(input, output, false, responseCallBack); - } - - private ExternalCommand getExternalCommand(String profileName) throws Exception { - ExternalProfilesConfig config = SystemConfiguration.getInstance().load("settings/external-commands.yaml", ExternalProfilesConfig.class); - - if (config == null) - throw new Exception("Missing external-commands.yaml configuration"); - - ExternalProfile selectedProfile = null; - for (ExternalProfile profile : config.getProfiles()) { - if (profileName.equals(profile.getName())) { - selectedProfile = profile; - break; - } - } - - if (selectedProfile == null) - throw new Exception("Missing profile " + profileName + " in external-commands.yaml configuration"); - - return new ExternalCommand(selectedProfile); - } - -} diff --git a/server/-product/production/LOCAL/jobs/steps/ExternalProfile.java b/server/-product/production/LOCAL/jobs/steps/ExternalProfile.java deleted file mode 100644 index 68e22f4c..00000000 --- a/server/-product/production/LOCAL/jobs/steps/ExternalProfile.java +++ /dev/null @@ -1,33 +0,0 @@ -package user.jobengine.server.steps.shared; - -import java.util.List; - -public class ExternalProfile { - private String executable; - private String name; - private List arguments; - - public List getArguments() { - return arguments; - } - - public String getExecutable() { - return executable; - } - - public String getName() { - return name; - } - - public void setArguments(List arguments) { - this.arguments = arguments; - } - - public void setExecutable(String executable) { - this.executable = executable; - } - - public void setName(String name) { - this.name = name; - } -} \ No newline at end of file diff --git a/server/-product/production/LOCAL/jobs/steps/ExternalProfilesConfig.java b/server/-product/production/LOCAL/jobs/steps/ExternalProfilesConfig.java deleted file mode 100644 index 7ef77291..00000000 --- a/server/-product/production/LOCAL/jobs/steps/ExternalProfilesConfig.java +++ /dev/null @@ -1,15 +0,0 @@ -package user.jobengine.server.steps.shared; - -import java.util.List; - -public class ExternalProfilesConfig { - private List profiles; - - public List getProfiles() { - return profiles; - } - - public void setProfiles(List profiles) { - this.profiles = profiles; - } -} \ No newline at end of file diff --git a/server/-product/production/LOCAL/jobs/steps/FileCopyStep.java b/server/-product/production/LOCAL/jobs/steps/FileCopyStep.java deleted file mode 100644 index c4172550..00000000 --- a/server/-product/production/LOCAL/jobs/steps/FileCopyStep.java +++ /dev/null @@ -1,95 +0,0 @@ -package user.jobengine.server.steps; - -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; -import org.apache.logging.log4j.Marker; - -import user.commons.StoreUri; -import user.commons.remotestore.IProgressEventListener; -import user.commons.remotestore.ProgressEvent; -import user.commons.remotestore.RemoteStoreProtocol; -import user.jobengine.db.IItemManager; -import user.jobengine.server.IJobEngine; -import user.jobengine.server.IJobRuntime; - -public class FileCopyStep extends JobStep { - private static final Logger logger = LogManager.getLogger(); - private IItemManager manager; - private Marker marker; - - private void check(String sourceProtocol, String sourcePath, String sourceFileName, String targetProtocol, String targetPath, String targetFileName, - IJobEngine jobEngine, IJobRuntime jobRuntime) { - if (jobEngine == null) { - logger.error(marker, "A folyamatkezelő réteg nem elérhető."); - throw new NullPointerException("Internal error, missing JobEngine reference."); - } - manager = jobEngine.getItemManager(); - if (manager == null) { - logger.error(marker, "Az adatbáziskezelő réteg nem elérhető."); - throw new NullPointerException("Internal error, missing ItemManager reference."); - } - if (sourceProtocol == null) { - logger.error(marker, "A forrás protokol bemeneti paramétere üres."); - throw new NullPointerException("System is not configured properly, 'sourceProtocol' input parameter missing."); - } - if (sourcePath == null) { - logger.error(marker, "A forrás fájl elérés bemeneti paramétere üres."); - throw new NullPointerException("System is not configured properly, 'sourcePath' input parameter missing."); - } - if (sourceFileName == null) { - logger.error(marker, "A forrás fájlnév bemeneti paramétere üres."); - throw new NullPointerException("System is not configured properly, 'sourceFileName' input parameter missing."); - } - if (targetProtocol == null) { - logger.error(marker, "A cél protokol bemeneti paramétere üres."); - throw new NullPointerException("System is not configured properly, 'targetProtocol' input parameter missing."); - } - if (targetPath == null) { - logger.error(marker, "A cél fájl elérés bemeneti paramétere üres."); - throw new NullPointerException("System is not configured properly, 'targetPath' input parameter missing."); - } - if (targetFileName == null) { - logger.error(marker, "A cél fájlnév bemeneti paramétere üres."); - throw new NullPointerException("System is not configured properly, 'targetFileName' input parameter missing."); - } - } - - @StepEntry - public Object[] execute(String sourceProtocol, String sourcePath, String sourceFileName, String targetProtocol, String targetPath, String targetFileName, - int killDateDays, IJobEngine jobEngine, IJobRuntime jobRuntime) throws Exception { - marker = jobRuntime.getSessionMarker(); - - check(sourceProtocol, sourcePath, sourceFileName, targetProtocol, targetPath, targetFileName, jobEngine, jobRuntime); - - StoreUri source = null; - StoreUri target = null; - - try { - source = manager.createStoreUri(Enum.valueOf(RemoteStoreProtocol.class, sourceProtocol), sourcePath); - target = manager.createStoreUri(Enum.valueOf(RemoteStoreProtocol.class, targetProtocol), targetPath); - source.addProgressListener(new IProgressEventListener() { - @Override - public void progressChanged(ProgressEvent evt) { - jobRuntime.incrementProgress(evt.getProgress()); - } - }); - - source.transferFrom(target, sourceFileName, targetFileName); - if (killDateDays > -1) - EscortFiles.createUNCKillDate(targetPath, targetFileName, killDateDays, marker); - - } catch (Exception e) { - logger.catching(e); - throw e; - } finally { - if (source != null) { - source.cleanUp(); - } - if (target != null) { - target.cleanUp(); - } - } - - return null; - } -} diff --git a/server/-product/production/LOCAL/jobs/steps/FileSearchFilterOptions.java b/server/-product/production/LOCAL/jobs/steps/FileSearchFilterOptions.java deleted file mode 100644 index 436c3be0..00000000 --- a/server/-product/production/LOCAL/jobs/steps/FileSearchFilterOptions.java +++ /dev/null @@ -1,41 +0,0 @@ -package user.jobengine.server.steps; - -import java.nio.file.Path; -import java.util.regex.Matcher; -import java.util.regex.Pattern; - -import com.ibm.nosql.json.api.BasicDBObject; - -public class FileSearchFilterOptions { - - private BasicDBObject filter; - - public FileSearchFilterOptions(BasicDBObject filter) { - this.filter = filter; - } - - public boolean acceptFile(Path file) { - if (filter == null) - return true; - - if (filter.containsKey("fileName")) { - //.*\.(sh|ini|conf|vhost|xml|php)$ - String fileNamePattern = filter.getString("fileName"); - if (fileNamePattern == null || fileNamePattern.trim().length() == 0) - return true; - - Pattern pattern = Pattern.compile(fileNamePattern, Pattern.CASE_INSENSITIVE); - - Matcher matcher = pattern.matcher(file.getFileName().toString()); - if (matcher.find()) - return true; - - } - - return false; - } - - public boolean preAcceptDirectory(Path file) { - return true; - } -} diff --git a/server/-product/production/LOCAL/jobs/steps/HSMMigrateStep.java b/server/-product/production/LOCAL/jobs/steps/HSMMigrateStep.java deleted file mode 100644 index 6b85f1d7..00000000 --- a/server/-product/production/LOCAL/jobs/steps/HSMMigrateStep.java +++ /dev/null @@ -1,401 +0,0 @@ -package user.jobengine.server.steps; - -import java.io.BufferedInputStream; -import java.io.BufferedOutputStream; -import java.io.File; -import java.io.FileInputStream; -import java.io.FileOutputStream; -import java.io.IOException; -import java.io.InputStream; -import java.io.OutputStream; -import java.nio.file.Files; -import java.nio.file.Path; -import java.nio.file.Paths; -import java.nio.file.attribute.BasicFileAttributes; -import java.text.SimpleDateFormat; -import java.util.Arrays; -import java.util.LinkedHashMap; -import java.util.List; -import java.util.Map; - -import org.apache.commons.io.FilenameUtils; -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; -import org.apache.logging.log4j.Marker; - -import com.ibm.nosql.json.api.BasicDBObject; -import com.ibm.nosql.json.api.DB; -import com.ibm.nosql.json.api.DBCollection; -import com.ibm.nosql.json.api.QueryBuilder; - -import user.commons.IEntityBase; -import user.commons.MediaCubeMarker; -import user.commons.nosql.NoSQLUtils; -import user.jobengine.db.IItemManager; -import user.jobengine.db.MediaFile; -import user.jobengine.db.MediaFileDAO; -import user.mediacube.metadata.interfaces.IMetadata; -import user.mediacube.metadata.interfaces.IMetadataProvider; -import user.mediacube.metadata.interfaces.IMetadataProviderFactory; -import user.mediacube.metadata.interfaces.MetadataProviderType; -import user.mediacube.metadata.interfaces.MetadataType; -import user.mediacube.metadata.interfaces.PlanAirMetadataListOptions; - -public class HSMMigrateStep extends JobStep { - private static final Logger logger = LogManager.getLogger(); - private static final String MXFEXT = ".mxf"; - private static final String MOVEXT = ".mov"; - private Marker marker = null; - private IMetadataProvider hsmProvider; - private Map tapeContents = new LinkedHashMap<>(); - private DBCollection excludes; - private DBCollection fileHistory; - private DBCollection volumeHistory; - private DB db; - private IMetadataProvider planairProvider; - private SimpleDateFormat df = new SimpleDateFormat("yyyy.MM.dd"); - - private void cleanupHistory() { - if (db == null) - db = NoSQLUtils.getNoSQLDB(); - if (fileHistory == null) - fileHistory = db.getCollection("hsm_migrate_file_history"); - if (volumeHistory == null) - volumeHistory = db.getCollection("hsm_migrate_volume_history"); - fileHistory.drop(); - volumeHistory.drop(); - } - - private BasicDBObject createMetadata(String volumeName, String fileName) throws Exception { - - Path filePath = Paths.get(fileName); - String mediaHouseId = FilenameUtils.removeExtension(filePath.getFileName().toString()); - BasicDBObject result = null; - try { - result = getPlanAirMetadata(mediaHouseId); - } catch (Exception e) { - - logger.error("PlanAir metadata error", e); - //nem latja a drivert pl. - //throw e; - } - - if (result == null) { - result = new BasicDBObject(); - BasicFileAttributes attr = Files.readAttributes(filePath, BasicFileAttributes.class); - result.put("itemHouseId", df.format(attr.lastModifiedTime().toMillis())); - result.put("itemTitle", filePath.getParent().toString()); - result.put("mediaHouseId", mediaHouseId); - result.put("mediaTitle", fileName); - result.put("mediaDescription", volumeName); - result.put("mediaType", "Generic"); - } - result.put("userName", "mediacube"); - return result; - } - - @SuppressWarnings("serial") - @StepEntry - public Object[] execute(String sourceLocation, String targetLocation) throws Exception { - marker = getJobRuntime().getSessionMarker(); - //remove from prod - //cleanupHistory(); - hsmProvider = getMetadataProvider(MetadataProviderType.HSM); - if (hsmProvider == null) - throw new NullPointerException("No HSM metadata provider available"); - planairProvider = getMetadataProvider(MetadataProviderType.PLANAIR); - if (planairProvider == null) - throw new NullPointerException("No PLANAIR metadata provider available"); - Path targetPath = Paths.get(targetLocation); - try { - List volumes = hsmProvider.list(new BasicDBObject()); - for (IMetadata volume : volumes) { - String volumeName = volume.getTitle(); - - BasicDBObject historyResult = queryVolumeHistory(volumeName); - if (historyResult != null) { - logger.info(marker, "A kazetta már feldolgozásra került: {}", volumeName); - continue; - } - logger.info(marker, "A kazetta feldolgozása elindul: {}", volumeName); - List contents = getContents(volumeName); - int p = 0; - boolean oneSuccess = false; - boolean hasError = false; - for (IMetadata content : contents) { - BasicDBObject c = content.asJSON(); - String hsmFileName = c.getString("fileName"); - long contentFileSize = NoSQLUtils.asLong(c, "fileSize"); - if (!tapeContents.containsKey(hsmFileName)) { - tapeContents.put(hsmFileName, c); - Path sourceFilePath = Paths.get(sourceLocation, hsmFileName); - if (contentFileSize < Files.getFileStore(targetPath).getUsableSpace()) { - Path targetFilePath = Paths.get(targetLocation, sourceFilePath.getFileName().toString()); - - try { - if (processHSMFile(volumeName, hsmFileName, sourceFilePath, targetFilePath)) { - oneSuccess = true; - } - } catch (Exception e) { - hasError = true; - } - } - } - setProgress(p++ * 100 / contents.size()); - - if (getJobRuntime().isWaitingCancel()) { - logger.info("Job canceled by user"); - //ne mentsuk a szalagot meg - return null; - } - - } - - if (oneSuccess && !hasError) { - saveVolumeHistory(volumeName); - String subject = "A kazetta eltávolítható a HSM rendszerből: " + volumeName; - logger.info(new MediaCubeMarker() { - { - setSubject(subject); - } - }, subject); - } - - //ha mar sikerult valamit archivalni kilepunk - if (oneSuccess) - break; - } - } catch (Exception e) { - logger.error(marker, "Hiba a migráció során. A rendszer hibaüzenete: {}", e.getMessage()); - throw e; - } - - return null; - } - - private List getContents(String volumeName) throws Exception { - List contents = null; - contents = hsmProvider.list(new BasicDBObject("volumeName", volumeName)); - return contents; - } - - protected IMetadataProvider getMetadataProvider(MetadataProviderType type) { - IMetadataProviderFactory factory = getService(IMetadataProviderFactory.class); - if (factory == null) - return null; - return factory.getProvider(type); - } - - private BasicDBObject getPlanAirMetadata(String mediaHouseId) throws Exception { - PlanAirMetadataListOptions opt = new PlanAirMetadataListOptions(); - opt.setSearch(mediaHouseId); - - List result = null; - opt.setType(MetadataType.Material); - result = planairProvider.list(opt); - if (result.size() != 0) - return result.get(0).asJSON(); - - opt.setType(MetadataType.Promo); - result = planairProvider.list(opt); - if (result.size() != 0) - return result.get(0).asJSON(); - - opt.setType(MetadataType.AD); - result = planairProvider.list(opt); - if (result.size() != 0) - return result.get(0).asJSON(); - - return null; - } - - //true if need copy - public boolean prepareCopy(String hsmFileName, Path source, Path target) throws IOException { - boolean result = true; - BasicDBObject excludeResult = queryExclude(hsmFileName); - if (excludeResult != null) { - logger.warn(marker, "Kivételként megjelölt: {}", hsmFileName); - return false; - } - - File sourceFile = source.toFile(); - File targetFile = target.toFile(); - - if (!sourceFile.exists()) { - logger.warn(marker, "A forrás nem elérhető: {}", source); - return false; - } - - // BasicDBObject historyResult = queryFileHistory(contentFileName); - // if (historyResult != null) - // return false; - - // if (!mediaFile.getName().toLowerCase().endsWith(MOVEXT.toLowerCase()) && !mediaFile.getName().toLowerCase().endsWith(MXFEXT.toLowerCase())) - // return; - // logger.info("Start copy from {} to {}", sourceFilePath, targetFilePath); - - boolean targetExists = targetFile.exists(); - - long targetLength = targetFile.length(); - long sourceLength = sourceFile.length(); - - if (targetLength == sourceLength) { - logger.warn(marker, "A fájl már fel van dolgozva: {}, {} -> {}", source, sourceLength, targetLength); - return false; - } - - if (targetLength > sourceLength) { - logger.warn(marker, "A célfájl nagyobb, törlöm: {}", target); - Files.delete(target); - targetLength = 0; - targetExists = false; - } - - if (targetExists) { - logger.warn(marker, "A fájl már létezik, a másolás folytatódik: {}, {} -> {}", target, sourceLength, targetLength); - } else - logger.warn(marker, "Migrálás: {}, {} -> {}", source, sourceLength, targetLength); - - return result; - } - - private boolean processHSMFile(String volumeName, String hsmFileName, Path sourceFilePath, Path targetFilePath) throws Exception { - int repeat = 4; - boolean successCopy = false; - - IItemManager manager = getManager(); - MediaFileDAO mfDAO = (MediaFileDAO) manager.getBaseDAO(MediaFile.class); - List mediaFiles = mfDAO.getByHouseId(sourceFilePath.getFileName().toString()); - if (mediaFiles != null && mediaFiles.size() > 0) { - logger.warn(marker, "Már archivált: {}", hsmFileName); - return false; - } - - if (prepareCopy(hsmFileName, sourceFilePath, targetFilePath)) { - while (repeat > 0) { - try { - resumeableCopy(sourceFilePath, targetFilePath); - repeat = 0; - successCopy = true; - } catch (Exception e) { - if (Files.exists(targetFilePath) && targetFilePath.toFile().length() == 0) - Files.delete(targetFilePath); - //logger.warn(marker, "Hiba a másolás során: {} ({})", sourceFilePath, e.getMessage()); - repeat--; - } - } - } - String metadataFileName = sourceFilePath.getFileName() + EscortFiles.DOT_JSON; - Path metadataPath = Paths.get(targetFilePath.getParent().toString(), EscortFiles.STATUSFOLDER, metadataFileName); - boolean createMetadata = Files.exists(targetFilePath) && !Files.exists(metadataPath); - - if (successCopy || createMetadata) { - String metadata = null; - try { - metadata = createMetadata(volumeName, hsmFileName).toPrettyString(""); - EscortFiles.createMetadata(targetFilePath.getParent().toString(), targetFilePath.getFileName().toString(), metadata); - //saveFileHistory(contentFileName); - - } catch (Exception e) { - logger.error(marker, "Metadata error", e); - return false; - } - return true; - } - - logger.error(marker, "A fájl másolása nem lehetséges: {}", sourceFilePath); - return false; - } - - public BasicDBObject queryExclude(String fileName) { - if (db == null) - db = NoSQLUtils.getNoSQLDB(); - if (excludes == null) - excludes = db.getCollection("hsm_migrate_exclude"); - Path filePath = Paths.get(fileName); - String pureFileName = FilenameUtils.removeExtension(filePath.getFileName().toString()); - QueryBuilder qb = QueryBuilder.start("name").in(Arrays.asList(fileName, pureFileName)); - BasicDBObject exceptionResult = NoSQLUtils.asSingle(excludes.find(qb.get())); - return exceptionResult; - } - - public BasicDBObject queryFileHistory(String fileName) { - if (db == null) - db = NoSQLUtils.getNoSQLDB(); - if (fileHistory == null) - fileHistory = db.getCollection("hsm_migrate_file_history"); - QueryBuilder qb = QueryBuilder.start("name").in(Arrays.asList(fileName)); - BasicDBObject historyResult = NoSQLUtils.asSingle(fileHistory.find(qb.get())); - return historyResult; - } - - public BasicDBObject queryVolumeHistory(String volumeName) { - if (db == null) - db = NoSQLUtils.getNoSQLDB(); - if (volumeHistory == null) - volumeHistory = db.getCollection("hsm_migrate_volume_history"); - QueryBuilder qb = QueryBuilder.start("name").in(Arrays.asList(volumeName)); - BasicDBObject historyResult = NoSQLUtils.asSingle(volumeHistory.find(qb.get())); - return historyResult; - } - - public void resumeableCopy(Path source, Path target) throws Exception { - File sourceFile = source.toFile(); - File targetFile = target.toFile(); - boolean targetExists = targetFile.exists(); - - long targetLength = targetFile.length(); - long sourceLength = sourceFile.length(); - - try (InputStream in = new BufferedInputStream(new FileInputStream(sourceFile)); - OutputStream out = new BufferedOutputStream(new FileOutputStream(targetFile, targetExists))) { - - byte[] buffer = new byte[128 * 1024]; - int lengthRead; - - if (targetExists) - in.skip(targetLength); - - while ((lengthRead = in.read(buffer)) > 0) { - out.write(buffer, 0, lengthRead); - out.flush(); - targetLength = targetFile.length(); - if (targetLength > sourceLength) { - throw new Exception("Hiba! A fájl túl nagy lett."); - } - - if (getJobRuntime().isWaitingCancel()) { - break; - } - } - - targetLength = targetFile.length(); - sourceLength = sourceFile.length(); - if (targetLength != sourceLength) { - throw new Exception("Hiba! A fájl mérete nem egyezik."); - } - } - } - - private void saveFileHistory(String fileName) { - if (db == null) - db = NoSQLUtils.getNoSQLDB(); - if (fileHistory == null) - fileHistory = db.getCollection("hsm_migrate_file_history"); - BasicDBObject item = new BasicDBObject(); - item.put("name", fileName); - fileHistory.save(item); - - } - - private void saveVolumeHistory(String volumeName) { - if (db == null) - db = NoSQLUtils.getNoSQLDB(); - if (volumeHistory == null) - volumeHistory = db.getCollection("hsm_migrate_volume_history"); - BasicDBObject item = new BasicDBObject(); - item.put("name", volumeName); - volumeHistory.save(item); - - } -} diff --git a/server/-product/production/LOCAL/jobs/steps/IExternalCallback.java b/server/-product/production/LOCAL/jobs/steps/IExternalCallback.java deleted file mode 100644 index 1e071913..00000000 --- a/server/-product/production/LOCAL/jobs/steps/IExternalCallback.java +++ /dev/null @@ -1,5 +0,0 @@ -package user.jobengine.server.steps.shared; - -public interface IExternalCallback { - void onResponse(String data); -} \ No newline at end of file diff --git a/server/-product/production/LOCAL/jobs/steps/MXFCutterStep.java b/server/-product/production/LOCAL/jobs/steps/MXFCutterStep.java deleted file mode 100644 index 58a5944d..00000000 --- a/server/-product/production/LOCAL/jobs/steps/MXFCutterStep.java +++ /dev/null @@ -1,129 +0,0 @@ -package user.jobengine.server.steps; - -import org.apache.commons.lang.StringUtils; -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; -import org.apache.logging.log4j.Marker; - -import user.commons.RemoteFile; -import user.commons.StoreUri; -import user.commons.remotestore.IProgressEventListener; -import user.commons.remotestore.ProgressEvent; -import user.commons.remotestore.RemoteStoreProtocol; -//import user.jobengine.db.Media; -import user.jobengine.db.ArchivedMedia; -import user.jobengine.db.IItemManager; -import user.jobengine.db.Store; -import user.jobengine.server.IJobEngine; -import user.jobengine.server.IJobRuntime; - -public class MXFCutterStep extends JobStep { - private static final String TARGETNAMEPATTERN = "-ARCH-%s"; - private static final Logger logger = LogManager.getLogger(); - private IItemManager manager; - private StoreUri tempTargetUri; - private StoreUri tempSourceUri; - private String sourceFileName; - private Marker marker; - private int nexioPort; - private String nexioUserName, nexioPassword; - private String nexioHost; - - protected void checkTargetPath(String targetPath) { - if (StringUtils.isBlank(targetPath)) { - logger.error(marker, "A folyamat 'targetPath' bemeneti paramétere üres."); - throw new NullPointerException("System is not configured properly, 'targetPath' input parameter missing."); - } - } - - protected StoreUri createTargetUri(IItemManager manager, String targetPath) throws NullPointerException { - StoreUri result = null; - result = manager.createStoreUri(RemoteStoreProtocol.FTP, nexioHost); - result.setPortNumber(nexioPort); - result.setUserName(nexioUserName); - result.setPassword(nexioPassword); - - return result; - } - - @StepEntry - public Object[] execute(ArchivedMedia archivedMedia, String targetPath, String houseId, String successRecipient, int killDateDays, boolean useNexioTarget, - String nexioAgency, int nexioPort, String nexioUserName, String nexioPassword, IJobEngine jobEngine, IJobRuntime jobRuntime) throws Exception { - this.nexioPort = nexioPort; - this.nexioUserName = nexioUserName; - this.nexioPassword = nexioPassword; - nexioHost = System.getProperty("nexio.host"); - marker = jobRuntime.getSessionMarker(); - - if (useNexioTarget && archivedMedia.getTcIn() != null && archivedMedia.getTcOut() != null) { - setAndCheck(archivedMedia, houseId, targetPath, useNexioTarget, jobEngine); - - final IJobRuntime runtime = jobRuntime; - sourceFileName = houseId + TARGETNAMEPATTERN; - tempSourceUri.addProgressListener(new IProgressEventListener() { - @Override - public void progressChanged(ProgressEvent evt) { - runtime.incrementProgress(evt.getProgress()); - } - }); - - RemoteFile result = tempSourceUri.transferFrom(tempTargetUri, sourceFileName, sourceFileName); - - EscortFiles.setNEXIOKillDate(killDateDays, houseId, nexioAgency, tempTargetUri); - - logger.info("A {} videó kivágva {}s - {}s", sourceFileName, archivedMedia.getTcIn(), archivedMedia.getTcOut()); - } - - return null; - } - - // private String getSourceFileName(ArchivedMedia archivedMedia, Store - // store) { - // List mediaFiles = archivedMedia.getMedia().getMediaFiles(); - // if (mediaFiles == null) - // return null; - // for (MediaFile mediaFile : mediaFiles) { - // if (mediaFile.getStore().getId() == store.getId()) - // return mediaFile.getRelativePath(); - // } - // return null; - // } - - private void setAndCheck(ArchivedMedia archivedMedia, String houseId, String targetPath, boolean useNexioTarget, IJobEngine jobEngine) { - if (jobEngine == null) { - logger.error(marker, "A folyamatkezelő réteg nem elérhető."); - throw new NullPointerException("Internal error, missing JobEngine reference."); - } - manager = jobEngine.getItemManager(); - if (manager == null) { - logger.error(marker, "Az adatbáziskezelő réteg nem elérhető."); - throw new NullPointerException("Internal error, missing ItemManager reference."); - } - if (archivedMedia == null) { - logger.error(marker, "A folyamat 'mediaCubeMedia' bemeneti paramétere üres."); - throw new NullPointerException("System is not configured properly, 'mediaCubeMedia' input parameter missing."); - } - checkTargetPath(targetPath); - if (StringUtils.isBlank(houseId)) { - logger.error(marker, "A folyamat 'houseId' bemeneti paramétere üres."); - throw new NullPointerException("System is not configured properly, 'houseId' input parameter missing."); - } - Store tsmStore = manager.getSystemStore(false); - if (tsmStore == null) { - logger.error(marker, "A TSM rendszer beállítás nem elérhető."); - throw new NullPointerException("System is not configured properly, missing TSM Store."); - } - - tempSourceUri = manager.createStoreUri(RemoteStoreProtocol.LOCAL, targetPath); - if (tempSourceUri == null) { - logger.error(marker, "A TSM rendszer beállítás paraméterei nem elérhetőek."); - throw new NullPointerException("System is not configured properly, missing TSM StoreUri."); - } - tempTargetUri = createTargetUri(manager, targetPath); - // sourceFileName = getSourceFileName(archivedMedia, tsmStore); - if (sourceFileName == null) { - logger.error(marker, "Adatbázis bejegyzés hiba, a visszatöltendő fájl neve nem található."); - throw new NullPointerException("Database error, missing MediaFile 'relativePath'."); - } - } -} diff --git a/server/-product/production/LOCAL/jobs/steps/MediaCubeClient.java b/server/-product/production/LOCAL/jobs/steps/MediaCubeClient.java deleted file mode 100644 index 840925e4..00000000 --- a/server/-product/production/LOCAL/jobs/steps/MediaCubeClient.java +++ /dev/null @@ -1,77 +0,0 @@ -package user.jobengine.server.steps; - -import javax.ws.rs.client.Entity; -import javax.ws.rs.client.Invocation.Builder; -import javax.ws.rs.core.MediaType; -import javax.ws.rs.core.MultivaluedMap; -import javax.ws.rs.core.Response; -import javax.ws.rs.core.Response.Status; - -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; -import org.jboss.resteasy.client.jaxrs.ResteasyClient; -import org.jboss.resteasy.client.jaxrs.ResteasyClientBuilder; -import org.jboss.resteasy.client.jaxrs.ResteasyWebTarget; -import org.jboss.resteasy.specimpl.MultivaluedMapImpl; - -import com.ibm.nosql.json.JSONUtil; -import com.ibm.nosql.json.api.BasicDBObject; - -public class MediaCubeClient { - private static Logger logger = LogManager.getLogger(); - private ResteasyWebTarget webTarget; - - public MediaCubeClient(String address) { - ResteasyClient client = new ResteasyClientBuilder().build(); - webTarget = client.target(address); - } - - BasicDBObject getDbObject(String json) { - BasicDBObject result = (BasicDBObject) JSONUtil.jsonToDbObject(json); - - if (result == null) - throw new NullPointerException("API Result is null!"); - - if (result.containsKey("exception")) { - BasicDBObject e = (BasicDBObject) result.get("exception"); - throw new RuntimeException(e.getString("message")); - } - //{"exception":{"message":"Invalid credentials.","publicName":"AuthenticationFailedException"}} - return result; - } - - public BasicDBObject getStatus(long jobId) { - MultivaluedMap vars = new MultivaluedMapImpl<>(); - vars.add("jobId", jobId); - Response response = query("services/rest/jobengine/jobstatus", vars).get(); - if (response.getStatus() != Status.OK.getStatusCode()) { - logger.error(response.readEntity(String.class)); - System.out.println(response.readEntity(String.class)); - return null; - } - String result = response.readEntity(String.class); - return getDbObject(result); - } - - private Builder query(String path, MultivaluedMap vars) { - ResteasyWebTarget target = webTarget.path(path).queryParams(vars); - Builder result = target.request(); - return result; - } - - public long startjob(String template, String name, BasicDBObject jobParams) throws Exception { - MultivaluedMap vars = new MultivaluedMapImpl<>(); - vars.add("template", template); - vars.add("name", name); - Response response = query("services/rest/jobengine/startjob", vars).post(Entity.entity(jobParams.toString(), MediaType.APPLICATION_JSON)); - - if (response.getStatus() != Status.OK.getStatusCode()) { - logger.error(response.readEntity(String.class)); - return 0; - } - - String resultObject = response.readEntity(String.class); - return Long.parseLong(resultObject); - } - -} diff --git a/server/-product/production/LOCAL/jobs/steps/MediaToolStep.java b/server/-product/production/LOCAL/jobs/steps/MediaToolStep.java deleted file mode 100644 index c0c56f68..00000000 --- a/server/-product/production/LOCAL/jobs/steps/MediaToolStep.java +++ /dev/null @@ -1,41 +0,0 @@ -package user.jobengine.server.steps; - -import java.nio.file.Path; -import java.nio.file.Paths; - -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; -import org.apache.logging.log4j.Marker; - -import user.commons.mediatool.MediaInfo; -import user.jobengine.db.Media; - -public class MediaToolStep extends JobStep { - private static final Logger logger = LogManager.getLogger(); - private Marker marker; - - @StepEntry - public Object[] execute(ArchiveItem archiveItem, Media mediaCubeMedia) throws Exception { - marker = getSessionMarker(); - - //TODO ez most minden esetben ujranezi - // if (mediaCubeMedia.getLength() > 0) - // return null; - - Path filePath = Paths.get(archiveItem.getMediaFile()); - MediaInfo mi = new MediaInfo(filePath); - try { - mi.process(); - long frames = mi.getFrames(); - if (frames > 0) { - mediaCubeMedia.setLength(frames); - getManager().modify(mediaCubeMedia); - } - } catch (Exception e) { - logger.catching(e); - logger.error(marker, "A '{}' média analizálása sikertelen. A rendszer üzenete: {}", filePath, e.getMessage()); - } - return null; - } - -} diff --git a/server/-product/production/LOCAL/jobs/steps/MetadataTransformStep.java b/server/-product/production/LOCAL/jobs/steps/MetadataTransformStep.java deleted file mode 100644 index d1938203..00000000 --- a/server/-product/production/LOCAL/jobs/steps/MetadataTransformStep.java +++ /dev/null @@ -1,168 +0,0 @@ -package user.jobengine.server.steps; - -import java.io.File; -import java.nio.file.Files; -import java.nio.file.Path; -import java.nio.file.Paths; -import java.nio.file.attribute.FileAttribute; -import java.nio.file.attribute.PosixFilePermission; -import java.nio.file.attribute.PosixFilePermissions; -import java.util.Date; -import java.util.Set; - -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; -import org.apache.logging.log4j.Marker; - -import com.ibm.nosql.json.api.BasicDBList; - -import user.jobengine.db.Item; -import user.jobengine.db.ItemManager; -import user.jobengine.db.ItemType; -import user.jobengine.db.Media; -import user.jobengine.server.IJobEngine; -import user.jobengine.server.IJobRuntime; - -/** - * Itemek es mediak krealasa a ArchiveItem objektum alapjan. - * - * @author robi - */ -public class MetadataTransformStep extends JobStep { - private static final String CONFLICT = ".CONFLICT"; - private static final Logger logger = LogManager.getLogger(); - private static final String ITEM_MANAGER_IS_NULL = "ItemManager is null"; - public static final String DEFAULT_MEDIATYPE = "Generic"; - private ItemManager itemManager; - - private Marker marker;; - - private void addTags(ArchiveItem archiveItem, Media mediaCubeMedia) { - BasicDBList tags = archiveItem.getTags(); - if (tags != null) { - for (Object tag : tags) { - - try { - String tagText = String.valueOf(tag); - itemManager.addMediaTag(tagText, mediaCubeMedia.getId()); - System.out.println(); - - } catch (Exception e) { - logger.catching(e); - } - } - } - } - - private void checkDuplicates(ArchiveItem archiveItem, String sourceFileName) throws Exception { - if (itemManager.isMediaFileExists(sourceFileName)) { - try { - Path sourcePath = Paths.get(archiveItem.getMediaFile()); - Path parent = sourcePath.getParent(); - Path conflictPath = Paths.get(parent.toString(), CONFLICT); - File folder = conflictPath.toFile(); - if (!folder.exists() || !folder.isDirectory()) { - Set perms = PosixFilePermissions.fromString("rwxrwxrwx"); - FileAttribute> attr = PosixFilePermissions.asFileAttribute(perms); - try { - Files.createDirectories(conflictPath, attr); - } catch (Exception e) { - try { - Files.createDirectory(conflictPath); - } catch (Exception e1) { - logger.catching(e); - throw e; - } - } - } - - Files.move(sourcePath, Paths.get(conflictPath.toString(), sourceFileName + (new Date()).getTime())); - } catch (Exception e1) { - logger.catching(e1); - logger.error(marker, "Hiba az '{}' állomány mappába másolásakor. A rendszer üzenete: {}", CONFLICT, e1.getMessage()); - } - throw new Exception("Az '" + sourceFileName + "' állomány már megtalálható az archívumban, archiválása nem lehetséges."); - } - } - - private Item createItem(ArchiveItem archiveItem) { - Item mediaCubeItem = getExistingItem(archiveItem.getItemHouseId(), archiveItem.getItemTitle()); - if (mediaCubeItem == null) - mediaCubeItem = itemManager.createItem(DEFAULT_MEDIATYPE, archiveItem.getItemTitle(), archiveItem.getItemDescription(), - archiveItem.getItemHouseId()); - return mediaCubeItem; - } - - private Media createMedia(ArchiveItem archiveItem, Item mediaCubeItem, String mediaType) { - Media mediaCubeMedia; - mediaCubeMedia = itemManager.createMedia(mediaType, archiveItem.getMediaTitle(), archiveItem.getMediaDescription(), archiveItem.getMediaHouseId()); - mediaCubeMedia.setLength(archiveItem.getDuration()); - mediaCubeItem.appendMedia(mediaCubeMedia); - - return mediaCubeMedia; - } - - @StepEntry - public Object[] execute(ArchiveItem archiveItem, IJobEngine jobEngine, IJobRuntime jobRuntime) throws Exception { - marker = jobRuntime.getSessionMarker(); - Media mediaCubeMedia = null; - itemManager = (ItemManager) jobEngine.getItemManager(); - if (itemManager == null) - throw new NullPointerException(ITEM_MANAGER_IS_NULL); - try { - File sourceMediaFile = new File(archiveItem.getMediaFile()); - String sourceFileName = sourceMediaFile.getName(); - checkDuplicates(archiveItem, sourceFileName); - Item mediaCubeItem = createItem(archiveItem); - jobRuntime.incrementProgress(50); - String mediaType = getCreateType(archiveItem); - mediaCubeMedia = createMedia(archiveItem, mediaCubeItem, mediaType); - //ha itemid 0 akkor merge, egyebkent media insert - - if (mediaCubeItem.getId() == 0) - itemManager.mergeItemStructure(mediaCubeItem); - else { - mediaCubeMedia.setItemId(mediaCubeItem.getId()); - mediaCubeMedia.add(); - } - - addTags(archiveItem, mediaCubeMedia); - - } catch (Exception e) { - logger.catching(e); - String fileName = new File(archiveItem.getMediaFile()).getName(); - logger.error(marker, "Az '{}' állomány nem archiválható, mert a metaadat transzformáció sikertelen. A rendszer üzenete: {}", fileName, - e.getMessage()); - if (!archiveItem.removeCatchedFile()) - logger.error(marker, "Az '{}' állomány .catched jelző állománya nem törölhető.", fileName); - throw e; - } finally { - jobRuntime.incrementProgress(100); - } - return new Object[] { mediaCubeMedia }; - } - - private String getCreateType(ArchiveItem archiveItem) { - String mediaType = archiveItem.getMediaType(); - if (mediaType == null || mediaType.length() == 0) - mediaType = DEFAULT_MEDIATYPE; - else { - ItemType mediaItemType = itemManager.getItemType(mediaType); - if (mediaItemType == null) - itemManager.createItemType(mediaType, mediaType).add(); - } - return mediaType; - } - - private Item getExistingItem(String itemHouseId, String itemTitle) { - Item[] result = new Item[] { null }; - String sql = String.format("select id from item where houseid='%s' and title='%s'", itemHouseId, itemTitle); - itemManager.executeQuery(sql, rs -> { - long id = rs.getLong("id"); - result[0] = itemManager.getItem(id); - return true; - }, null); - return result[0]; - } - -} diff --git a/server/-product/production/LOCAL/jobs/steps/OutputPathAndNameSelectorStep.java b/server/-product/production/LOCAL/jobs/steps/OutputPathAndNameSelectorStep.java deleted file mode 100644 index 7148df4c..00000000 --- a/server/-product/production/LOCAL/jobs/steps/OutputPathAndNameSelectorStep.java +++ /dev/null @@ -1,151 +0,0 @@ -package user.jobengine.server.steps; - -import java.io.IOException; -import java.nio.file.FileVisitResult; -import java.nio.file.FileVisitor; -import java.nio.file.Files; -import java.nio.file.Path; -import java.nio.file.Paths; -import java.nio.file.SimpleFileVisitor; -import java.nio.file.attribute.BasicFileAttributes; - -import org.apache.commons.lang.StringUtils; -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; -import org.apache.logging.log4j.Marker; - -import user.jobengine.db.ArchivedMedia; -import user.jobengine.server.IJobEngine; -import user.jobengine.server.IJobRuntime; - -public class OutputPathAndNameSelectorStep extends JobStep { - - private static final String TARGETNAMEPATTERN = "-ARCH-%s"; - - private static final Logger logger = LogManager.getLogger(); - - private Marker marker; - - private void check(String localRetrievePath, String materialOutputFolder, String promoOutputFolder, String advertisementOutputFolder, - String octopusOutputFolder, String genericOutputFolder, String houseId, String targetPathType) { - if (StringUtils.isBlank(localRetrievePath)) { - logger.error(marker, "A folyamat 'localRetrievePath' bemeneti paramétere üres."); - throw new NullPointerException("System is not configured properly, 'localRetrievePath' input parameter missing."); - } - if (StringUtils.isBlank(materialOutputFolder)) { - logger.error(marker, "A folyamat 'materialOutputFolder' bemeneti paramétere üres."); - throw new NullPointerException("System is not configured properly, 'materialOutputFolder' input parameter missing."); - } - if (StringUtils.isBlank(promoOutputFolder)) { - logger.error(marker, "A folyamat 'promoOutputFolder' bemeneti paramétere üres."); - throw new NullPointerException("System is not configured properly, 'promoOutputFolder' input parameter missing."); - } - if (StringUtils.isBlank(advertisementOutputFolder)) { - logger.error(marker, "A folyamat 'advertisementOutputFolder' bemeneti paramétere üres."); - throw new NullPointerException("System is not configured properly, 'advertisementOutputFolder' input parameter missing."); - } - if (StringUtils.isBlank(octopusOutputFolder)) { - logger.error(marker, "A folyamat 'octopusOutputFolder' bemeneti paramétere üres."); - throw new NullPointerException("System is not configured properly, 'octopusOutputFolder' input parameter missing."); - } - if (StringUtils.isBlank(genericOutputFolder)) { - logger.error(marker, "A folyamat 'genericOutputFolder' bemeneti paramétere üres."); - throw new NullPointerException("System is not configured properly, 'genericOutputFolder' input parameter missing."); - } - if (StringUtils.isBlank(houseId)) { - logger.error(marker, "A folyamat 'houseId' bemeneti paramétere üres."); - throw new NullPointerException("System is not configured properly, 'houseId' input parameter missing."); - } - if (StringUtils.isBlank(targetPathType)) { - logger.error(marker, "A folyamat 'targetPathType' bemeneti paramétere üres."); - throw new NullPointerException("System is not configured properly, 'targetPathType' input parameter missing."); - } - } - - @StepEntry - public Object[] execute(String localRetrievePath, String materialOutputFolder, String promoOutputFolder, String advertisementOutputFolder, - String octopusOutputFolder, String genericOutputFolder, String onlineOutputFolder, String houseId, String targetPathType, - ArchivedMedia archivedMedia, IJobEngine jobEngine, IJobRuntime jobRuntime) throws Exception { - marker = jobRuntime.getSessionMarker(); - check(localRetrievePath, materialOutputFolder, promoOutputFolder, advertisementOutputFolder, octopusOutputFolder, genericOutputFolder, houseId, - targetPathType); - Object[] result = null; - switch (Integer.parseInt(targetPathType)) { - case 0: - String outputFolder = getFolderById(materialOutputFolder, promoOutputFolder, advertisementOutputFolder, octopusOutputFolder, genericOutputFolder, - houseId, archivedMedia); - result = localTargetInit(localRetrievePath, outputFolder, houseId, jobRuntime); - break; - case 1: - result = localTargetInit(localRetrievePath, onlineOutputFolder, houseId, jobRuntime); - break; - case 2: - - if (archivedMedia.getTcIn() != null && archivedMedia.getTcOut() != null) - result = new Object[] { genericOutputFolder, houseId, true }; - else - result = new Object[] { null, houseId, true }; - break; - } - return result; - } - - private String getFolderById(String materialOutputFolder, String promoOutputFolder, String advertisementOutputFolder, String octopusOutputFolder, - String genericOutputFolder, String houseId, ArchivedMedia archivedMedia) throws Exception { - String id = houseId.toUpperCase(); - MetadataType mdType = MetadataTypeDetector.GuessMetadataType(id); - String result = null; - - //a groovy nem latja enumnak, hanem az objektum tulajdonsaganak - switch (mdType.toString()) { - case "OctopusPlaceholder": - case "OctopusStory": - result = octopusOutputFolder; - break; - case "TrafficMaterial": - result = materialOutputFolder; - break; - case "TrafficPromo": - result = promoOutputFolder; - break; - case "TrafficAD": - result = advertisementOutputFolder; - break; - case "Generic": - result = genericOutputFolder; - break; - } - return result; - } - - private String getPossiblePath(String id, Path targetPath) throws IOException { - String[] result = new String[] { targetPath.toString() }; - FileVisitor matcherVisitor = new SimpleFileVisitor() { - @Override - public FileVisitResult preVisitDirectory(Path dir, BasicFileAttributes attrs) throws IOException { - String dirName = dir.getFileName().toString(); - if (dirName.startsWith(id + "-") || dirName.equals(id)) { - result[0] = dir.toString(); - return FileVisitResult.TERMINATE; - } - return FileVisitResult.CONTINUE; - } - }; - Files.walkFileTree(targetPath.getParent(), matcherVisitor); - return result[0]; - } - - private Object[] localTargetInit(String localRetrievePath, String outputFolder, String houseId, IJobRuntime jobRuntime) throws IOException { - String id = houseId.toUpperCase(); - String targetPath = getPossiblePath(id, Paths.get(localRetrievePath, outputFolder, id)).toString(); - String targetNamePattern = houseId + TARGETNAMEPATTERN; - try { - EscortFiles.ensureUNCFolder(Paths.get(targetPath)); - } catch (Exception e) { - logger.error(jobRuntime.getSessionMarker(), "A cél mappa '{}' nem létezik és nem hozható létre. A rendszer hibaüzenete: {}", targetPath, - e.getMessage()); - throw e; - } - return new Object[] { targetPath, targetNamePattern, false }; - } -} diff --git a/server/-product/production/LOCAL/jobs/steps/PathItemsCollectorStep.java b/server/-product/production/LOCAL/jobs/steps/PathItemsCollectorStep.java deleted file mode 100644 index de1ae3a9..00000000 --- a/server/-product/production/LOCAL/jobs/steps/PathItemsCollectorStep.java +++ /dev/null @@ -1,68 +0,0 @@ -package user.jobengine.server.steps; - -import java.io.IOException; -import java.nio.file.FileVisitResult; -import java.nio.file.FileVisitor; -import java.nio.file.Files; -import java.nio.file.Path; -import java.nio.file.Paths; -import java.nio.file.SimpleFileVisitor; -import java.nio.file.attribute.BasicFileAttributes; -import java.util.ArrayList; -import java.util.List; - -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; - -import com.ibm.nosql.json.api.BasicDBObject; - -/* DO NOT REMOVE! -import user.jobengine.server.steps.FileSearchFilterOptions; -*/ - -public class PathItemsCollectorStep extends JobStep { - private static final Logger logger = LogManager.getLogger(); - - @StepEntry - public Object[] execute(String sourceFolder, BasicDBObject filter) throws Exception { - Path sourcePath = Paths.get(sourceFolder); - - List files = new ArrayList<>(); - - //teljes rekurzivitas szuressel - FileSearchFilterOptions filterOptions = new FileSearchFilterOptions(filter); - FileVisitor visitor = new SimpleFileVisitor() { - @Override - public FileVisitResult preVisitDirectory(Path dir, BasicFileAttributes attrs) throws IOException { - if (!filterOptions.preAcceptDirectory(dir)) - return FileVisitResult.SKIP_SUBTREE; - return FileVisitResult.CONTINUE; - } - - @Override - public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) throws IOException { - if (filterOptions.acceptFile(file)) { - files.add(file.toString()); - logger.info(file.getFileName()); - } - return FileVisitResult.CONTINUE; - } - - @Override - public FileVisitResult visitFileFailed(Path file, IOException exc) throws IOException { - return FileVisitResult.CONTINUE; - } - - }; - - try { - Files.walkFileTree(sourcePath, visitor); - } catch (Exception e) { - logger.error(getSessionMarker(), "Hiba a '{}' mappa feldolgozásában. A rendszer hibaüzenete: {}", sourcePath, e.getMessage()); - logger.catching(e); - throw e; - } finally { - } - return new Object[] { files }; - } -} diff --git a/server/-product/production/LOCAL/jobs/steps/PrepareRemoteTranscodeJobParametersStep.java b/server/-product/production/LOCAL/jobs/steps/PrepareRemoteTranscodeJobParametersStep.java deleted file mode 100644 index 37306e60..00000000 --- a/server/-product/production/LOCAL/jobs/steps/PrepareRemoteTranscodeJobParametersStep.java +++ /dev/null @@ -1,20 +0,0 @@ -package user.jobengine.server.steps; - -import java.nio.file.Paths; - -import com.ibm.nosql.json.api.BasicDBObject; - -public class PrepareRemoteTranscodeJobParametersStep extends JobStep { - @StepEntry - public Object[] execute(String profileName, String fileName) throws Exception { - String transcoderInputDir = "m:\"; - String transcoderOutputDir = "m:\"; - String outFileName = fileName.substring(0, fileName.lastIndexOf(".")) + ".mp4"; - BasicDBObject parameters = new BasicDBObject(); - parameters.put("profile", profileName); - parameters.put("input", Paths.get(transcoderInputDir, fileName)); - parameters.put("output", Paths.get(transcoderOutputDir, outFileName)); - return new Object[] { parameters }; - } - -} diff --git a/server/-product/production/LOCAL/jobs/steps/QueryMissingProxyMediaStep.java b/server/-product/production/LOCAL/jobs/steps/QueryMissingProxyMediaStep.java deleted file mode 100644 index 9e8d1949..00000000 --- a/server/-product/production/LOCAL/jobs/steps/QueryMissingProxyMediaStep.java +++ /dev/null @@ -1,76 +0,0 @@ -package user.jobengine.server.steps; - -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; - -import com.ibm.nosql.json.api.BasicDBObject; -import com.ibm.nosql.json.api.DB; -import com.ibm.nosql.json.api.DBCollection; - -import user.commons.MediaCubeUndoMarker; -import user.commons.nosql.NoSQLUtils; -import user.jobengine.db.IItemManager; -import user.jobengine.db.Media; - -public class QueryMissingProxyMediaStep extends JobStep { - private static final Logger logger = LogManager.getLogger(); - - @StepEntry - public Object[] execute() throws Exception { - Object[] result = new Object[] { null }; - DB db = NoSQLUtils.getNoSQLDB(); - DBCollection collection = db.getCollection("missing_lowres"); - Media media = getFirstUntranscodedMedia(collection); - - try { - if (media == null) { - logger.info(new MediaCubeUndoMarker(getSessionMarker().getSessionID()), "Nincs feldolgozandó hiány."); - // throw new Exception("Nincs feldolgozandó hiány."); - cancel(); - return null; - } - - String name = media.getMediaFileRealName(); - collection.save(new BasicDBObject("name", name)); - logger.info(getSessionMarker(), "Processing mediaID: {}", media.getId()); - result[0] = media; - } catch (Exception e) { - logger.catching(e); - logger.error(getSessionMarker(), e.getMessage()); - throw e; - } finally { - setProgress(100); - } - return result; - } - - private Media getFirstUntranscodedMedia(DBCollection collection) { - Media[] result = new Media[] { null }; - //MV - String query = "SELECT mediaid FROM VW_MISSING_PROXY_IDS WHERE HOUSEID like 'M%' or HOUSEID like 'P%' or HOUSEID like 'R%' ORDER BY modified DESC"; - - //HTV - //String query = "SELECT mediaid FROM VW_MISSING_PROXY_IDS"; - IItemManager manager = getManager(); - manager.executeQuery(query, rs -> { - try { - long mediaId = rs.getLong(1); - Media media = manager.getMedia(mediaId); - //a nevgeneralas miatt az eredeti MediaFilesName nem jo, a pontos nev kell nekunk - String name = media.getMediaFileRealName(); - logger.info(getSessionMarker(), "Checking {}", name); - long existing = collection.find(new BasicDBObject("name", name)).count(); - if (existing > 0) { - logger.info(getSessionMarker(), "{} is on missing_lowres list", name); - return true; - } - - result[0] = media; - } catch (Exception e) { - logger.error(e); - } - return false; - }, null); - return result[0]; - } -} diff --git a/server/-product/production/LOCAL/jobs/steps/RemoteJobStep.java b/server/-product/production/LOCAL/jobs/steps/RemoteJobStep.java deleted file mode 100644 index 8e022f43..00000000 --- a/server/-product/production/LOCAL/jobs/steps/RemoteJobStep.java +++ /dev/null @@ -1,43 +0,0 @@ -package user.jobengine.server.steps; - -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; - -import com.ibm.nosql.json.api.BasicDBObject; - -/* -import user.jobengine.server.steps.MediaCubeClient; -*/ -public class RemoteJobStep extends JobStep { - private static final Logger logger = LogManager.getLogger(); - - @StepEntry - public Object[] execute(String server, String template, String name, String profile) throws Exception { - try { - MediaCubeClient mc = new MediaCubeClient(server); - BasicDBObject params = new BasicDBObject(); - params.put("profile", profile); - params.put("input", "c:/_workspace/data/video/proba1.mxf"); - params.put("output", "c:/_video/proba1.mp4"); - - long jobId = mc.startjob(template, name, params); - logger.info(getMarker(), "Started {} on server {}", jobId, server); - while (true) { - BasicDBObject status = mc.getStatus(jobId); - if (status != null) - setProgress(status.getInt("progress")); - - String jobStatus = status.getString("status"); - if ("SUSPENDED".equals(jobStatus)) - throw new Exception(status.getString("description")); - - if ("FINISHED".equals(jobStatus)) - break; - } - - } catch (Exception e) { - throw e; - } - return new Object[] { null }; - } -} diff --git a/server/-product/production/LOCAL/jobs/steps/TSMBackupStep.java b/server/-product/production/LOCAL/jobs/steps/TSMBackupStep.java deleted file mode 100644 index 60adc661..00000000 --- a/server/-product/production/LOCAL/jobs/steps/TSMBackupStep.java +++ /dev/null @@ -1,207 +0,0 @@ -package user.jobengine.server.steps; - -import java.io.File; -import java.io.IOException; -import java.nio.file.Files; -import java.nio.file.attribute.BasicFileAttributes; -import java.sql.Timestamp; -import java.util.Date; -import java.util.List; - -import org.apache.commons.lang.RandomStringUtils; -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; -import org.apache.logging.log4j.Marker; -import org.apache.logging.log4j.message.Message; -import org.apache.logging.log4j.message.ParameterizedMessage; - -import user.commons.RemoteFile; -import user.commons.StoreUri; -import user.commons.mediatool.Timecode; -import user.commons.mediatool.Timecode.Type; -import user.commons.remotestore.IProgressEventListener; -import user.commons.remotestore.IStatusEventListener; -import user.commons.remotestore.ProgressEvent; -import user.commons.remotestore.RemoteStoreProtocol; -import user.commons.remotestore.StatusEvent; -import user.jobengine.db.FileType; -import user.jobengine.db.IItemManager; -import user.jobengine.db.Media; -import user.jobengine.db.MediaFile; -import user.jobengine.db.Store; -import user.jobengine.server.IJobEngine; -import user.jobengine.server.IJobRuntime; -import user.jobengine.server.JobEngineException; - -public class TSMBackupStep extends JobStep { - private static final String MXFEXT = ".MXF"; - private static final Logger logger = LogManager.getLogger(); - private IItemManager manager; - private File sourceMediaFile; - private Store tsmStore; - private StoreUri targetUri; - private FileType fileType; - private Marker marker; - - @StepEntry - public Object[] execute(ArchiveItem archiveItem, Media mediaCubeMedia, int killDateDays) throws Exception { - marker = getSessionMarker(); - - File sourceMediaFile = new File(archiveItem.getMediaFile()); - String sourceFileName = sourceMediaFile.getName(); - Timecode timecode = new Timecode(mediaCubeMedia.getLength(), Type.PAL); - String details = String.format("%s (%s, %d bytes)", sourceFileName, timecode.toString(), sourceMediaFile.length()); - getJobRuntime().setDescription(details); - try { - - setAndCheck(archiveItem, mediaCubeMedia, getEngine()); - - //TODO mxf helyett az osszes kiterjesztest!!!!! - //A dupla ellenorzes a napon beluli ismetlesek miatt kell - long existingMediaId = archiveItem.getExistingMediaId(); - if (sourceMediaFile.length() == 0 && existingMediaId == 0) { - existingMediaId = ItemManagerExtensions.getExistingRundownMedia(manager, sourceFileName.replace(MXFEXT, "")); - if (existingMediaId == 0) - existingMediaId = -1; - } - - if (existingMediaId == 0) - existingMediaId = ItemManagerExtensions.getExistingRundownMedia(manager, sourceFileName.replace(MXFEXT, "")); - - boolean randomizeName = Boolean.parseBoolean(System.getProperty("jobengine.randomize.archives", "false")); - - String targetFileName; - if (randomizeName) { - // a-z, A-Z, 0-9. For example: WRMcpIk7, s57JwCVA - //veletlenszeru neveket adunk! - targetFileName = String.format("%s-%s", RandomStringUtils.randomAlphanumeric(8), sourceFileName); - } else - targetFileName = sourceFileName; - - if (existingMediaId == 0) { - StoreUri sourceUri = manager.createStoreUri(RemoteStoreProtocol.LOCAL, sourceMediaFile.getParent().toString()); - - final IJobRuntime runtime = getJobRuntime(); - sourceUri.addProgressListener(new IProgressEventListener() { - @Override - public void progressChanged(ProgressEvent evt) { - runtime.incrementProgress(evt.getProgress()); - } - }); - sourceUri.addStatusListener(new IStatusEventListener() { - @Override - public void statusChanged(StatusEvent evt) { - evt.setCancel(!canContinue()); - } - }); - - RemoteFile remoteFile = sourceUri.transferFrom(targetUri, sourceFileName, targetFileName); - } - - if (existingMediaId > 0) - logger.info(marker, "Az '{}' TSM mentése nem szükséges, mert már megtalálható az archívumban.", sourceFileName); - - //Fel kell szabadítani, hogy a kovetkezo archivalaskor is nekifusson - if (existingMediaId == -1) { - logger.info(marker, "Az '{}' mentése jelenleg nem lehetséges, mert a szükséges metaadat még nem található meg az archívumban.", sourceFileName); - if (!archiveItem.removeCatchedFile()) - logger.error(marker, - "Az '{}' állomány .catched jelző állománya nem törölhető. Az újabb archiválási kísérlethez annak kézi eltávolítása szükséges!", - sourceMediaFile.getName()); - mediaCubeMedia.remove(); - } else { - saveMetadata(mediaCubeMedia, sourceMediaFile, targetFileName, existingMediaId); - logger.info(marker, "Az '{}' archiválása sikeres.", sourceFileName); - if (killDateDays != 0) - EscortFiles.createUNCKillDate(sourceMediaFile.getParent(), sourceFileName, killDateDays, marker); - } - - } catch (Exception e) { - logger.catching(e); - Message m = new ParameterizedMessage("Az '{}' állomány archiválása sikertelen. A rendszer hibaüzenete: {}", details, e.getMessage()); - logger.error(marker, m); - if (!archiveItem.removeCatchedFile()) - logger.error(marker, - "Az '{}' állomány .catched jelző állománya nem törölhető. Az újabb archiválási kísérlethez annak kézi eltávolítása szükséges!", - sourceMediaFile.getName()); - throw new Exception(m.getFormattedMessage()); - } - return null; - } - - private void saveMetadata(Media mediaCubeMedia, File sourceFile, String targetFileName, long existingMediaId) { - - if (existingMediaId == 0) { - MediaFile mediaFile = manager.createMediaFile(targetFileName, fileType, tsmStore, mediaCubeMedia); - mediaFile.setHouseId(sourceFile.getName()); - mediaFile.add(); - } else { - Media existingMedia = manager.getMedia(existingMediaId); - List mediaFiles = existingMedia.getMediaFiles(); - if (mediaFiles != null) { - for (MediaFile mf : mediaFiles) { - mf.setPersister(manager); - mf.setId(0); - mf.setMedia(mediaCubeMedia); - mf.add(); - } - } - } - mediaCubeMedia.setPersister(manager); - - try { - BasicFileAttributes attr = Files.readAttributes(sourceFile.toPath(), BasicFileAttributes.class); - mediaCubeMedia.setArchived(new Timestamp(attr.creationTime().toMillis())); - } catch (IOException e) { - logger.catching(e); - mediaCubeMedia.setArchived(new Timestamp(new Date().getTime())); - } - - mediaCubeMedia.modify(); - } - - private void setAndCheck(ArchiveItem archiveItem, Media mediaCubeMedia, IJobEngine jobEngine) throws JobEngineException, IOException { - if (jobEngine == null) { - logger.error(marker, "Az folyamatkezelő réteg nem elérhető."); - throw new NullPointerException("Internal error, missing JobEngine reference."); - } - manager = jobEngine.getItemManager(); - if (manager == null) { - logger.error(marker, "Az adatbáziskezelő réteg nem elérhető."); - throw new NullPointerException("Internal error, missing ItemManager reference."); - } - if (archiveItem == null) { - logger.error(marker, "A folyamat 'archiveItem' bemeneti paramétere üres."); - throw new NullPointerException("System is not configured properly, missing 'archiveItem' input parameter."); - } - sourceMediaFile = new File(archiveItem.getMediaFile()); - if (sourceMediaFile == null) { - logger.error(marker, "A folyamat 'archiveItem' bemeneti paraméter 'mediaFile' értéke üres."); - throw new NullPointerException("System is not configured properly, missing 'mediaFile' value in 'archiveItem' input parameter."); - } - if (!sourceMediaFile.exists()) { - logger.error(marker, "Az {} állomány nem létezik vagy nem érhető el.", sourceMediaFile.getName()); - throw new IOException(String.format("Input file {} not exists or unreachable.", sourceMediaFile.getName())); - } - tsmStore = manager.getSystemStore(false); - if (tsmStore == null) { - logger.error(marker, "A TSM rendszer beállítás nem elérhető."); - throw new NullPointerException("System is not configured properly, missing TSM Store."); - } - targetUri = tsmStore.getSourceStoreUri(RemoteStoreProtocol.TSM); - if (targetUri == null) { - logger.error(marker, "A TSM rendszer beállítás paraméterei nem elérhetőek."); - throw new NullPointerException("System is not configured properly, missing TSM StoreUri."); - } - fileType = manager.getFileType("High-res"); - if (fileType == null) { - logger.error(marker, "Adatbázis bejegyzés hiba, a 'High-res' FileType nem található."); - throw new NullPointerException("System is not configured properly, missing 'High-res' FileType."); - } - if (mediaCubeMedia == null) { - logger.error(marker, "A folyamat 'mediaCubeMedia' bemeneti paramétere üres."); - throw new NullPointerException("System is not configured properly, 'mediaCubeMedia' input parameter missing."); - } - - } -} diff --git a/server/-product/production/LOCAL/jobs/steps/TSMExtendedRetrieveStep.java b/server/-product/production/LOCAL/jobs/steps/TSMExtendedRetrieveStep.java deleted file mode 100644 index 79a601ca..00000000 --- a/server/-product/production/LOCAL/jobs/steps/TSMExtendedRetrieveStep.java +++ /dev/null @@ -1,91 +0,0 @@ -package user.jobengine.server.steps; - -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; - -import user.commons.StoreUri; -import user.commons.remotestore.RemoteStoreProtocol; -import user.jobengine.db.ArchivedMedia; -import user.jobengine.db.IItemManager; -import user.jobengine.server.IJobEngine; -import user.jobengine.server.IJobRuntime; - -public class TSMExtendedRetrieveStep extends TSMRestoreStep { - private static String NEXIO_HOST = System.getProperty("nexio.host"); - // private static String NEXIO_HOST = SystemConfiguration.getInstance().value("services.nexio.host"); - private static final Logger logger = LogManager.getLogger(); - - private boolean useNexioTarget; - private int nexioPort; - private String nexioUserName, nexioPassword; - private String nexioAgency; - - @Override - protected void afterRestore(StoreUri targetUri, String targetPath, int killDateDays, String targetFileName) throws Exception { - if (useNexioTarget) { - EscortFiles.setNEXIOKillDate(killDateDays, targetFileName, nexioAgency, targetUri); - } else { - super.afterRestore(targetUri, targetPath, killDateDays, targetFileName); - } - } - - @Override - protected void beforeRestore(StoreUri targetURI, String targetName) throws Exception { - String newTargetName = targetName; - if (targetName.contains(".")) - newTargetName = targetName.substring(0, targetName.lastIndexOf('.')); - if (useNexioTarget) - if (targetURI.fileExists(newTargetName + ".mxf")) - throw new Exception(String.format("%s-The newly created file name is existed.", getClass().getSimpleName())); - } - - @Override - protected void checkTargetPath(String targetPath) { - if (!useNexioTarget) - super.checkTargetPath(targetPath); - } - - @Override - protected StoreUri createTargetUri(IItemManager manager, String targetPath) throws NullPointerException { - StoreUri result = null; - logger.info(getSessionMarker(), "Create target uri {}", targetPath); - if (useNexioTarget) { - if (NEXIO_HOST == null) { - throw new NullPointerException("Missing system property on 'nexio.host' name"); - } - result = manager.createStoreUri(RemoteStoreProtocol.FTP, NEXIO_HOST); - result.setPortNumber(nexioPort); - result.setUserName(nexioUserName); - result.setPassword(nexioPassword); - } else - result = super.createTargetUri(manager, targetPath); - return result; - } - - @StepEntry - public Object[] execute(ArchivedMedia archivedMedia, String targetPath, String targetNamePattern, String successRecipient, int killDateDays, - String localRetrievePath, String globalRetrievePath, boolean useNexioTarget, String nexioAgency, int nexioPort, String nexioUserName, - String nexioPassword, IJobEngine jobEngine, IJobRuntime jobRuntime) throws Exception { - this.useNexioTarget = useNexioTarget; - this.nexioAgency = nexioAgency; - this.nexioPort = nexioPort; - this.nexioUserName = nexioUserName; - this.nexioPassword = nexioPassword; - if (nexioPort == 0) { - throw new NullPointerException("System is not configured properly, 'nexioPort' input parameter missing."); - } - if (nexioUserName == null) { - throw new NullPointerException("System is not configured properly, 'nexioUserName' input parameter missing."); - } - if (nexioPassword == null) { - throw new NullPointerException("System is not configured properly, 'nexioPassword' input parameter missing."); - } - if (nexioAgency == null) { - throw new NullPointerException("System is not configured properly, 'nexioAgency' input parameter missing."); - } - - return super.execute(archivedMedia.getMedia(), targetPath, targetNamePattern, successRecipient, killDateDays, localRetrievePath, globalRetrievePath, - jobEngine, jobRuntime); - } - -} diff --git a/server/-product/production/LOCAL/jobs/steps/TSMRestoreStep.java b/server/-product/production/LOCAL/jobs/steps/TSMRestoreStep.java deleted file mode 100644 index 5b410055..00000000 --- a/server/-product/production/LOCAL/jobs/steps/TSMRestoreStep.java +++ /dev/null @@ -1,159 +0,0 @@ -package user.jobengine.server.steps; - -import java.io.IOException; -import java.nio.file.Paths; -import java.util.List; - -import org.apache.commons.lang.StringUtils; -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; -import org.apache.logging.log4j.Marker; -import org.apache.logging.log4j.message.Message; - -import user.commons.LogUtils; -import user.commons.RemoteFile; -import user.commons.StoreUri; -import user.commons.mediatool.Timecode; -import user.commons.mediatool.Timecode.Type; -import user.commons.remotestore.IProgressEventListener; -import user.commons.remotestore.IStatusEventListener; -import user.commons.remotestore.ProgressEvent; -import user.commons.remotestore.RemoteStoreProtocol; -import user.commons.remotestore.StatusEvent; -import user.jobengine.db.IItemManager; -import user.jobengine.db.Media; -import user.jobengine.db.MediaFile; -import user.jobengine.db.Store; -import user.jobengine.server.IJobEngine; -import user.jobengine.server.IJobRuntime; - -public class TSMRestoreStep extends JobStep { - private static final Logger logger = LogManager.getLogger(); - private IItemManager manager; - private StoreUri targetUri; - private StoreUri sourceUri; - private String sourceFileName; - private Marker marker; - - protected void afterRestore(StoreUri targetUri, String targetPath, int killDateDays, String targetFileName) throws IOException, Exception { - if (killDateDays > 0) - EscortFiles.createUNCKillDate(targetPath, targetFileName, killDateDays, marker); - } - - protected void beforeRestore(StoreUri targetURI, String targetFileName) throws Exception { - } - - protected void checkTargetPath(String targetPath) { - if (StringUtils.isBlank(targetPath)) { - logger.error(marker, "A folyamat 'targetPath' bemeneti paramétere üres."); - throw new NullPointerException("System is not configured properly, 'targetPath' input parameter missing."); - } - } - - protected StoreUri createTargetUri(IItemManager manager, String targetPath) { - return manager.createStoreUri(RemoteStoreProtocol.LOCAL, targetPath); - } - - @StepEntry - public Object[] execute(Media mediaCubeMedia, String targetPath, String targetNamePattern, String successRecipient, int killDateDays, - String localRetrievePath, String globalRetrievePath, IJobEngine jobEngine, IJobRuntime jobRuntime) throws Exception { - marker = jobRuntime.getSessionMarker(); - setAndCheck(mediaCubeMedia, targetPath, targetNamePattern, localRetrievePath, globalRetrievePath, jobEngine); - String targetFileName = String.format(targetNamePattern, sourceFileName); - Timecode timecode = new Timecode(mediaCubeMedia.getLength(), Type.PAL); - try { - String details = String.format("%s (%s)", sourceFileName, timecode.toString()); - jobRuntime.setDescription(details); - beforeRestore(targetUri, targetNamePattern); - final IJobRuntime runtime = jobRuntime; - sourceUri.addProgressListener(new IProgressEventListener() { - @Override - public void progressChanged(ProgressEvent evt) { - runtime.incrementProgress(evt.getProgress()); - } - }); - sourceUri.addStatusListener(new IStatusEventListener() { - @Override - public void statusChanged(StatusEvent evt) { - evt.setCancel(!canContinue()); - } - }); - RemoteFile result = sourceUri.transferFrom(targetUri, sourceFileName, targetFileName); - - String globalTargetPath = Paths.get(targetPath, targetFileName).getParent().toString().replace(Paths.get(localRetrievePath).toString(), - globalRetrievePath); - - logger.info(marker, - "Az '{}' állomány visszatöltése sikeres volt '{}' néven. A célmappa a ide kattintva nyitható meg.", - sourceFileName, targetFileName, globalTargetPath); - afterRestore(targetUri, targetPath, killDateDays, targetFileName); - - } catch (Exception e) { - Message msg = LogUtils.format("Az '{}' állomány visszatöltése sikertelen. A rendszer hibaüzenete: {}", sourceFileName, e.getMessage()); - logger.error(marker, msg); - // logger.error(jobRuntime.marker, msg); - logger.catching(e); - throw e; - } - - return null; - } - - private String getSourceFileName(Media mediaCubeMedia, Store store) { - List mediaFiles = mediaCubeMedia.getMediaFiles(); - if (mediaFiles == null) - return null; - for (MediaFile mediaFile : mediaFiles) { - if (mediaFile.getStore().getId() == store.getId()) - return mediaFile.getRelativePath(); - } - return null; - } - - private void setAndCheck(Media mediaCubeMedia, String targetPath, String targetNamePattern, String localRetrievePath, String globalRetrievePath, - IJobEngine jobEngine) { - if (jobEngine == null) { - logger.error(marker, "Az folyamatkezelő réteg nem elérhető."); - throw new NullPointerException("Internal error, missing JobEngine reference."); - } - manager = jobEngine.getItemManager(); - if (manager == null) { - logger.error(marker, "Az adatbáziskezelő réteg nem elérhető."); - throw new NullPointerException("Internal error, missing ItemManager reference."); - } - if (mediaCubeMedia == null) { - logger.error(marker, "A folyamat 'mediaCubeMedia' bemeneti paramétere üres."); - throw new NullPointerException("System is not configured properly, 'mediaCubeMedia' input parameter missing."); - } - checkTargetPath(targetPath); - if (StringUtils.isBlank(targetNamePattern)) { - logger.error(marker, "A folyamat 'targetNamePattern' bemeneti paramétere üres."); - throw new NullPointerException("System is not configured properly, 'targetNamePattern' input parameter missing."); - } - Store tsmStore = manager.getSystemStore(false); - if (tsmStore == null) { - logger.error(marker, "A TSM rendszer beállítás nem elérhető."); - throw new NullPointerException("System is not configured properly, missing TSM Store."); - } - sourceUri = tsmStore.getSourceStoreUri(RemoteStoreProtocol.TSM); - if (sourceUri == null) { - logger.error(marker, "A TSM rendszer beállítás paraméterei nem elérhetőek."); - throw new NullPointerException("System is not configured properly, missing TSM StoreUri."); - } - targetUri = createTargetUri(manager, targetPath); - sourceFileName = getSourceFileName(mediaCubeMedia, tsmStore); - if (sourceFileName == null) { - logger.error(marker, "Adatbázis bejegyzés hiba, a visszatöltendő fájl neve nem található."); - throw new NullPointerException("Database error, missing MediaFile 'relativePath'."); - } - - if (StringUtils.isBlank(localRetrievePath)) { - logger.error(marker, "A folyamat 'localRetrievePath' bemeneti paramétere üres."); - throw new NullPointerException("System is not configured properly, 'localRetrievePath' input parameter missing."); - } - if (StringUtils.isBlank(globalRetrievePath)) { - logger.error(marker, "A folyamat 'globalRetrievePath' bemeneti paramétere üres."); - throw new NullPointerException("System is not configured properly, 'globalRetrievePath' input parameter missing."); - } - } -} diff --git a/server/-product/production/LOCAL/jobs/steps/TSMSimpleRestoreStep.java b/server/-product/production/LOCAL/jobs/steps/TSMSimpleRestoreStep.java deleted file mode 100644 index ba63212c..00000000 --- a/server/-product/production/LOCAL/jobs/steps/TSMSimpleRestoreStep.java +++ /dev/null @@ -1,57 +0,0 @@ -package user.jobengine.server.steps; - -import java.nio.file.Files; -import java.nio.file.Paths; - -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; - -import user.commons.StoreUri; -import user.commons.mediatool.Timecode; -import user.commons.mediatool.Timecode.Type; -import user.commons.remotestore.IProgressEventListener; -import user.commons.remotestore.IStatusEventListener; -import user.commons.remotestore.ProgressEvent; -import user.commons.remotestore.RemoteStoreProtocol; -import user.commons.remotestore.StatusEvent; -import user.jobengine.db.Media; -import user.jobengine.db.MediaFile; -import user.jobengine.db.Store; - -public class TSMSimpleRestoreStep extends JobStep { - private static final Logger logger = LogManager.getLogger(); - - @StepEntry - public Object[] execute(long mediaId, String targetPath) throws Exception { - String fileName = null; - try { - Media media = getManager().getMedia(mediaId); - MediaFile mediaFile = getManager().getSystemMediaFile(media); - fileName = mediaFile.getRelativePath(); - Timecode timecode = new Timecode(media.getLength(), Type.PAL); - getJobRuntime().setDescription(String.format("%s (%s)", fileName, timecode.toString())); - Store tsmStore = getManager().getSystemStore(false); - StoreUri sourceStoreUri = getManager().getStoreUri(tsmStore.getName(), RemoteStoreProtocol.TSM); - sourceStoreUri.addProgressListener(new IProgressEventListener() { - @Override - public void progressChanged(ProgressEvent evt) { - setProgress(evt.getProgress()); - } - }); - sourceStoreUri.addStatusListener(new IStatusEventListener() { - @Override - public void statusChanged(StatusEvent evt) { - evt.setCancel(!canContinue()); - } - }); - StoreUri targetStoreUri = getManager().createStoreUri(RemoteStoreProtocol.LOCAL, targetPath); - sourceStoreUri.transferFrom(targetStoreUri, fileName, fileName + ".part"); - Files.move(Paths.get(targetPath, fileName + ".part"), Paths.get(targetPath, fileName)); - } catch (Exception e) { - logger.error("Az '{}' állomány visszatöltése sikertelen. A rendszer üzenete: {}", fileName, e.getMessage()); - throw e; - } - - return null; - } -} diff --git a/server/-product/production/LOCAL/jobs/steps/TestForkCancelableStep.java b/server/-product/production/LOCAL/jobs/steps/TestForkCancelableStep.java deleted file mode 100644 index c1b49f76..00000000 --- a/server/-product/production/LOCAL/jobs/steps/TestForkCancelableStep.java +++ /dev/null @@ -1,35 +0,0 @@ -package user.jobengine.server.steps; - -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; - -import user.commons.IJob; -import user.commons.JobStatus; -import user.commons.ListUtils; -import user.jobengine.server.IJobEngine; -import user.jobengine.server.IJobRuntime; - -public class TestForkCancelableStep extends JobStep { - private static final String CHILD_TITLE = "Párhuzamosított alfolyamat"; - // private static final String CHILD_TEMPLATE = "fake-concurrent.xml"; - private static final String CHILD_TEMPLATE = "cancelable.xml"; - private static final Logger logger = LogManager.getLogger(); - int count = 3; - - @StepEntry - public Object[] execute(IJobEngine jobEngine, IJobRuntime jobRuntime) throws Exception { - for (int i = 0; i < count; i++) { - IJobRuntime rt = getEngine().submit(null, e -> { - if (e.getStatus().equals(JobStatus.SUSPENDED)) { - IJobRuntime rt = (IJobRuntime) e.getSource(); - logger.info("Cleanup on SUSPEND {}, {}", rt.getId(), rt.isDisableRetry()); - } - }, CHILD_TEMPLATE, "JOB " + i, 0, IJobEngine.DEFAULT_OWNER, ListUtils.asMap("param", i)); - - rt.setRelated("TEST" + rt.getId()); - } - - logger.info("Done"); - return null; - } -} diff --git a/server/-product/production/LOCAL/jobs/steps/TestLib.java b/server/-product/production/LOCAL/jobs/steps/TestLib.java deleted file mode 100644 index 040153ec..00000000 --- a/server/-product/production/LOCAL/jobs/steps/TestLib.java +++ /dev/null @@ -1,8 +0,0 @@ -package user.jobengine.server.steps; - -public class TestLib { - - public void hello() { - System.out.println("Hello from lib"); - } -} diff --git a/server/-product/production/LOCAL/jobs/steps/TranscodeFFAStranStep.java b/server/-product/production/LOCAL/jobs/steps/TranscodeFFAStranStep.java deleted file mode 100644 index a73ff663..00000000 --- a/server/-product/production/LOCAL/jobs/steps/TranscodeFFAStranStep.java +++ /dev/null @@ -1,151 +0,0 @@ -package user.jobengine.server.steps; - -import java.io.File; -import java.io.IOException; -import java.nio.file.Files; -import java.nio.file.Path; -import java.nio.file.Paths; - -import org.apache.commons.io.FilenameUtils; -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; -import org.apache.logging.log4j.Marker; -import org.apache.logging.log4j.message.Message; -import org.apache.logging.log4j.message.ParameterizedMessage; - -import user.commons.FFAStransAPI; -import user.commons.IFFAStransAPI; -import user.commons.StoreUri; -import user.commons.mediatool.Timecode; -import user.commons.mediatool.Timecode.Type; -import user.commons.remotestore.RemoteStoreProtocol; -import user.jobengine.db.FileType; -import user.jobengine.db.IItemManager; -import user.jobengine.db.Media; -import user.jobengine.db.Store; -import user.jobengine.server.IJobEngine; -import user.jobengine.server.IJobRuntime; - -public class TranscodeFFAStranStep extends JobStep { - private static final int POLL_INTERVALL = 3000; - private static final String MP4EXT = ".MP4"; - private static final String MXFEXT = ".MXF"; - private static final String LOWRES_FILETYPE = "Low-res"; - private static final Logger logger = LogManager.getLogger("TranscodeFFAStranStep"); - private IItemManager manager; - private Store store; - private FileType fileType; - private Media mediaCubeMedia; - private Marker marker; - - @StepEntry - public Object[] execute(ArchiveItem archiveItem, Media mediaCubeMedia, String transcoderAddress, String transcoderTemplateName, - String globalHiresSourcePath, String localLowresTargetPath, boolean deleteSource, IJobEngine jobEngine, IJobRuntime jobRuntime) throws Exception { - - this.marker = jobRuntime.getSessionMarker(); - this.manager = jobEngine.getItemManager(); - this.store = check(manager.getCurrentLowresStore(), "lowres Store"); - this.fileType = check(manager.getFileType(LOWRES_FILETYPE), "lowres FileType"); - this.mediaCubeMedia = check(mediaCubeMedia, "mediaCubeMedia"); - check(archiveItem, "archiveItem"); - check(transcoderAddress, "transcoderAddress"); - check(transcoderTemplateName, "transcoderTemplateName"); - check(globalHiresSourcePath, "globalHiresSourcePath"); - check(localLowresTargetPath, "localLowresTargetPath"); - - File sourceMediaFile = new File(archiveItem.getMediaFile()); - logger.info("Transcoding {}", archiveItem.getMediaFile()); - String sourceFileName = sourceMediaFile.getName(); - Timecode timecode = new Timecode(mediaCubeMedia.getLength(), Type.PAL); - - String details = String.format("%s (%s, %d bytes)", sourceFileName, timecode.toString(), sourceMediaFile.length()); - - StoreUri storeUri = store.getTargetStoreUri(RemoteStoreProtocol.LOCAL); - if (storeUri == null) - throw new Exception("Can not detect proxy folder."); - - String webPath = storeUri.toString(true); - - Path targetPath = null; - try { - String targetFileName = FilenameUtils.removeExtension(sourceFileName) + MP4EXT; - targetPath = Paths.get(localLowresTargetPath, targetFileName); - if (!targetPath.toFile().exists()) { - // jobRuntime.setDescription(String.format("%s: %s", jobRuntime.getDescription(), details)); - jobRuntime.setDescription(String.format("%s transzkódolása", details)); - String sourceFile = Paths.get(globalHiresSourcePath, sourceFileName).toString(); - IFFAStransAPI api = new FFAStransAPI(transcoderAddress, p -> { - if (p <= 100) - jobRuntime.incrementProgress(p); - }); - - api.submit(transcoderTemplateName, sourceFile); - api.monitor(POLL_INTERVALL); - } - - //a sikeres transzkod utan nem mindig van ott egybol a fajl - long started = System.currentTimeMillis(); - while (!targetPath.toFile().exists()) { - long current = System.currentTimeMillis(); - //max 5 perc varakozas - if (current - started > 5 * 60 * 1000) - throw new Exception("Transcode job target file access timed out"); - Thread.sleep(POLL_INTERVALL); - } - - postprocess(targetPath, webPath); - - } catch (Exception e) { - logger.catching(e); - Message m = new ParameterizedMessage("{} átkódolás hiba: {}", sourceFileName, e.getMessage()); - logger.error(marker, m); - throw new Exception(m.getFormattedMessage()); - } finally { - try { - if (deleteSource && sourceMediaFile != null && sourceMediaFile.exists()) - sourceMediaFile.delete(); - } catch (Exception e) { - logger.catching(e); - } - try { - if (deleteSource && targetPath != null && targetPath.toFile().exists()) - Files.delete(targetPath); - } catch (Exception e) { - logger.catching(e); - } - } - return null; - } - - private void postprocess(Path transcodedFilePath, String webPath) throws IOException { - Path lowresPath = null; - try { - String transcodedFileName = transcodedFilePath.getFileName().toString(); - String targetPath = null; - if (transcodedFileName.indexOf(".") > 2) { - Path subdir = Paths.get(transcodedFileName.substring(0, 1), transcodedFileName.substring(1, 2), transcodedFileName.substring(2, 3)); - EscortFiles.ensureUNCFolder(webPath, subdir.toString()); - targetPath = Paths.get(subdir.toString(), transcodedFileName).toString(); - } else { - targetPath = transcodedFileName; - } - lowresPath = Paths.get(webPath, targetPath); - int version = 1; - while (lowresPath.toFile().exists()) { - String fileName = transcodedFileName + version + MP4EXT; - lowresPath = Paths.get(lowresPath.toString().replace(transcodedFileName, fileName)); - targetPath = targetPath.replace(transcodedFileName, fileName); - transcodedFileName = fileName; - version++; - } - - Files.move(transcodedFilePath, lowresPath); - manager.createMediaFile(targetPath, fileType, store, mediaCubeMedia).add(); - } catch (IOException e) { - logger.catching(e); - logger.error(marker, "A '{}' állomány mozgatása a '{}' helyre nem sikerült.", transcodedFilePath, lowresPath); - throw e; - } - } - -} diff --git a/server/-product/production/LOCAL/jobs/steps/TranscodeFFMpegStep.java b/server/-product/production/LOCAL/jobs/steps/TranscodeFFMpegStep.java deleted file mode 100644 index 4033b8e3..00000000 --- a/server/-product/production/LOCAL/jobs/steps/TranscodeFFMpegStep.java +++ /dev/null @@ -1,52 +0,0 @@ -package user.jobengine.server.steps; - -/* -import user.jobengine.server.steps.shared.IExternalCallback; -import user.jobengine.server.steps.shared.ExternalProfile; -import user.jobengine.server.steps.shared.ExternalProfilesConfig; -import user.jobengine.server.steps.shared.ExternalCommand; -import user.jobengine.server.steps.shared.ExternalCommandExecutor; - */ - -import java.nio.file.Paths; - -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; - -import user.commons.mediaarea.MediaArea; -import user.jobengine.server.steps.shared.ExternalCommandExecutor; -import user.jobengine.server.steps.shared.IExternalCallback; - -public class TranscodeFFMpegStep extends JobStep implements IExternalCallback { - private static final Logger logger = LogManager.getLogger(); - private long allFrameCount; - private long processedFrameCount; - - @StepEntry - public Object[] execute(String input, String output, String profile) throws Exception { - try { - MediaArea ma = new MediaArea(Paths.get(input)); - ma.process(); - allFrameCount = ma.getFrameCount(); - - ExternalCommandExecutor executor = new ExternalCommandExecutor(); - executor.execute(profile, input, output, this); - } catch (Exception e) { - logger.error(getMarker(), "Media information not available. System message: {}", e.getMessage()); - throw e; - } - return null; - } - - @Override - public void onResponse(String line) { - String[] values = line.split("="); - if ("frame".equals(values[0])) { - processedFrameCount = Integer.parseInt(values[1]); - int progress = (int) (processedFrameCount * 100 / allFrameCount); - if (progress > 0) - setProgress(progress); - } - } - -} diff --git a/server/-product/production/LOCAL/jobs/steps/UpdateGhostMediaDataStep.java b/server/-product/production/LOCAL/jobs/steps/UpdateGhostMediaDataStep.java deleted file mode 100644 index e59008d1..00000000 --- a/server/-product/production/LOCAL/jobs/steps/UpdateGhostMediaDataStep.java +++ /dev/null @@ -1,88 +0,0 @@ -package user.jobengine.server.steps; - -import java.util.List; - -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; -import org.apache.logging.log4j.Marker; - -import user.commons.remotestore.RemoteStoreProtocol; -import user.jobengine.db.IItemManager; -import user.jobengine.db.Media; -import user.jobengine.db.MediaFile; -import user.jobengine.db.Store; -import user.jobengine.server.IJobEngine; -import user.jobengine.server.IJobRuntime; -import user.jobengine.server.steps.MetadataTypeDetector.MetadataType; - -public class UpdateGhostMediaDataStep extends JobStep { - private static final Logger logger = LogManager.getLogger(); - private Marker marker; - - @StepEntry - public Object[] execute(Media mediaCubeMedia, IJobEngine jobEngine, IJobRuntime jobRuntime) throws Exception { - marker = jobRuntime.getSessionMarker(); - - IItemManager manager = jobEngine.getItemManager(); - //Refresh from db - List mediaFiles = manager.getMedia(mediaCubeMedia.getId()).getMediaFiles(); - if (mediaFiles != null && mediaFiles.size() == 2) { - MediaFile lowres = null; - MediaFile highres = null; - - for (MediaFile mf : mediaFiles) { - if (mf.getStore().getSourceStoreUri(RemoteStoreProtocol.HTTP) != null) - lowres = mf; - else - highres = mf; - } - - if (highres == null) { - logger.info(marker, "Nincs highres mediaId: {}", mediaCubeMedia.getId()); - return null; - } - if (lowres == null) { - logger.info(marker, "Nincs lowres mediaId: {}", mediaCubeMedia.getId()); - return null; - } - - String id = MetadataTypeDetector.truncateExtension(highres.getRelativePath()); - id = MetadataTypeDetector.truncateVersion(id); - boolean detect = MetadataTypeDetector.GuessMetadataType(id) == MetadataType.OctopusPlaceholder - || MetadataTypeDetector.GuessMetadataType(id) == MetadataType.OctopusStory; - if (!detect) { - logger.info(marker, "Nem bejátszó mediaId: {}, file: {}", mediaCubeMedia.getId(), highres.getRelativePath()); - return null; - } - - Store highresStore = manager.getSystemStore(false); - final long sourceMediaId = lowres.getId(); - final long highresMediaFileId = highres.getId(); - final String highresRealtivePath = highres.getRelativePath(); - - manager.executeQuery("SELECT mediaid FROM mediafile WHERE relativepath=? and storeid=? and id!=?", rs -> { - long mediaId = rs.getLong(1); - Media media = manager.getMedia(mediaId); - if (media.getMediaFilesCount() == 1) { - logger.info(marker, "Hiányzó szellem lowres hozzáadása {} alapján", media.getId()); - - MediaFile mf = (MediaFile) manager.get(MediaFile.class, sourceMediaId); - mf.setMedia(media); - mf.setId(0); - manager.add(mf); - media.setLength(mediaCubeMedia.getLength()); - manager.modify(media); - } - return true; - }, st -> { - st.setString(1, highresRealtivePath); - st.setLong(2, highresStore.getId()); - st.setLong(3, highresMediaFileId); - }); - - } - - return null; - } - -} diff --git a/server/-product/production/LOCAL/settings/application-htv.yaml b/server/-product/production/LOCAL/settings/application-htv.yaml index 3dbde683..b579b23e 100644 --- a/server/-product/production/LOCAL/settings/application-htv.yaml +++ b/server/-product/production/LOCAL/settings/application-htv.yaml @@ -1,28 +1,16 @@ datasource: mediacube: -# url: jdbc:db2://10.11.1.90:50000/mc -# user: db2admin -# password: password - url: jdbc:db2://192.168.100.2:50000/mc - user: db2inst1 + url: jdbc:db2://10.10.1.27:50000/mc + user: db2admin password: password -# url: jdbc:db2://10.10.1.27:50000/mc -# user: db2admin -# password: password external-indexer: false simple-search: true login-timeout: 3 pool-size: 10 mediacube-nosql: -# url: jdbc:db2://10.11.1.90:50000/mc -# user: db2admin -# password: password - url: jdbc:db2://192.168.100.2:50000/mc - user: db2inst1 + url: jdbc:db2://10.10.1.27:50000/mc + user: db2admin password: password -# url: jdbc:db2://10.10.1.27:50000/mc -# user: db2admin -# password: password schema: test login-timeout: 3 hsm: diff --git a/server/-product/production/LOCAL/settings/application-local.yaml b/server/-product/production/LOCAL/settings/application-mv.yaml similarity index 100% rename from server/-product/production/LOCAL/settings/application-local.yaml rename to server/-product/production/LOCAL/settings/application-mv.yaml diff --git a/server/-product/production/config-210526.ini b/server/-product/production/config-210526.ini new file mode 100644 index 00000000..0ff1f357 --- /dev/null +++ b/server/-product/production/config-210526.ini @@ -0,0 +1,9 @@ +#This configuration file was written by: org.eclipse.equinox.internal.frameworkadmin.equinox.EquinoxFwConfigFileParser +#Thu May 13 15:59:00 CEST 2021 +osgi.bundles=reference\:file\:cglib_2.2.2.jar@4,reference\:file\:com.fasterxml.jackson.core.jackson-annotations_2.4.5.jar@4,reference\:file\:com.fasterxml.jackson.core.jackson-core_2.4.5.jar@4,reference\:file\:com.fasterxml.jackson.core.jackson-databind_2.4.5.jar@4,reference\:file\:com.fasterxml.jackson.dataformat.jackson-dataformat-yaml_2.4.5.jar@4,reference\:file\:com.fasterxml.jackson.datatype.jackson-datatype-joda_2.4.5.jar@4,reference\:file\:com.fasterxml.jackson.jaxrs.jackson-jaxrs-base_2.4.5.jar@4,reference\:file\:com.fasterxml.jackson.jaxrs.jackson-jaxrs-json-provider_2.4.5.jar@4,reference\:file\:com.fasterxml.jackson.module.jackson-module-jaxb-annotations_2.4.5.jar@4,reference\:file\:com.ibm.db2.jcc_1.4.0.jar@4,reference\:file\:com.ibm.nosql_4.19.26.jar@4,reference\:file\:com.microsoft.sqlserver.sqljdbc_6.0.8112.100.jar@4,reference\:file\:com.sun.jna_5.8.0.jar@4,reference\:file\:com.sun.jna.platform_5.8.0.jar@4,reference\:file\:groovy_3.0.3.jar@4,reference\:file\:io.humble.video-arch-x86_64-pc-linux-gnu6_0.2.1.jar@4,reference\:file\:io.humble.video-arch-x86_64-w64-mingw32_0.2.1.jar@4,reference\:file\:io.humble.video-noarch_0.2.1.jar@4,reference\:file\:javax.annotation-api_1.2.0.jar@4,reference\:file\:javax.mail_1.5.0.b01.jar@4,reference\:file\:javax.servlet-api_3.1.0.jar@4,reference\:file\:javax.ws.rs-api_2.0.1.jar@4,reference\:file\:jcifs_1.3.17.jar@4,reference\:file\:joda-time_2.2.0.jar@4,reference\:file\:junit_4.12.0.jar@4,reference\:file\:org.apache.aries.spifly.dynamic.bundle_1.0.8.jar@4,reference\:file\:org.apache.aries.util_1.0.0.jar@4,reference\:file\:org.apache.commons.beanutils_1.8.3.jar@4,reference\:file\:org.apache.commons.collections_3.2.2.jar@4,reference\:file\:org.apache.commons.digester_3.2.0.jar@4,reference\:file\:org.apache.commons.io_2.6.0.jar@4,reference\:file\:org.apache.commons.lang_2.6.0.jar@4,reference\:file\:org.apache.commons.net_3.6.0.jar@4,reference\:file\:org.apache.felix.gogo.command_0.12.0.jar@4,reference\:file\:org.apache.felix.gogo.runtime_0.12.0.jar@4,reference\:file\:org.apache.felix.gogo.shell_0.12.0.jar@4,reference\:file\:org.apache.httpcomponents.httpclient_4.2.6.jar@4,reference\:file\:org.apache.httpcomponents.httpcore_4.2.5.jar@4,reference\:file\:org.apache.logging.log4j.api_2.8.2.jar@1\:start,reference\:file\:org.apache.logging.log4j.core_2.8.2.jar@4,reference\:file\:org.apache.logging.log4j.slf4j-impl_2.8.2.jar@4,reference\:file\:org.apache.servicemix.bundles.quartz_2.3.0.2.jar@4,reference\:file\:org.eclipse.equinox.common_3.8.0.v20160509-1230.jar@2\:start,reference\:file\:org.eclipse.equinox.console_1.1.200.v20150929-1405.jar@4,reference\:file\:org.eclipse.equinox.ds_1.4.400.v20160226-2036.jar@1\:start,reference\:file\:org.eclipse.equinox.launcher_1.3.201.v20161025-1711.jar@4,reference\:file\:org.eclipse.equinox.util_1.0.500.v20130404-1337.jar@4,reference\:file\:org.eclipse.jetty.client_9.3.9.v20160517.jar@4,reference\:file\:org.eclipse.jetty.deploy_9.3.9.v20160517.jar@4,reference\:file\:org.eclipse.jetty.http_9.3.9.v20160517.jar@4,reference\:file\:org.eclipse.jetty.io_9.3.9.v20160517.jar@4,reference\:file\:org.eclipse.jetty.osgi.boot_9.3.9.v20160517.jar@4\:start,reference\:file\:org.eclipse.jetty.schemas_3.1.0.jar@4,reference\:file\:org.eclipse.jetty.security_9.3.9.v20160517.jar@4,reference\:file\:org.eclipse.jetty.server_9.3.9.v20160517.jar@4,reference\:file\:org.eclipse.jetty.servlet_9.3.9.v20160517.jar@4,reference\:file\:org.eclipse.jetty.util_9.3.9.v20160517.jar@4,reference\:file\:org.eclipse.jetty.webapp_9.3.9.v20160517.jar@4,reference\:file\:org.eclipse.jetty.websocket.api_9.3.9.v20160517.jar@4,reference\:file\:org.eclipse.jetty.websocket.client_9.3.9.v20160517.jar@4,reference\:file\:org.eclipse.jetty.websocket.common_9.3.9.v20160517.jar@4,reference\:file\:org.eclipse.jetty.websocket.server_9.3.9.v20160517.jar@4,reference\:file\:org.eclipse.jetty.websocket.servlet_9.3.9.v20160517.jar@4,reference\:file\:org.eclipse.jetty.xml_9.3.9.v20160517.jar@4,reference\:file\:org.eclipse.osgi.services_3.2.100.v20100503.jar@4,reference\:file\:org.hamcrest.core_1.3.0.jar@4,reference\:file\:org.jboss.resteasy.client_3.0.11.Final.jar@4,reference\:file\:org.jboss.resteasy.jaxb-provider_3.0.11.Final.jar@4,reference\:file\:org.jboss.resteasy.jaxrs_3.0.11.Final.jar@4,reference\:file\:org.jboss.resteasy.jaxrs-api_3.0.11.Final.jar@4,reference\:file\:org.jmock_2.6.0.jar@4,reference\:file\:org.jmock.junit4_2.6.0.jar@4,reference\:file\:org.mybatis.mybatis_3.5.2.jar@4,reference\:file\:org.objectweb.asm_5.0.1.jar@4,reference\:file\:org.objectweb.asm.commons_5.0.1.jar@4,reference\:file\:org.objectweb.asm.tree_5.0.1.jar@4,reference\:file\:org.omnifaces_3.4.1.jar@4,reference\:file\:slf4j.api_1.7.24.jar@4,reference\:file\:slf4j.simple_1.7.2.jar@4,reference\:file\:user.commons.log4j2_1.0.0.jar@4,reference\:file\:user.commons.zk_8.0.3.jar@4,reference\:file\:user.jobengine.osgi.commons_1.0.0.jar@4\:start,reference\:file\:user.jobengine.osgi.db_1.0.0.jar@4,reference\:file\:user.jobengine.osgi.server_1.0.0.jar@4,reference\:file\:user.jobengine.osgi.services_1.0.0.jar@4,reference\:file\:user.mediacube.gui_1.0.0.jar@4,reference\:file\:user.mediacube.metadata_1.0.0.jar@4,reference\:file\:user.tsm.client_1.2.0.jar@4 +equinox.use.ds=true +osgi.bundles.defaultStartLevel=4 +osgi.framework=file\:plugins/org.eclipse.osgi_3.11.2.v20161107-1947.jar +osgi.framework.extensions= +eclipse.p2.data.area=@config.dir/../p2 +eclipse.p2.profile=DefaultProfile diff --git a/server/hu.user.mediacube.executors.tests/META-INF/MANIFEST.MF b/server/hu.user.mediacube.executors.tests/META-INF/MANIFEST.MF index 4bf9bf02..db99a328 100644 --- a/server/hu.user.mediacube.executors.tests/META-INF/MANIFEST.MF +++ b/server/hu.user.mediacube.executors.tests/META-INF/MANIFEST.MF @@ -9,6 +9,7 @@ Import-Package: org.apache.commons.io.filefilter;version="2.2.0", org.apache.commons.io.output;version="2.2.0", org.apache.commons.logging, org.apache.commons.logging.impl, + org.apache.commons.net.ftp;version="3.6.0", org.apache.ibatis.jdbc;version="3.5.2", org.codehaus.groovy.control;version="3.0.3", org.junit diff --git a/server/hu.user.mediacube.executors.tests/src/hu/user/mediacube/executors/tests/Support.java b/server/hu.user.mediacube.executors.tests/src/hu/user/mediacube/executors/tests/Support.java index 1797e98c..58d2470d 100644 --- a/server/hu.user.mediacube.executors.tests/src/hu/user/mediacube/executors/tests/Support.java +++ b/server/hu.user.mediacube.executors.tests/src/hu/user/mediacube/executors/tests/Support.java @@ -29,7 +29,10 @@ import com.ibm.nosql.json.api.DBCursor; import com.ibm.nosql.json.api.QueryBuilder; import user.commons.ListUtils; +import user.commons.RemoteFile; +import user.commons.StoreUri; import user.commons.nosql.NoSQLUtils; +import user.commons.remotestore.RemoteStoreProtocol; import user.jobengine.db.IItemManager; import user.jobengine.db.Item; import user.jobengine.db.ItemManager; @@ -284,6 +287,20 @@ public class Support { manager.remove(i); } + @Test + public void testFTP() throws Exception { + + StoreUri sourceUri = manager.createStoreUri(RemoteStoreProtocol.FTP, "10.10.1.55"); + sourceUri.setPortNumber(2098); + sourceUri.setUserName("administrator"); + sourceUri.setPassword("system"); + List remoteFiles = sourceUri.getRemoteFiles(); + RemoteFile remoteFile = sourceUri.getRemoteFile("210506_0000_HIRADO.mxf"); + + assertNotNull(remoteFile); + + } + @Test public void testLoadMVExclude() throws Exception { DB db = NoSQLUtils.getNoSQLDB(); diff --git a/server/user.jobengine.executors/src/user/jobengine/server/steps/CancelableStep.java b/server/user.jobengine.executors/src/user/jobengine/server/steps/CancelableStep.java index 287a2650..164e552c 100644 --- a/server/user.jobengine.executors/src/user/jobengine/server/steps/CancelableStep.java +++ b/server/user.jobengine.executors/src/user/jobengine/server/steps/CancelableStep.java @@ -13,7 +13,7 @@ public class CancelableStep extends JobStep { logger.info(getMarker(), "Executing {}", i); if (getJobRuntime().isWaitingCancel()) break; - Thread.sleep(100); + Thread.sleep(1000); int progress = (i + 1) * 100 / 10; setProgress(progress); } diff --git a/server/user.jobengine.executors/src/user/jobengine/server/steps/CreateMissingLowresStep.java b/server/user.jobengine.executors/src/user/jobengine/server/steps/CreateMissingLowresStep.java index 19bad479..0767e726 100644 --- a/server/user.jobengine.executors/src/user/jobengine/server/steps/CreateMissingLowresStep.java +++ b/server/user.jobengine.executors/src/user/jobengine/server/steps/CreateMissingLowresStep.java @@ -22,6 +22,7 @@ public class CreateMissingLowresStep extends JobStep { @StepEntry public Object[] execute(String localHiresPath, IJobEngine jobEngine, IJobRuntime jobRuntime) throws Exception { Object[] result = new Object[] { null, null, "%s", null, 0, true }; + DB db = NoSQLUtils.getNoSQLDB(); DBCollection collection = db.getCollection("missing_lowres"); IItemManager manager = jobEngine.getItemManager(); @@ -56,10 +57,10 @@ public class CreateMissingLowresStep extends JobStep { private Media getFirstUntranscodedMedia(IItemManager manager, DBCollection collection) { Media[] result = new Media[] { null }; //MV - String query = "SELECT mediaid FROM VW_MISSING_PROXY_IDS WHERE HOUSEID like 'M%' or HOUSEID like 'P%' or HOUSEID like 'R%' ORDER BY modified DESC"; + //String query = "SELECT mediaid FROM VW_MISSING_PROXY_IDS WHERE HOUSEID like 'M%' or HOUSEID like 'P%' or HOUSEID like 'R%' ORDER BY modified DESC"; //HTV - //String query = "SELECT mediaid FROM VW_MISSING_PROXY_IDS"; + String query = "SELECT mediaid FROM VW_MISSING_PROXY_IDS"; manager.executeQuery(query, rs -> { try { long mediaId = rs.getLong(1); @@ -69,7 +70,7 @@ public class CreateMissingLowresStep extends JobStep { logger.info(getSessionMarker(), "Checking {}", name); long existing = collection.find(new BasicDBObject("name", name)).count(); if (existing > 0) { - logger.info(getSessionMarker(), "{} is on missing_lowres list", name); + //logger.info(getSessionMarker(), "{} is on missing_lowres list", name); return true; } diff --git a/server/user.jobengine.executors/src/user/jobengine/server/steps/DeleteNEXIOMaterialsStep.java b/server/user.jobengine.executors/src/user/jobengine/server/steps/DeleteNEXIOMaterialsStep.java index cad3d110..d6e06d5d 100644 --- a/server/user.jobengine.executors/src/user/jobengine/server/steps/DeleteNEXIOMaterialsStep.java +++ b/server/user.jobengine.executors/src/user/jobengine/server/steps/DeleteNEXIOMaterialsStep.java @@ -27,7 +27,7 @@ import user.jobengine.server.IJobEngine; import user.jobengine.server.IJobRuntime; public class DeleteNEXIOMaterialsStep extends JobStep { - private static final Logger logger = LogManager.getLogger(); + private static final Logger logger = LogManager.getLogger(DeleteNEXIOMaterialsStep.class); private static final String NEXIOCLIPS = "nexioclips"; private static final String KILLDATE = "killdate"; private static final String LONGNAMEID = "longnameid"; @@ -45,9 +45,9 @@ public class DeleteNEXIOMaterialsStep extends JobStep { return; } sourceUri.delete(remoteFile); - logger.info(jobRuntime.getSessionMarker(), "A '{}' fájl törlése sikeres volt.", remoteFile.getName()); + logger.info(jobRuntime.getSessionMarker(), "A '{}' fájl törlése sikeres volt.", name); } catch (Exception e) { - logger.error(jobRuntime.getSessionMarker(), "A '{}' fájl nem törölhető. A rendszer hibaüzenete: {}", remoteFile.getName(), e.getMessage()); + logger.error(jobRuntime.getSessionMarker(), "A '{}' fájl nem törölhető. A rendszer hibaüzenete: {}", name, e.getMessage()); } } @@ -55,6 +55,7 @@ public class DeleteNEXIOMaterialsStep extends JobStep { public Object[] execute(int port, String userName, String password, String filterAgencies, int gracePeriodDays, boolean notificationOnly, IJobEngine jobEngine, IJobRuntime jobRuntime) throws Exception { this.jobRuntime = jobRuntime; + logger.info("TESZT"); if (StringUtils.isBlank(NEXIO_HOST)) { logger.error(jobRuntime.getSessionMarker(), "A 'nexio.host' rendszer paraméter nem található."); throw new NullPointerException("System is not configured properly, 'nexio.host' startup parameter missing."); @@ -100,7 +101,8 @@ public class DeleteNEXIOMaterialsStep extends JobStep { else delete(name); i++; - setProgress(i * 100 / clips.size()); + int progress = i * 100 / clips.size(); + setProgress(progress); } return null; diff --git a/server/user.jobengine.executors/src/user/jobengine/server/steps/PASAPOOLTransferToStep.java b/server/user.jobengine.executors/src/user/jobengine/server/steps/PASAPOOLTransferToStep.java index 0c69ee3a..fe8ca7e2 100644 --- a/server/user.jobengine.executors/src/user/jobengine/server/steps/PASAPOOLTransferToStep.java +++ b/server/user.jobengine.executors/src/user/jobengine/server/steps/PASAPOOLTransferToStep.java @@ -30,6 +30,7 @@ public class PASAPOOLTransferToStep extends TransferStep { this.sourceStoreUri = sourceStoreUri; this.sourceFileName = sourceFileName; long start = System.currentTimeMillis(); + Object[] result = super.execute(sourceStoreUri, sourceFileName, targetStoreUri, targetFileName); RemoteFile remoteFile = sourceStoreUri.getRemoteFile(targetFileName); if (remoteFile != null && remoteFile.getSize() > 0) { @@ -40,6 +41,7 @@ public class PASAPOOLTransferToStep extends TransferStep { logger.info(getMarker(), "{} size is {}, upload speed was {}/sec", targetFileName, FileSizeUtils.sizeAsString(remoteFile.getSize()), speed); } } + return result; } @@ -77,6 +79,7 @@ public class PASAPOOLTransferToStep extends TransferStep { } catch (Exception e) { logger.error(getSessionMarker(), e.getMessage()); } + return result; } diff --git a/server/user.jobengine.executors/src/user/jobengine/server/steps/SyncOCTOPUSDataStep.java b/server/user.jobengine.executors/src/user/jobengine/server/steps/SyncOCTOPUSDataStep.java index a7a0832f..2560cfb5 100644 --- a/server/user.jobengine.executors/src/user/jobengine/server/steps/SyncOCTOPUSDataStep.java +++ b/server/user.jobengine.executors/src/user/jobengine/server/steps/SyncOCTOPUSDataStep.java @@ -4,33 +4,20 @@ import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import user.commons.remotestore.IProgressEventListener; -import user.commons.remotestore.ProgressEvent; -import user.jobengine.server.IJobEngine; -import user.jobengine.server.IJobRuntime; import user.jobengine.server.steps.shared.OctopusDataMiner; public class SyncOCTOPUSDataStep extends JobStep { private static final Logger logger = LogManager.getLogger(); private IProgressEventListener progressListener; - private IProgressEventListener createListener(final IJobRuntime jobRuntime) { - return new IProgressEventListener() { - @Override - public void progressChanged(ProgressEvent evt) { - //logger.debug("Progress changed to " + evt.getProgress() + "%"); - jobRuntime.incrementProgress(evt.getProgress()); - } - }; - } - @StepEntry - public Object[] execute(boolean includeArchived, IJobEngine jobEngine, final IJobRuntime jobRuntime) throws Exception { + public Object[] execute(boolean includeArchived, String address, String user, String pwd) throws Exception { OctopusDataMiner dataMiner = null; try { - //jobRuntime.incrementProgress(10); - progressListener = createListener(jobRuntime); - dataMiner = new OctopusDataMiner(); - dataMiner.addProgressListener(progressListener); + dataMiner = new OctopusDataMiner(address, user, pwd); + dataMiner.addProgressListener(e -> { + setProgress(e.getProgress()); + }); dataMiner.execute(includeArchived); } catch (Exception e) { logger.error(getMarker(), "Általános folyamat hiba. A rendszer hibaüzenete: {}", e.getMessage()); diff --git a/server/user.jobengine.executors/src/user/jobengine/server/steps/TransferStep.java b/server/user.jobengine.executors/src/user/jobengine/server/steps/TransferStep.java index 67cc025c..c10084fa 100644 --- a/server/user.jobengine.executors/src/user/jobengine/server/steps/TransferStep.java +++ b/server/user.jobengine.executors/src/user/jobengine/server/steps/TransferStep.java @@ -7,6 +7,7 @@ import org.apache.commons.net.ftp.FTPClient; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; +import user.commons.RemoteFile; import user.commons.StoreUri; import user.commons.remotestore.FtpDirectoryLister; import user.commons.remotestore.RemoteStoreProtocol; @@ -16,42 +17,7 @@ public class TransferStep extends JobStep { private static final String DOT_PART = ".part"; private static final Logger logger = LogManager.getLogger(); - @StepEntry - public Object[] execute(StoreUri sourceStoreUri, String sourceFileName, StoreUri targetStoreUri, String targetFileName) throws Exception { - try { - getJobRuntime().setCancelable(false); - - Store sourceStore = getManager().getStore(sourceStoreUri.getStoreId()); - Store targetStore = getManager().getStore(targetStoreUri.getStoreId()); - - String description = String.format("%s -> %s : %s", sourceStore.getName(), targetStore.getName(), sourceFileName); - getJobRuntime().setDescription(description); - - sourceStoreUri.addProgressListener(e -> setProgress(e.getProgress())); - - StoreUri currentTargetStoreUri = getTargetStoreUri(targetStoreUri); - tryCopy(sourceStoreUri, sourceFileName, currentTargetStoreUri, targetFileName); - } catch (Exception e) { - logger.error(getMarker(), "Error in transfer of {} when copying from {} to {}.", sourceFileName, sourceStoreUri, targetStoreUri); - throw e; - } finally { - if (sourceStoreUri != null) - sourceStoreUri.cleanUp(); - if (targetStoreUri != null) - targetStoreUri.cleanUp(); - } - return null; - } - - protected StoreUri getTargetStoreUri(StoreUri targetStoreUri) { - return targetStoreUri; - } - - protected String getTmpExtension() { - return DOT_PART; - } - - private void tryCopy(StoreUri sourceStoreUri, String sourceFileName, StoreUri targetStoreUri, String targetFileName) throws Exception { + private void copy(StoreUri sourceStoreUri, String sourceFileName, StoreUri targetStoreUri, String targetFileName) throws Exception { String currentTargetFileName = targetFileName; boolean renameAfterCopy = false; @@ -97,4 +63,48 @@ public class TransferStep extends JobStep { } } } + + @StepEntry + public Object[] execute(StoreUri sourceStoreUri, String sourceFileName, StoreUri targetStoreUri, String targetFileName) throws Exception { + try { + getJobRuntime().setCancelable(false); + + Store sourceStore = getManager().getStore(sourceStoreUri.getStoreId()); + Store targetStore = getManager().getStore(targetStoreUri.getStoreId()); + + setDescription("{} -> {} : {}", sourceStore.getName(), targetStore.getName(), sourceFileName); + + sourceStoreUri.addProgressListener(e -> setProgress(e.getProgress())); + + StoreUri currentTargetStoreUri = getTargetStoreUri(targetStoreUri); + + try { + RemoteFile remoteFile = currentTargetStoreUri.getRemoteFile(targetFileName); + if (remoteFile != null) { + logger.info("File {} already exists on target {}, skipping transfer", targetFileName, targetStore.getName()); + } + } catch (Exception e) { + } + + copy(sourceStoreUri, sourceFileName, currentTargetStoreUri, targetFileName); + } catch (Exception e) { + logger.error(getMarker(), "Error in transfer of {} when copying from {} to {}.", sourceFileName, sourceStoreUri, targetStoreUri); + throw e; + } finally { + getJobRuntime().setDescription(null); + if (sourceStoreUri != null) + sourceStoreUri.cleanUp(); + if (targetStoreUri != null) + targetStoreUri.cleanUp(); + } + return null; + } + + protected StoreUri getTargetStoreUri(StoreUri targetStoreUri) { + return targetStoreUri; + } + + protected String getTmpExtension() { + return DOT_PART; + } } diff --git a/server/user.jobengine.executors/src/user/jobengine/server/steps/shared/ExternalCommandExecutor.java b/server/user.jobengine.executors/src/user/jobengine/server/steps/shared/ExternalCommandExecutor.java index 34512b96..a34ff7f0 100644 --- a/server/user.jobengine.executors/src/user/jobengine/server/steps/shared/ExternalCommandExecutor.java +++ b/server/user.jobengine.executors/src/user/jobengine/server/steps/shared/ExternalCommandExecutor.java @@ -2,13 +2,6 @@ package user.jobengine.server.steps.shared; import user.commons.configuration.SystemConfiguration; -/* - package user.jobengine.server.steps.IExternalCallback; - package user.jobengine.server.steps.ExternalProfile; - package user.jobengine.server.steps.ExternalProfilesConfig; - package user.jobengine.server.steps.ExternalCommand; - - */ public class ExternalCommandExecutor { public void execute(String profileName, String input, String output, IExternalCallback responseCallBack) throws Exception { diff --git a/server/user.jobengine.executors/src/user/jobengine/server/steps/shared/OctopusDataMiner.java b/server/user.jobengine.executors/src/user/jobengine/server/steps/shared/OctopusDataMiner.java index 8d12f04f..cd4ebc75 100644 --- a/server/user.jobengine.executors/src/user/jobengine/server/steps/shared/OctopusDataMiner.java +++ b/server/user.jobengine.executors/src/user/jobengine/server/steps/shared/OctopusDataMiner.java @@ -32,7 +32,6 @@ import com.ibm.nosql.json.api.DBCursor; import user.commons.CalendarUtils; import user.commons.ListUtils; -import user.commons.configuration.SystemConfiguration; import user.commons.nosql.NoSQLUtils; import user.commons.octopus.IOctopusAPI; import user.commons.remotestore.IProgressEventListener; @@ -53,8 +52,8 @@ public class OctopusDataMiner { private static final String FIELDS_STORYFOLDER = "id,name,modified"; private static final String FIELDS_STORYFOLDER_STORYIDS = "id,stories,Story.id"; private static final String RUNDOWN = "Rundown"; - private static final String OCTOPUS_DEVICE_NAME = "Octopus-Device-Name"; - private static final String OCTOPUS_DEVICE_ID = "Octopus-Device-Id"; + // private static final String OCTOPUS_DEVICE_NAME = "Octopus-Device-Name"; + // private static final String OCTOPUS_DEVICE_ID = "Octopus-Device-Id"; private static final String FIELDS = "fields"; private static final String EXIT = "Exit"; private static final String RESULT = "result"; @@ -65,9 +64,9 @@ public class OctopusDataMiner { private static final String MOSLABEL = "MOS: "; private static final Object STORY = "Story"; - private static String OCTOPUS_API_ADDRESS = SystemConfiguration.getInstance().value("services.octopus.api.address"); - private static String OCTOPUS_API_USER = SystemConfiguration.getInstance().value("services.octopus.api.user"); - private static String OCTOPUS_API_PASSWORD = SystemConfiguration.getInstance().value("services.octopus.api.password"); + // private static String OCTOPUS_API_ADDRESS = SystemConfiguration.getInstance().value("services.octopus.api.address"); + // private static String OCTOPUS_API_USER = SystemConfiguration.getInstance().value("services.octopus.api.user"); + // private static String OCTOPUS_API_PASSWORD = SystemConfiguration.getInstance().value("services.octopus.api.password"); private DB db; private ResteasyWebTarget webTarget; private HashSet storyIDs = new HashSet<>(); @@ -86,10 +85,10 @@ public class OctopusDataMiner { private int currentObjectIndex; private boolean disableWrite = false; - public OctopusDataMiner() { + public OctopusDataMiner(String address, String user, String pwd) { db = NoSQLUtils.getNoSQLDB(); ResteasyClient client = new ResteasyClientBuilder().build(); - webTarget = client.target(OCTOPUS_API_ADDRESS).register(new BasicAuthentication(OCTOPUS_API_USER, OCTOPUS_API_PASSWORD)); + webTarget = client.target(address).register(new BasicAuthentication(user, pwd)); } public void addProgressListener(IProgressEventListener listener) { @@ -440,7 +439,8 @@ public class OctopusDataMiner { private Builder query(String path, String fields) { ResteasyWebTarget target = webTarget.path(path).queryParam(FIELDS, fields); - Builder result = target.request().header(OCTOPUS_DEVICE_ID, OCTOPUS_API_USER).header(OCTOPUS_DEVICE_NAME, OCTOPUS_API_PASSWORD); + //Builder result = target.request().header(OCTOPUS_DEVICE_ID, OCTOPUS_API_USER).header(OCTOPUS_DEVICE_NAME, OCTOPUS_API_PASSWORD); + Builder result = target.request(); return result; } diff --git a/server/user.jobengine.osgi.commons/src/user/commons/StoreUri.java b/server/user.jobengine.osgi.commons/src/user/commons/StoreUri.java index 4bfc04d1..6770706e 100644 --- a/server/user.jobengine.osgi.commons/src/user/commons/StoreUri.java +++ b/server/user.jobengine.osgi.commons/src/user/commons/StoreUri.java @@ -425,7 +425,9 @@ public class StoreUri extends EntityBase implements Serializable { if (remoteFile == null) throw new Exception(String.format("RemoteFile '%s' not exists.", fileName)); logger.info("Transfering RemoteFile {}, size {}", remoteFile.getName(), remoteFile.getSize()); + IDirectoryLister outputLister = RemoteFileHandler.createLister(targetStoreUri); + if (outputLister == null) throw new Exception("OutputLister is null."); diff --git a/server/user.jobengine.osgi.commons/src/user/commons/remotestore/FtpDirectoryLister.java b/server/user.jobengine.osgi.commons/src/user/commons/remotestore/FtpDirectoryLister.java index 7339d006..247762fb 100644 --- a/server/user.jobengine.osgi.commons/src/user/commons/remotestore/FtpDirectoryLister.java +++ b/server/user.jobengine.osgi.commons/src/user/commons/remotestore/FtpDirectoryLister.java @@ -120,7 +120,7 @@ public class FtpDirectoryLister implements IDirectoryLister { ftpClient.enterLocalPassiveMode(); //210510 ha tud, akkor menjen a gyoker konyvtarba ld. AMC ISILON valtas - ftpClient.changeWorkingDirectory("/"); + //ftpClient.changeWorkingDirectory("/"); if (pathElements.length > 1) { for (int i = 1; i < pathElements.length; i++) { diff --git a/server/user.jobengine.osgi.db/migrations/scripts/036_alter_BeabbleBeach_storeuri.sql b/server/user.jobengine.osgi.db/migrations/scripts/036_alter_BeabbleBeach_storeuri.sql new file mode 100644 index 00000000..1f5df25c --- /dev/null +++ b/server/user.jobengine.osgi.db/migrations/scripts/036_alter_BeabbleBeach_storeuri.sql @@ -0,0 +1,52 @@ +-- +-- Copyright 2010-2016 the original author or authors. +-- +-- Licensed under the Apache License, Version 2.0 (the "License"); +-- you may not use this file except in compliance with the License. +-- You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, software +-- distributed under the License is distributed on an "AS IS" BASIS, +-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +-- See the License for the specific language governing permissions and +-- limitations under the License. +-- + +-- // Alter PASA STOREURI change to FTP protocol +-- Migration SQL that makes the change goes here. + +update storeuri set protocol='FTP', URI='isilon.bcplay.local', username='mcube', password='K6NV5aFemp', rootpath='ifs/MEDIA/PasaPool' where storeid in (select id from store where name = 'PASAPOOL'); +@ +update storeuri set protocol='FTP', URI='isilon.bcplay.local', username='mcube', password='K6NV5aFemp', rootpath='ifs/MEDIA/BeachPool' where storeid in (select id from store where name = 'PEABLEBEACH'); +@ +update storeuri set protocol='FTP', URI='isilon.bcplay.local', username='mcube', password='K6NV5aFemp', rootpath='ifs/MEDIA/SelenioPool' where storeid in (select id from store where name = 'SELENIOPOOL'); +@ +update storeuri set protocol='FTP', URI='isilon.bcplay.local', username='mcube', password='K6NV5aFemp', rootpath='ifs/MEDIA/SelenioPool/16_9' where storeid in (select id from store where name = 'SELENIOPOOL_16_9'); +@ +update storeuri set protocol='FTP', URI='isilon.bcplay.local', username='mcube', password='K6NV5aFemp', rootpath='ifs/MEDIA/SelenioPool/16_9/Processiong' where storeid in (select id from store where name = 'SELENIOPOOL_16_9_PROCESSING'); +@ +update storeuri set protocol='FTP', URI='isilon.bcplay.local', username='mcube', password='K6NV5aFemp', rootpath='ifs/MEDIA/SelenioPool/4_3' where storeid in (select id from store where name = 'SELENIOPOOL_4_3'); +@ +update storeuri set protocol='FTP', URI='isilon.bcplay.local', username='mcube', password='K6NV5aFemp', rootpath='ifs/MEDIA/SelenioPool/4_3/Processiong' where storeid in (select id from store where name = 'SELENIOPOOL_4_3_PROCESSING'); +@ + +-- //@UNDO +-- SQL to undo the change goes here. + +update storeuri set protocol='FTP', URI='10.170.100.21', username='10.170.100.21\pbadmin', password='Txhkkmot52', rootpath='MEDIA_FS/PasaPool' where storeid in (select id from store where name = 'PASAPOOL'); +@ +update storeuri set protocol='FTP', URI='10.170.100.21', username='10.170.100.21\pbadmin', password='Txhkkmot52', rootpath='MEDIA_FS/BeachPool' where storeid in (select id from store where name = 'PEABLEBEACH'); +@ +update storeuri set protocol='FTP', URI='10.170.100.21', username='10.170.100.21\pbadmin', password='Txhkkmot52', rootpath='MEDIA_FS/SelenioPool' where storeid in (select id from store where name = 'SELENIOPOOL'); +@ +update storeuri set protocol='FTP', URI='10.170.100.21', username='10.170.100.21\pbadmin', password='Txhkkmot52', rootpath='MEDIA_FS/SelenioPool/16_9' where storeid in (select id from store where name = 'SELENIOPOOL_16_9'); +@ +update storeuri set protocol='FTP', URI='10.170.100.21', username='10.170.100.21\pbadmin', password='Txhkkmot52', rootpath='MEDIA_FS/SelenioPool/16_9/Processiong' where storeid in (select id from store where name = 'SELENIOPOOL_16_9_PROCESSING'); +@ +update storeuri set protocol='FTP', URI='10.170.100.21', username='10.170.100.21\pbadmin', password='Txhkkmot52', rootpath='MEDIA_FS/SelenioPool/4_3' where storeid in (select id from store where name = 'SELENIOPOOL_4_3'); +@ +update storeuri set protocol='FTP', URI='10.170.100.21', username='10.170.100.21\pbadmin', password='Txhkkmot52', rootpath='MEDIA_FS/SelenioPool/4_3/Processiong' where storeid in (select id from store where name = 'SELENIOPOOL_4_3_PROCESSING'); +@ + diff --git a/server/user.jobengine.osgi.db/src/user/jobengine/search/MediaFinder.java b/server/user.jobengine.osgi.db/src/user/jobengine/search/MediaFinder.java index 026fc1ba..99252c36 100644 --- a/server/user.jobengine.osgi.db/src/user/jobengine/search/MediaFinder.java +++ b/server/user.jobengine.osgi.db/src/user/jobengine/search/MediaFinder.java @@ -36,6 +36,7 @@ public class MediaFinder implements IMediaFinder { sql.append("i.title itemtitle,"); sql.append("m.id mediaid,"); sql.append("m.title mediatitle,"); + sql.append("m.archived,"); sql.append("m.creation,"); sql.append("m.length,"); sql.append("m.itemtypeid mediaitemtypeid,"); @@ -79,10 +80,10 @@ public class MediaFinder implements IMediaFinder { } if (options.getFrom() != null) - sql.append(String.format(" AND m.creation > '%s'", df.format(options.getFrom()))); + sql.append(String.format(" AND m.archived > '%s'", df.format(options.getFrom()))); if (options.getTo() != null) - sql.append(String.format(" AND m.creation < '%s'", df.format(options.getTo()))); + sql.append(String.format(" AND m.archived < '%s'", df.format(options.getTo()))); if (options.isMissingProxy()) sql.append(" AND vw_mf.mediafilecount=1"); @@ -187,6 +188,7 @@ public class MediaFinder implements IMediaFinder { media.setTitle(rs.getString("mediatitle")); media.setHouseId(rs.getString("mediahouseid")); media.setCreation(rs.getTimestamp("creation")); + media.setArchived(rs.getTimestamp("archived")); media.setLength(rs.getLong("length")); media.setMediaFilesCount(rs.getInt("mediafilecount")); media.setMediaFilesName(rs.getString("mediafilehouseid")); diff --git a/server/user.jobengine.osgi.server/src/user/jobengine/server/IJobEngineConfiguration.java b/server/user.jobengine.osgi.server/src/user/jobengine/server/IJobEngineConfiguration.java index 58138c02..9c18761e 100644 --- a/server/user.jobengine.osgi.server/src/user/jobengine/server/IJobEngineConfiguration.java +++ b/server/user.jobengine.osgi.server/src/user/jobengine/server/IJobEngineConfiguration.java @@ -33,4 +33,6 @@ public interface IJobEngineConfiguration { String loadTemplateXml(String template); + void resetStepClassLoader(); + } diff --git a/server/user.jobengine.osgi.server/src/user/jobengine/server/JobEngine.java b/server/user.jobengine.osgi.server/src/user/jobengine/server/JobEngine.java index 88e4177a..7857e8e1 100644 --- a/server/user.jobengine.osgi.server/src/user/jobengine/server/JobEngine.java +++ b/server/user.jobengine.osgi.server/src/user/jobengine/server/JobEngine.java @@ -711,6 +711,7 @@ public class JobEngine implements IJobEngine { if (schedulerService != null) schedulerService.shutdown(); + jobEngineConfiguration.resetStepClassLoader(); jobEngineConfiguration.load(this, true); schedulerService = new SchedulerService(this); @@ -812,6 +813,7 @@ public class JobEngine implements IJobEngine { startUpBarrier.reset(); logger.info("JobEngine gracefully stopped"); + jobEngineConfiguration.resetStepClassLoader(); jobEngineConfiguration.load(this, false); vm.start(); @@ -821,6 +823,7 @@ public class JobEngine implements IJobEngine { schedulerService = new SchedulerService(this); schedulerService.startup(); + logger.info("JobEngine gracefully started"); isRunning = true; } diff --git a/server/user.jobengine.osgi.server/src/user/jobengine/server/JobEngineConfiguration.java b/server/user.jobengine.osgi.server/src/user/jobengine/server/JobEngineConfiguration.java index 86d2fdc3..470e617f 100644 --- a/server/user.jobengine.osgi.server/src/user/jobengine/server/JobEngineConfiguration.java +++ b/server/user.jobengine.osgi.server/src/user/jobengine/server/JobEngineConfiguration.java @@ -54,6 +54,7 @@ public class JobEngineConfiguration implements IJobEngineConfiguration { private final List templates = Collections.synchronizedList(new ArrayList()); private final Map schedules = new LinkedHashMap(); private URLClassLoader stepsClassLoader; + private boolean resetStepClassLoader; public synchronized void bindSystemConfiguration(Object service) { if (service instanceof IConfiguration) { @@ -88,6 +89,12 @@ public class JobEngineConfiguration implements IJobEngineConfiguration { if (stepsClassLoader == null) stepsClassLoader = createParentClassLoader(); + else { + if (resetStepClassLoader) { + stepsClassLoader.close(); + stepsClassLoader = createParentClassLoader(); + } + } Class stepClass = null; @@ -108,19 +115,16 @@ public class JobEngineConfiguration implements IJobEngineConfiguration { } private URLClassLoader createParentClassLoader() throws MalformedURLException { + logger.info("Creating URLClassLoader"); try { String stepClassesDir = systemConfig.getConfig(DIR_CLASSES); Path path = Paths.get(stepClassesDir); URL[] urls = { path.toUri().toURL() }; - logger.info("Creating URLClassLoader with {}", this.getClass().getClassLoader()); - // stepsClassLoader = URLClassLoader.newInstance(urls, Thread.currentThread().getContextClassLoader()); stepsClassLoader = URLClassLoader.newInstance(urls, Thread.currentThread().getContextClassLoader()); } catch (FileNotFoundException e) { //nincs jobs/classes mappa URL[] urls = {}; - logger.info("Creating URLClassLoader"); - // stepsClassLoader = URLClassLoader.newInstance(urls, Thread.currentThread().getContextClassLoader()); - stepsClassLoader = URLClassLoader.newInstance(urls, this.getClass().getClassLoader()); + stepsClassLoader = URLClassLoader.newInstance(urls, Thread.currentThread().getContextClassLoader()); } return stepsClassLoader; } @@ -366,4 +370,9 @@ public class JobEngineConfiguration implements IJobEngineConfiguration { } return result; } + + @Override + public void resetStepClassLoader() { + resetStepClassLoader = true; + } } diff --git a/server/user.jobengine.osgi.server/src/user/jobengine/server/JobStepExecutor.java b/server/user.jobengine.osgi.server/src/user/jobengine/server/JobStepExecutor.java index 588bfe59..4a6e602a 100644 --- a/server/user.jobengine.osgi.server/src/user/jobengine/server/JobStepExecutor.java +++ b/server/user.jobengine.osgi.server/src/user/jobengine/server/JobStepExecutor.java @@ -152,11 +152,11 @@ public class JobStepExecutor implements IJobStepExecutor { GroovyClassLoader stepClassLoader = (GroovyClassLoader) cl; stepClassLoader.close(); - URLClassLoader stepParentClassLoader = (URLClassLoader) cl.getParent(); - stepParentClassLoader.close(); + // URLClassLoader stepParentClassLoader = (URLClassLoader) cl.getParent(); + // stepParentClassLoader.close(); } else if (cl instanceof URLClassLoader) { - URLClassLoader stepClassLoader = (URLClassLoader) cl; - stepClassLoader.close(); + // URLClassLoader stepClassLoader = (URLClassLoader) cl; + // stepClassLoader.close(); } } diff --git a/server/user.jobengine.osgi.server/src/user/jobengine/server/scheduler/ScheduledJob.java b/server/user.jobengine.osgi.server/src/user/jobengine/server/scheduler/ScheduledJob.java index 8a1fcef0..b0f23b2e 100644 --- a/server/user.jobengine.osgi.server/src/user/jobengine/server/scheduler/ScheduledJob.java +++ b/server/user.jobengine.osgi.server/src/user/jobengine/server/scheduler/ScheduledJob.java @@ -61,7 +61,7 @@ public class ScheduledJob extends NativeJob { //implements InterruptableJob{ try { realJob.doJob(); } catch (Exception e) { - logger.warn(e.getMessage()); + //logger.warn(e.getMessage()); } } } diff --git a/server/user.mediacube.gui/WEB-INF/web.xml b/server/user.mediacube.gui/WEB-INF/web.xml index 7d84e06c..867e1ac4 100644 --- a/server/user.mediacube.gui/WEB-INF/web.xml +++ b/server/user.mediacube.gui/WEB-INF/web.xml @@ -68,6 +68,7 @@ Everything on the app @@ -77,5 +78,4 @@ CONFIDENTIAL ---> diff --git a/server/user.mediacube.gui/pages/searchitems/results.zul b/server/user.mediacube.gui/pages/searchitems/results.zul index ea00300c..48146dea 100644 --- a/server/user.mediacube.gui/pages/searchitems/results.zul +++ b/server/user.mediacube.gui/pages/searchitems/results.zul @@ -44,7 +44,7 @@