From 2c4c8dcd9f001b427615366b4e083162d6182c61 Mon Sep 17 00:00:00 2001 From: "vasary.daniel" Date: Wed, 14 Apr 2021 09:39:40 +0000 Subject: [PATCH] git-tfs-id: [http://tfs.userrendszerhaz.hu:8080/tfs/DefaultCollection]$/MediaCube;C32207 --- .../run-mediacube-server-hirtv.launch | 2 +- .../run-mediacube-server-local.launch | 119 ++++++ server/-product/.tpignore | 2 + .../production/AMC/configuration/config.ini | 2 +- .../production/AMC/jobs/executors.xml | 1 + .../production/AMC/jobs/schedules.json | 11 +- .../jobs/steps/PASAPOOLTransferToStep.java | 136 ++---- .../AMC/jobs/steps/TransferStep.java | 32 +- .../AMC/jobs/steps/VODTransferToStep.java | 13 +- .../production/HIRTV/configuration/config.ini | 2 +- .../HIRTV/settings/application.yaml | 4 +- .../production/HIRTV/settings/jetty/http.xml | 41 -- .../HIRTV/settings/jetty/jetty-http.xml | 41 -- .../HIRTV/settings/jetty/jetty-https.xml | 29 -- .../settings/jetty/jetty-ssl-context.xml | 29 -- .../HIRTV/settings/jetty/jetty-ssl.xml | 58 --- .../production/HIRTV/settings/jetty/jetty.xml | 129 ------ .../HIRTV/settings/jetty/localhost.jks | Bin 2256 -> 0 bytes .../production/HIRTV/settings/mediacube.json | 139 ------ .../production/HIRTV/settings/mediacube.yaml | 62 +++ .../HIRTV/settings/test-jetty/jetty-http.xml | 41 -- .../HIRTV/settings/test-jetty/jetty-https.xml | 29 -- .../settings/test-jetty/jetty-ssl-context.xml | 29 -- .../HIRTV/settings/test-jetty/jetty-ssl.xml | 58 --- .../HIRTV/settings/test-jetty/jetty.xml | 129 ------ .../HIRTV/settings/test-jetty/localhost.jks | Bin 2256 -> 0 bytes .../LOCAL/configuration/etc/gosh_profile | 1 + .../jobengine/server/steps/EscortFiles.class | Bin 0 -> 14968 bytes .../server/steps/ItemManagerExtensions.class | Bin 0 -> 4134 bytes .../jobengine/server/steps/MetadataType.class | Bin 0 -> 1320 bytes .../MetadataTypeDetector$MetadataType.class | Bin 0 -> 1554 bytes .../server/steps/MetadataTypeDetector.class | Bin 0 -> 2084 bytes .../server/steps/TSMRestoreStep$1.class | Bin 0 -> 1303 bytes .../server/steps/TSMRestoreStep$2.class | Bin 0 -> 1187 bytes .../server/steps/TSMRestoreStep.class | Bin 0 -> 9183 bytes .../production/LOCAL/jobs/executors.xml | 25 ++ .../production/LOCAL/jobs/schedules.json | 192 +++++++++ .../jobs/steps/ArchiveListBuilderStep.java | 167 ++++++++ .../jobs/steps/ArchiveMaterialSubmitStep.java | 53 +++ .../LOCAL/jobs/steps/ArchiveRecursive.java | 291 +++++++++++++ .../jobs/steps/BatchRetrieveForkStep.java | 76 ++++ .../LOCAL/jobs/steps/CancelableStep.java | 46 ++ .../steps/CleanupMountedLocationStep.java | 316 ++++++++++++++ .../jobs/steps/CreateArchiveItemStep.java | 45 ++ .../jobs/steps/CreateMissingLowresStep.java | 84 ++++ .../LOCAL/jobs/steps/FileCopyStep.java | 95 +++++ .../jobs/steps/FileSearchFilterOptions.java | 41 ++ .../LOCAL/jobs/steps/HSMMigrateStep.java | 401 ++++++++++++++++++ .../LOCAL/jobs/steps/MXFCutterStep.java | 129 ++++++ .../LOCAL/jobs/steps/MediaToolStep.java | 41 ++ .../jobs/steps/MetadataTransformStep.java | 168 ++++++++ .../steps/OutputPathAndNameSelectorStep.java | 151 +++++++ .../jobs/steps/PathItemsCollectorStep.java | 67 +++ .../LOCAL/jobs/steps/TSMBackupStep.java | 207 +++++++++ .../jobs/steps/TSMExtendedRetrieveStep.java | 91 ++++ .../LOCAL/jobs/steps/TSMRestoreStep.java | 159 +++++++ .../jobs/steps/TSMSimpleRestoreStep.java | 57 +++ .../jobs/steps/TestForkCancelableStep.java | 35 ++ .../jobs/steps/TranscodeFFAStranStep.java | 151 +++++++ .../jobs/steps/UpdateGhostMediaDataStep.java | 88 ++++ .../LOCAL/jobs/templates/archive-limited.xml | 41 ++ .../LOCAL/jobs/templates/archive-material.xml | 49 +++ .../LOCAL/jobs/templates/archive-ondemand.xml | 41 ++ .../jobs/templates/archive-recursive.xml | 25 ++ .../templates/batch-retrieve-ondemand.xml | 29 ++ .../LOCAL/jobs/templates/cancelable.xml | 17 + .../LOCAL/jobs/templates/common-copy.xml | 41 ++ .../jobs/templates/create-lowres-ondemand.xml | 106 +++++ .../LOCAL/jobs/templates/delete-materials.xml | 17 + .../LOCAL/jobs/templates/migrate-hsm.xml | 21 + .../jobs/templates/retrieve-ondemand.xml | 150 +++++++ .../LOCAL/jobs/templates/retrieve.xml | 21 + .../LOCAL/jobs/templates/sync-subtitles.xml | 21 + .../jobs/templates/sys-recreate-lowres.xml | 118 ++++++ .../LOCAL/jobs/templates/template1.yaml | 28 ++ .../jobs/templates/test-fork-cancelable.xml | 6 + .../-product/production/LOCAL/mediacube.bat | 17 + .../LOCAL/settings/application.yaml | 51 +++ .../production/LOCAL/settings/dsm.opt | 3 + .../production/LOCAL/settings/ffmpeg.yaml | 14 + .../production/LOCAL/settings/jetty.xml | 45 ++ .../production/LOCAL/settings/log4j2.xml | 103 +++++ .../production/LOCAL/settings/maestro.yaml | 38 ++ .../production/LOCAL/settings/mediacube.yaml | 53 +++ .../MEDIAVIVANTIS/configuration/config.ini | 2 +- .../production/MEDIAVIVANTIS/jobs/dsm.opt | 3 - .../jobs/steps/CreateMissingLowresStep.java | 20 +- .../jobs/steps/TSMExtendedRetrieveStep.java | 14 +- .../production/MEDIAVIVANTIS/mediacube.sh | 24 +- .../MEDIAVIVANTIS/settings/application.yaml | 2 +- .../production/MEDIAVIVANTIS/settings/dsm.opt | 3 + server/RemoteSystemsTempFiles/.project | 12 + .../executors/tests/data/TestChild.java | 10 + .../executors/tests/data/TestParent.java | 8 + .../jobs/ffmpeg.yaml | 14 + .../mediacube/executors/tests/SmallTests.java | 103 ++++- .../server/steps/FileSearchFilterOptions.java | 41 ++ .../server/steps/PathItemsCollectorStep.java | 66 +++ .../server/steps/CreateMissingLowresStep.java | 21 +- .../user/jobengine/server/steps/FFMpeg.java | 89 ++++ .../server/steps/TSMExtendedRetrieveStep.java | 10 +- .../commons/configuration/IConfiguration.java | 2 - .../configuration/SystemConfiguration.java | 23 +- .../user/commons/nexio/NexioDataMiner.java | 43 +- .../migrations/environments/vm.properties | 83 ++++ .../scripts/!006_first_missing_proxy.sql | 42 -- .../scripts/001_create_structure.sql | 12 +- .../scripts/009_add_related_to_job.sql | 4 +- .../scripts/027_create_vod_store.sql | 4 +- .../028_alter_storeuri_incrase_username.sql | 4 +- .../user/jobengine/server/IJobRuntime.java | 2 + .../src/user/jobengine/server/JobEngine.java | 3 +- .../server/JobEngineConfiguration.java | 64 +-- .../src/user/jobengine/server/JobRuntime.java | 7 +- .../server/scheduler/ScheduledJob.java | 4 +- .../META-INF/MANIFEST.MF | 2 +- .../OSGI-INF/TestJobCommandProvider.xml | 8 + .../osgi/console/TestJobCommandProvider.java | 75 ++++ .../img/outline_theaters_black_18dp.png | Bin 0 -> 126 bytes .../pages/alternate_retrievebatchselector.zul | 4 +- .../user.mediacube.gui/pages/jobselector.zul | 3 +- .../pages/metadatadetails.zul | 195 +++++---- .../pages/retrievebatchselector.zul | 4 +- .../user.mediacube.gui/pages/searchitems.zul | 4 +- .../pages/searchitems/details.zul | 32 +- .../pages/searchitems/filters.zul | 10 +- .../user/jobengine/zk/model/MenuModel.java | 2 +- .../zk/model/MetadataDetailsModel.java | 23 +- .../user/jobengine/zk/model/SearchModel.java | 30 +- .../user/jobengine/zk/util/SessionUtil.java | 12 + .../src/user/tsm/client/TSMClient.java | 7 + 131 files changed, 5347 insertions(+), 1173 deletions(-) create mode 100644 server/-configuration/run-mediacube-server-local.launch delete mode 100644 server/-product/production/HIRTV/settings/jetty/http.xml delete mode 100644 server/-product/production/HIRTV/settings/jetty/jetty-http.xml delete mode 100644 server/-product/production/HIRTV/settings/jetty/jetty-https.xml delete mode 100644 server/-product/production/HIRTV/settings/jetty/jetty-ssl-context.xml delete mode 100644 server/-product/production/HIRTV/settings/jetty/jetty-ssl.xml delete mode 100644 server/-product/production/HIRTV/settings/jetty/jetty.xml delete mode 100644 server/-product/production/HIRTV/settings/jetty/localhost.jks delete mode 100644 server/-product/production/HIRTV/settings/mediacube.json create mode 100644 server/-product/production/HIRTV/settings/mediacube.yaml delete mode 100644 server/-product/production/HIRTV/settings/test-jetty/jetty-http.xml delete mode 100644 server/-product/production/HIRTV/settings/test-jetty/jetty-https.xml delete mode 100644 server/-product/production/HIRTV/settings/test-jetty/jetty-ssl-context.xml delete mode 100644 server/-product/production/HIRTV/settings/test-jetty/jetty-ssl.xml delete mode 100644 server/-product/production/HIRTV/settings/test-jetty/jetty.xml delete mode 100644 server/-product/production/HIRTV/settings/test-jetty/localhost.jks create mode 100644 server/-product/production/LOCAL/configuration/etc/gosh_profile create mode 100644 server/-product/production/LOCAL/jobs/classes/user/jobengine/server/steps/EscortFiles.class create mode 100644 server/-product/production/LOCAL/jobs/classes/user/jobengine/server/steps/ItemManagerExtensions.class create mode 100644 server/-product/production/LOCAL/jobs/classes/user/jobengine/server/steps/MetadataType.class create mode 100644 server/-product/production/LOCAL/jobs/classes/user/jobengine/server/steps/MetadataTypeDetector$MetadataType.class create mode 100644 server/-product/production/LOCAL/jobs/classes/user/jobengine/server/steps/MetadataTypeDetector.class create mode 100644 server/-product/production/LOCAL/jobs/classes/user/jobengine/server/steps/TSMRestoreStep$1.class create mode 100644 server/-product/production/LOCAL/jobs/classes/user/jobengine/server/steps/TSMRestoreStep$2.class create mode 100644 server/-product/production/LOCAL/jobs/classes/user/jobengine/server/steps/TSMRestoreStep.class create mode 100644 server/-product/production/LOCAL/jobs/executors.xml create mode 100644 server/-product/production/LOCAL/jobs/schedules.json create mode 100644 server/-product/production/LOCAL/jobs/steps/ArchiveListBuilderStep.java create mode 100644 server/-product/production/LOCAL/jobs/steps/ArchiveMaterialSubmitStep.java create mode 100644 server/-product/production/LOCAL/jobs/steps/ArchiveRecursive.java create mode 100644 server/-product/production/LOCAL/jobs/steps/BatchRetrieveForkStep.java create mode 100644 server/-product/production/LOCAL/jobs/steps/CancelableStep.java create mode 100644 server/-product/production/LOCAL/jobs/steps/CleanupMountedLocationStep.java create mode 100644 server/-product/production/LOCAL/jobs/steps/CreateArchiveItemStep.java create mode 100644 server/-product/production/LOCAL/jobs/steps/CreateMissingLowresStep.java create mode 100644 server/-product/production/LOCAL/jobs/steps/FileCopyStep.java create mode 100644 server/-product/production/LOCAL/jobs/steps/FileSearchFilterOptions.java create mode 100644 server/-product/production/LOCAL/jobs/steps/HSMMigrateStep.java create mode 100644 server/-product/production/LOCAL/jobs/steps/MXFCutterStep.java create mode 100644 server/-product/production/LOCAL/jobs/steps/MediaToolStep.java create mode 100644 server/-product/production/LOCAL/jobs/steps/MetadataTransformStep.java create mode 100644 server/-product/production/LOCAL/jobs/steps/OutputPathAndNameSelectorStep.java create mode 100644 server/-product/production/LOCAL/jobs/steps/PathItemsCollectorStep.java create mode 100644 server/-product/production/LOCAL/jobs/steps/TSMBackupStep.java create mode 100644 server/-product/production/LOCAL/jobs/steps/TSMExtendedRetrieveStep.java create mode 100644 server/-product/production/LOCAL/jobs/steps/TSMRestoreStep.java create mode 100644 server/-product/production/LOCAL/jobs/steps/TSMSimpleRestoreStep.java create mode 100644 server/-product/production/LOCAL/jobs/steps/TestForkCancelableStep.java create mode 100644 server/-product/production/LOCAL/jobs/steps/TranscodeFFAStranStep.java create mode 100644 server/-product/production/LOCAL/jobs/steps/UpdateGhostMediaDataStep.java create mode 100644 server/-product/production/LOCAL/jobs/templates/archive-limited.xml create mode 100644 server/-product/production/LOCAL/jobs/templates/archive-material.xml create mode 100644 server/-product/production/LOCAL/jobs/templates/archive-ondemand.xml create mode 100644 server/-product/production/LOCAL/jobs/templates/archive-recursive.xml create mode 100644 server/-product/production/LOCAL/jobs/templates/batch-retrieve-ondemand.xml create mode 100644 server/-product/production/LOCAL/jobs/templates/cancelable.xml create mode 100644 server/-product/production/LOCAL/jobs/templates/common-copy.xml create mode 100644 server/-product/production/LOCAL/jobs/templates/create-lowres-ondemand.xml create mode 100644 server/-product/production/LOCAL/jobs/templates/delete-materials.xml create mode 100644 server/-product/production/LOCAL/jobs/templates/migrate-hsm.xml create mode 100644 server/-product/production/LOCAL/jobs/templates/retrieve-ondemand.xml create mode 100644 server/-product/production/LOCAL/jobs/templates/retrieve.xml create mode 100644 server/-product/production/LOCAL/jobs/templates/sync-subtitles.xml create mode 100644 server/-product/production/LOCAL/jobs/templates/sys-recreate-lowres.xml create mode 100644 server/-product/production/LOCAL/jobs/templates/template1.yaml create mode 100644 server/-product/production/LOCAL/jobs/templates/test-fork-cancelable.xml create mode 100644 server/-product/production/LOCAL/mediacube.bat create mode 100644 server/-product/production/LOCAL/settings/application.yaml create mode 100644 server/-product/production/LOCAL/settings/dsm.opt create mode 100644 server/-product/production/LOCAL/settings/ffmpeg.yaml create mode 100644 server/-product/production/LOCAL/settings/jetty.xml create mode 100644 server/-product/production/LOCAL/settings/log4j2.xml create mode 100644 server/-product/production/LOCAL/settings/maestro.yaml create mode 100644 server/-product/production/LOCAL/settings/mediacube.yaml delete mode 100644 server/-product/production/MEDIAVIVANTIS/jobs/dsm.opt create mode 100644 server/-product/production/MEDIAVIVANTIS/settings/dsm.opt create mode 100644 server/RemoteSystemsTempFiles/.project create mode 100644 server/hu.user.mediacube.executors.tests/data/hu/user/mediacube/executors/tests/data/TestChild.java create mode 100644 server/hu.user.mediacube.executors.tests/data/hu/user/mediacube/executors/tests/data/TestParent.java create mode 100644 server/hu.user.mediacube.executors.tests/jobs/ffmpeg.yaml create mode 100644 server/user.jobengine.executors/amc/user/jobengine/server/steps/FileSearchFilterOptions.java create mode 100644 server/user.jobengine.executors/amc/user/jobengine/server/steps/PathItemsCollectorStep.java create mode 100644 server/user.jobengine.osgi.db/migrations/environments/vm.properties delete mode 100644 server/user.jobengine.osgi.db/migrations/scripts/!006_first_missing_proxy.sql create mode 100644 server/user.jobengine.osgi.services/OSGI-INF/TestJobCommandProvider.xml create mode 100644 server/user.jobengine.osgi.services/src/user/jobengine/osgi/console/TestJobCommandProvider.java create mode 100644 server/user.mediacube.gui/img/outline_theaters_black_18dp.png diff --git a/server/-configuration/run-mediacube-server-hirtv.launch b/server/-configuration/run-mediacube-server-hirtv.launch index 45f8f1e9..423db869 100644 --- a/server/-configuration/run-mediacube-server-hirtv.launch +++ b/server/-configuration/run-mediacube-server-hirtv.launch @@ -6,7 +6,7 @@ - + diff --git a/server/-configuration/run-mediacube-server-local.launch b/server/-configuration/run-mediacube-server-local.launch new file mode 100644 index 00000000..049fba25 --- /dev/null +++ b/server/-configuration/run-mediacube-server-local.launch @@ -0,0 +1,119 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + diff --git a/server/-product/.tpignore b/server/-product/.tpignore index ac77a1bc..51fcdd4a 100644 --- a/server/-product/.tpignore +++ b/server/-product/.tpignore @@ -9,3 +9,5 @@ \Q/production/HIRTV/tmp/\E.* \Q/production/MEDIAVIVANTIS/log/\E.* \Q/production/AMC/tmp/\E.* +\Q/production/LOCAL/log/\E.* +\Q/production/LOCAL/tmp/\E.* diff --git a/server/-product/production/AMC/configuration/config.ini b/server/-product/production/AMC/configuration/config.ini index fd6901cd..265bf754 100644 --- a/server/-product/production/AMC/configuration/config.ini +++ b/server/-product/production/AMC/configuration/config.ini @@ -1,5 +1,5 @@ #This configuration file was written by: org.eclipse.equinox.internal.frameworkadmin.equinox.EquinoxFwConfigFileParser -#Thu Mar 04 15:26:35 CET 2021 +#Thu Mar 11 15:00:06 CET 2021 osgi.bundles=reference\:file\:cglib_2.2.2.jar@4,reference\:file\:com.fasterxml.jackson.core.jackson-annotations_2.4.5.jar@4,reference\:file\:com.fasterxml.jackson.core.jackson-core_2.4.5.jar@4,reference\:file\:com.fasterxml.jackson.core.jackson-databind_2.4.5.jar@4,reference\:file\:com.fasterxml.jackson.dataformat.jackson-dataformat-yaml_2.4.5.jar@4,reference\:file\:com.fasterxml.jackson.datatype.jackson-datatype-joda_2.4.5.jar@4,reference\:file\:com.fasterxml.jackson.jaxrs.jackson-jaxrs-base_2.4.5.jar@4,reference\:file\:com.fasterxml.jackson.jaxrs.jackson-jaxrs-json-provider_2.4.5.jar@4,reference\:file\:com.fasterxml.jackson.module.jackson-module-jaxb-annotations_2.4.5.jar@4,reference\:file\:com.ibm.db2.jcc_1.4.0.jar@4,reference\:file\:com.ibm.nosql_4.19.26.jar@4,reference\:file\:com.microsoft.sqlserver.sqljdbc_6.0.8112.100.jar@4,reference\:file\:com.sun.jna_4.2.0.jar@4,reference\:file\:groovy_3.0.3.jar@4,reference\:file\:io.humble.video-arch-x86_64-pc-linux-gnu6_0.2.1.jar@4,reference\:file\:io.humble.video-arch-x86_64-w64-mingw32_0.2.1.jar@4,reference\:file\:io.humble.video-noarch_0.2.1.jar@4,reference\:file\:javax.annotation-api_1.2.0.jar@4,reference\:file\:javax.mail_1.5.0.b01.jar@4,reference\:file\:javax.servlet-api_3.1.0.jar@4,reference\:file\:javax.ws.rs-api_2.0.1.jar@4,reference\:file\:jcifs_1.3.17.jar@4,reference\:file\:joda-time_2.2.0.jar@4,reference\:file\:junit_4.12.0.jar@4,reference\:file\:org.apache.aries.spifly.dynamic.bundle_1.0.8.jar@4,reference\:file\:org.apache.aries.util_1.0.0.jar@4,reference\:file\:org.apache.commons.beanutils_1.8.3.jar@4,reference\:file\:org.apache.commons.collections_3.2.2.jar@4,reference\:file\:org.apache.commons.digester_3.2.0.jar@4,reference\:file\:org.apache.commons.io_2.6.0.jar@4,reference\:file\:org.apache.commons.lang_2.6.0.jar@4,reference\:file\:org.apache.commons.net_3.6.0.jar@4,reference\:file\:org.apache.felix.gogo.command_0.12.0.jar@4,reference\:file\:org.apache.felix.gogo.runtime_0.12.0.jar@4,reference\:file\:org.apache.felix.gogo.shell_0.12.0.jar@4,reference\:file\:org.apache.httpcomponents.httpclient_4.2.6.jar@4,reference\:file\:org.apache.httpcomponents.httpcore_4.2.5.jar@4,reference\:file\:org.apache.logging.log4j.api_2.8.2.jar@1\:start,reference\:file\:org.apache.logging.log4j.core_2.8.2.jar@4,reference\:file\:org.apache.logging.log4j.slf4j-impl_2.8.2.jar@4,reference\:file\:org.apache.servicemix.bundles.quartz_2.3.0.2.jar@4,reference\:file\:org.eclipse.equinox.common_3.8.0.v20160509-1230.jar@2\:start,reference\:file\:org.eclipse.equinox.console_1.1.200.v20150929-1405.jar@4,reference\:file\:org.eclipse.equinox.ds_1.4.400.v20160226-2036.jar@1\:start,reference\:file\:org.eclipse.equinox.launcher_1.3.201.v20161025-1711.jar@4,reference\:file\:org.eclipse.equinox.util_1.0.500.v20130404-1337.jar@4,reference\:file\:org.eclipse.jetty.client_9.3.9.v20160517.jar@4,reference\:file\:org.eclipse.jetty.deploy_9.3.9.v20160517.jar@4,reference\:file\:org.eclipse.jetty.http_9.3.9.v20160517.jar@4,reference\:file\:org.eclipse.jetty.io_9.3.9.v20160517.jar@4,reference\:file\:org.eclipse.jetty.osgi.boot_9.3.9.v20160517.jar@4\:start,reference\:file\:org.eclipse.jetty.schemas_3.1.0.jar@4,reference\:file\:org.eclipse.jetty.security_9.3.9.v20160517.jar@4,reference\:file\:org.eclipse.jetty.server_9.3.9.v20160517.jar@4,reference\:file\:org.eclipse.jetty.servlet_9.3.9.v20160517.jar@4,reference\:file\:org.eclipse.jetty.util_9.3.9.v20160517.jar@4,reference\:file\:org.eclipse.jetty.webapp_9.3.9.v20160517.jar@4,reference\:file\:org.eclipse.jetty.websocket.api_9.3.9.v20160517.jar@4,reference\:file\:org.eclipse.jetty.websocket.client_9.3.9.v20160517.jar@4,reference\:file\:org.eclipse.jetty.websocket.common_9.3.9.v20160517.jar@4,reference\:file\:org.eclipse.jetty.websocket.server_9.3.9.v20160517.jar@4,reference\:file\:org.eclipse.jetty.websocket.servlet_9.3.9.v20160517.jar@4,reference\:file\:org.eclipse.jetty.xml_9.3.9.v20160517.jar@4,reference\:file\:org.eclipse.osgi.services_3.2.100.v20100503.jar@4,reference\:file\:org.hamcrest.core_1.3.0.jar@4,reference\:file\:org.jboss.resteasy.client_3.0.11.Final.jar@4,reference\:file\:org.jboss.resteasy.jaxb-provider_3.0.11.Final.jar@4,reference\:file\:org.jboss.resteasy.jaxrs_3.0.11.Final.jar@4,reference\:file\:org.jboss.resteasy.jaxrs-api_3.0.11.Final.jar@4,reference\:file\:org.jmock_2.6.0.jar@4,reference\:file\:org.jmock.junit4_2.6.0.jar@4,reference\:file\:org.mybatis.mybatis_3.5.2.jar@4,reference\:file\:org.objectweb.asm_5.0.1.jar@4,reference\:file\:org.objectweb.asm.commons_5.0.1.jar@4,reference\:file\:org.objectweb.asm.tree_5.0.1.jar@4,reference\:file\:org.omnifaces_3.4.1.jar@4,reference\:file\:slf4j.api_1.7.24.jar@4,reference\:file\:slf4j.simple_1.7.2.jar@4,reference\:file\:user.commons.log4j2_1.0.0.jar@4,reference\:file\:user.commons.zk_8.0.3.jar@4,reference\:file\:user.jobengine.osgi.commons_1.0.0.jar@4\:start,reference\:file\:user.jobengine.osgi.db_1.0.0.jar@4,reference\:file\:user.jobengine.osgi.server_1.0.0.jar@4,reference\:file\:user.jobengine.osgi.services_1.0.0.jar@4,reference\:file\:user.mediacube.gui_1.0.0.jar@4,reference\:file\:user.mediacube.metadata_1.0.0.jar@4,reference\:file\:user.tsm.client_1.2.0.jar@4 equinox.use.ds=true osgi.bundles.defaultStartLevel=4 diff --git a/server/-product/production/AMC/jobs/executors.xml b/server/-product/production/AMC/jobs/executors.xml index 0f868e57..17e61057 100644 --- a/server/-product/production/AMC/jobs/executors.xml +++ b/server/-product/production/AMC/jobs/executors.xml @@ -1,5 +1,6 @@ + diff --git a/server/-product/production/AMC/jobs/schedules.json b/server/-product/production/AMC/jobs/schedules.json index a6bb1e5d..622f4da7 100644 --- a/server/-product/production/AMC/jobs/schedules.json +++ b/server/-product/production/AMC/jobs/schedules.json @@ -1,11 +1,10 @@ {"joblist":[ { - "template": "test-fork-cancelable.xml", - "parameters": [ {"name": "itemID", "value": 1, "type": "java.lang.Long"} ] - }, - { - "template": "validate-dir-mxf.xml", - "parameters": [ {"name": "source", "value": "h:/TEMP/", "type": "java.lang.String"} ] + "template": "server-status-report.xml", + "name" : "MediaCube státusz", + "active": true, + "executeimmediate": false, + "cronexpression": "0 */30 * * * ?" }, { "template": "nexio1-archive-checker.xml", diff --git a/server/-product/production/AMC/jobs/steps/PASAPOOLTransferToStep.java b/server/-product/production/AMC/jobs/steps/PASAPOOLTransferToStep.java index dfe2825d..0c69ee3a 100644 --- a/server/-product/production/AMC/jobs/steps/PASAPOOLTransferToStep.java +++ b/server/-product/production/AMC/jobs/steps/PASAPOOLTransferToStep.java @@ -3,7 +3,6 @@ package user.jobengine.server.steps; import java.nio.file.Path; import java.nio.file.Paths; -import org.apache.commons.io.FileUtils; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.apache.logging.log4j.Marker; @@ -11,7 +10,8 @@ import org.apache.logging.log4j.MarkerManager; import user.commons.RemoteFile; import user.commons.StoreUri; -import user.commons.mediatool.MediaInfo; +import user.commons.mediaarea.MediaArea; +import user.commons.strings.FileSizeUtils; public class PASAPOOLTransferToStep extends TransferStep { private static final Logger logger = LogManager.getLogger(); @@ -22,13 +22,13 @@ public class PASAPOOLTransferToStep extends TransferStep { private static final String DISPLAY_ASPECT_4_3 = "4:3"; private StoreUri sourceStoreUri; private String sourceFileName; + private Marker marker = MarkerManager.getMarker("MEDIAPROFILE"); @Override @StepEntry public Object[] execute(StoreUri sourceStoreUri, String sourceFileName, StoreUri targetStoreUri, String targetFileName) throws Exception { this.sourceStoreUri = sourceStoreUri; this.sourceFileName = sourceFileName; - logMediaProfile(); long start = System.currentTimeMillis(); Object[] result = super.execute(sourceStoreUri, sourceFileName, targetStoreUri, targetFileName); RemoteFile remoteFile = sourceStoreUri.getRemoteFile(targetFileName); @@ -36,119 +36,67 @@ public class PASAPOOLTransferToStep extends TransferStep { long diff = (System.currentTimeMillis() - start) / 1000; if (diff > 0) { long bytesSpeed = remoteFile.getSize() / diff; - String speed = FileUtils.byteCountToDisplaySize(bytesSpeed); - logger.info(getMarker(), "{} size is {}, upload speed was {}/sec", targetFileName, FileUtils.byteCountToDisplaySize(remoteFile.getSize()), speed); + String speed = FileSizeUtils.sizeAsString(bytesSpeed); + logger.info(getMarker(), "{} size is {}, upload speed was {}/sec", targetFileName, FileSizeUtils.sizeAsString(remoteFile.getSize()), speed); } } return result; } - // HD - // height = 1080 - // >PEABLEBEACH | \\10.170.100.21\media\BeachPool - private boolean isHD(MediaInfo mi) { - return mi.getHeight() == 1080; - } - - // @Override - // protected StoreUri getTargetStoreUri() { - // StoreUri result = null; - // try { - // Path filePath = Paths.get(sourceStoreUri.toString(true), sourceFileName); - // MediaInfo mi = new MediaInfo(filePath); - // mi.process(); - // - // if (isHD(mi)) { - // result = getManager().getStoreUri("PEABLEBEACH", targetStoreUri.getProtocol()); - // return result; - // } - // - // if (isSD_HIGH_16_9(mi)) { - // result = getManager().getStoreUri("SELENIOPOOL_16_9", targetStoreUri.getProtocol()); - // return result; - // } - // - // if (isSD_HIGH_4_3(mi)) { - // result = getManager().getStoreUri("SELENIOPOOL_4_3", targetStoreUri.getProtocol()); - // return result; - // } - // - // if (isSD_MAIN_422(mi)) { - // result = getManager().getStoreUri("PEABLEBEACH", targetStoreUri.getProtocol()); - // return result; - // } - // } catch (Exception e) { - // logger.error(getSessionMarker(), e.getMessage()); - // result = targetStoreUri; - // } - // return result; - // } - - // SD - HIGH@HIGH / 16:9 - // height < 650 - // profile=High - // aspect=16:9 - // >SELENIOPOOL_16_9 | \\10.170.100.21\media\SelenioPool\16_9 - private boolean isSD_HIGH_16_9(MediaInfo mi) { - String codecProfileName = mi.getCodecProfileName(); - if (codecProfileName != null) - codecProfileName = codecProfileName.toUpperCase(); - return mi.getHeight() < 650 && CODEC_PROFILE_HIGH.equals(codecProfileName) && DISPLAY_ASPECT_16_9.equals(mi.getDisplayAspect()); - } - - // SD - HIGH@HIGH / 4:3 - // height < 650 - // profile=High - // aspect=4:3 - // >SELENIOPOOL_4_3 | \\10.170.100.21\media\SelenioPool\4_3 - private boolean isSD_HIGH_4_3(MediaInfo mi) { - String codecProfileName = mi.getCodecProfileName(); - if (codecProfileName != null) - codecProfileName = codecProfileName.toUpperCase(); - return mi.getHeight() < 650 && CODEC_PROFILE_HIGH.equals(codecProfileName) && DISPLAY_ASPECT_4_3.equals(mi.getDisplayAspect()); - } - - // SD - MAIN/422@HIGH - // height < 650 - // profile=Main || profile=4:2:2 - // >PEABLEBEACH | \\10.170.100.21\media\BeachPool - private boolean isSD_MAIN_422(MediaInfo mi) { - String codecProfileName = mi.getCodecProfileName(); - if (codecProfileName != null) - codecProfileName = codecProfileName.toUpperCase(); - return mi.getHeight() < 650 && (CODEC_PROFILE_MAIN.equals(codecProfileName) || CODEC_PROFILE_422.equals(codecProfileName)); - } - - protected void logMediaProfile() { - Marker marker = MarkerManager.getMarker("MEDIAPROFILE"); + @Override + protected StoreUri getTargetStoreUri(StoreUri targetStoreUri) { + StoreUri result = targetStoreUri; try { Path filePath = Paths.get(sourceStoreUri.toString(true), sourceFileName); - MediaInfo mi = new MediaInfo(filePath); + MediaArea mi = new MediaArea(filePath); mi.process(); - logger.info(marker, "{} data : {} {} {}", sourceFileName, mi.getCodecProfileName(), mi.getHeight(), mi.getDisplayAspect()); if (isHD(mi)) { - logger.info(marker, "{} is HD", sourceFileName); - return; + logger.info(marker, "{} is MOVED_HD", sourceFileName); + result = getManager().getStoreUri("PEABLEBEACH", targetStoreUri.getProtocol()); + return result; } if (isSD_HIGH_16_9(mi)) { - logger.info(marker, "{} is SD_HIGH_16_9", sourceFileName); - return; + logger.info(marker, "{} is CONVSD16_9", sourceFileName); + result = getManager().getStoreUri("SELENIOPOOL_16_9", targetStoreUri.getProtocol()); + return result; } if (isSD_HIGH_4_3(mi)) { - logger.info(marker, "{} is SD_HIGH_4_3", sourceFileName); - return; + logger.info(marker, "{} is CONVSD4_3", sourceFileName); + result = getManager().getStoreUri("SELENIOPOOL_4_3", targetStoreUri.getProtocol()); + return result; } if (isSD_MAIN_422(mi)) { - logger.info(marker, "{} is SD_MAIN_422", sourceFileName); - return; + logger.info(marker, "{} is MOVED_SD", sourceFileName); + result = getManager().getStoreUri("PEABLEBEACH", targetStoreUri.getProtocol()); + return result; } } catch (Exception e) { - logger.error(marker, e.getMessage()); + logger.error(getSessionMarker(), e.getMessage()); } + return result; + } + + private boolean isHD(MediaArea mi) { + return mi.getHeight() == 1080; + } + + private boolean isSD_HIGH_16_9(MediaArea mi) { + String formatProfileName = mi.getFormatProfileName(); + return mi.getHeight() < 650 && formatProfileName.contains(CODEC_PROFILE_HIGH) && DISPLAY_ASPECT_16_9.equals(mi.getDisplayAspect()); + } + + private boolean isSD_HIGH_4_3(MediaArea mi) { + String formatProfileName = mi.getFormatProfileName(); + return mi.getHeight() < 650 && formatProfileName.contains(CODEC_PROFILE_HIGH) && DISPLAY_ASPECT_4_3.equals(mi.getDisplayAspect()); + } + + private boolean isSD_MAIN_422(MediaArea mi) { + String formatProfileName = mi.getFormatProfileName(); + return mi.getHeight() < 650 && (formatProfileName.contains(CODEC_PROFILE_MAIN) || formatProfileName.contains(CODEC_PROFILE_422)); } } diff --git a/server/-product/production/AMC/jobs/steps/TransferStep.java b/server/-product/production/AMC/jobs/steps/TransferStep.java index bdf02615..67cc025c 100644 --- a/server/-product/production/AMC/jobs/steps/TransferStep.java +++ b/server/-product/production/AMC/jobs/steps/TransferStep.java @@ -15,13 +15,10 @@ import user.jobengine.db.Store; public class TransferStep extends JobStep { private static final String DOT_PART = ".part"; private static final Logger logger = LogManager.getLogger(); - private static final boolean simulateTransferToTSM = Boolean.parseBoolean(System.getProperty("test.simulate.transfer.tsm", "false")); - protected StoreUri targetStoreUri; @StepEntry public Object[] execute(StoreUri sourceStoreUri, String sourceFileName, StoreUri targetStoreUri, String targetFileName) throws Exception { try { - this.targetStoreUri = targetStoreUri; getJobRuntime().setCancelable(false); Store sourceStore = getManager().getStore(sourceStoreUri.getStoreId()); @@ -30,30 +27,13 @@ public class TransferStep extends JobStep { String description = String.format("%s -> %s : %s", sourceStore.getName(), targetStore.getName(), sourceFileName); getJobRuntime().setDescription(description); - //logger.info(getSessionMarker(), String.valueOf(isSimulateTSMWrite(sourceStoreUri, targetStoreUri))); - if (isSimulateTSMWrite(sourceStoreUri, targetStoreUri)) { - logger.info(getMarker(), "Skipping real TSM write of {} from {} to {}", sourceFileName, sourceStoreUri, targetStoreUri); - return null; - } - sourceStoreUri.addProgressListener(e -> setProgress(e.getProgress())); - this.targetStoreUri = getTargetStoreUri(); - tryCopy(sourceStoreUri, sourceFileName, targetStoreUri, targetFileName); + StoreUri currentTargetStoreUri = getTargetStoreUri(targetStoreUri); + tryCopy(sourceStoreUri, sourceFileName, currentTargetStoreUri, targetFileName); } catch (Exception e) { logger.error(getMarker(), "Error in transfer of {} when copying from {} to {}.", sourceFileName, sourceStoreUri, targetStoreUri); throw e; - // logger.error(getMarker(), "Error in transfer of {} when copying from {} to {}. Retrying after 3 seconds.", sourceFileName, sourceStoreUri, - // targetStoreUri); - - // try { - // Thread.sleep(3000); - // tryCopy(sourceStoreUri, sourceFileName, targetStoreUri, targetFileName); - // } catch (Exception e1) { - // logger.error(getSessionMarker(), "Error in transfer of {} when copying from {} to {}. System message is: {}", sourceFileName, sourceStoreUri, - // targetStoreUri, e1.getMessage()); - // throw e1; - // } } finally { if (sourceStoreUri != null) sourceStoreUri.cleanUp(); @@ -63,7 +43,7 @@ public class TransferStep extends JobStep { return null; } - protected StoreUri getTargetStoreUri() { + protected StoreUri getTargetStoreUri(StoreUri targetStoreUri) { return targetStoreUri; } @@ -71,10 +51,6 @@ public class TransferStep extends JobStep { return DOT_PART; } - private boolean isSimulateTSMWrite(StoreUri sourceStoreUri, StoreUri targetStoreUri) { - return simulateTransferToTSM && (RemoteStoreProtocol.TSM.equals(targetStoreUri.getProtocol())); - } - private void tryCopy(StoreUri sourceStoreUri, String sourceFileName, StoreUri targetStoreUri, String targetFileName) throws Exception { String currentTargetFileName = targetFileName; @@ -96,7 +72,7 @@ public class TransferStep extends JobStep { sourceStoreUri.transferFrom(targetStoreUri, sourceFileName, currentTargetFileName); - logger.info(getMarker(), "Transfer completed from {} as {} to {} as {}", sourceFileName, sourceStoreUri, targetStoreUri, currentTargetFileName); + logger.info(getMarker(), "Transfer of {} completed from {} to {}", sourceFileName, sourceStoreUri, currentTargetFileName); if (renameAfterCopy) { Path tmpTargetFile = Paths.get(targetStoreUri.toString(true), currentTargetFileName); diff --git a/server/-product/production/AMC/jobs/steps/VODTransferToStep.java b/server/-product/production/AMC/jobs/steps/VODTransferToStep.java index 7bf240fd..d17eaa1e 100644 --- a/server/-product/production/AMC/jobs/steps/VODTransferToStep.java +++ b/server/-product/production/AMC/jobs/steps/VODTransferToStep.java @@ -5,12 +5,6 @@ import user.commons.StoreUri; public class VODTransferToStep extends TransferStep { private String relativeTargetPath; - @Override - protected StoreUri configureTargetUri(StoreUri targetStoreUri) { - targetStoreUri.setRootPath(relativeTargetPath); - return targetStoreUri; - } - @StepEntry public Object[] execute(StoreUri sourceStoreUri, String sourceFileName, StoreUri targetStoreUri, String relativeTargetPath, String targetFileName) throws Exception { @@ -18,4 +12,11 @@ public class VODTransferToStep extends TransferStep { return super.execute(sourceStoreUri, sourceFileName, targetStoreUri, targetFileName); } + @Override + protected StoreUri getTargetStoreUri(StoreUri targetStoreUri) { + StoreUri result = getManager().getStoreUri(targetStoreUri.getId()); + result.setRootPath(relativeTargetPath); + return result; + } + } diff --git a/server/-product/production/HIRTV/configuration/config.ini b/server/-product/production/HIRTV/configuration/config.ini index fd6901cd..265bf754 100644 --- a/server/-product/production/HIRTV/configuration/config.ini +++ b/server/-product/production/HIRTV/configuration/config.ini @@ -1,5 +1,5 @@ #This configuration file was written by: org.eclipse.equinox.internal.frameworkadmin.equinox.EquinoxFwConfigFileParser -#Thu Mar 04 15:26:35 CET 2021 +#Thu Mar 11 15:00:06 CET 2021 osgi.bundles=reference\:file\:cglib_2.2.2.jar@4,reference\:file\:com.fasterxml.jackson.core.jackson-annotations_2.4.5.jar@4,reference\:file\:com.fasterxml.jackson.core.jackson-core_2.4.5.jar@4,reference\:file\:com.fasterxml.jackson.core.jackson-databind_2.4.5.jar@4,reference\:file\:com.fasterxml.jackson.dataformat.jackson-dataformat-yaml_2.4.5.jar@4,reference\:file\:com.fasterxml.jackson.datatype.jackson-datatype-joda_2.4.5.jar@4,reference\:file\:com.fasterxml.jackson.jaxrs.jackson-jaxrs-base_2.4.5.jar@4,reference\:file\:com.fasterxml.jackson.jaxrs.jackson-jaxrs-json-provider_2.4.5.jar@4,reference\:file\:com.fasterxml.jackson.module.jackson-module-jaxb-annotations_2.4.5.jar@4,reference\:file\:com.ibm.db2.jcc_1.4.0.jar@4,reference\:file\:com.ibm.nosql_4.19.26.jar@4,reference\:file\:com.microsoft.sqlserver.sqljdbc_6.0.8112.100.jar@4,reference\:file\:com.sun.jna_4.2.0.jar@4,reference\:file\:groovy_3.0.3.jar@4,reference\:file\:io.humble.video-arch-x86_64-pc-linux-gnu6_0.2.1.jar@4,reference\:file\:io.humble.video-arch-x86_64-w64-mingw32_0.2.1.jar@4,reference\:file\:io.humble.video-noarch_0.2.1.jar@4,reference\:file\:javax.annotation-api_1.2.0.jar@4,reference\:file\:javax.mail_1.5.0.b01.jar@4,reference\:file\:javax.servlet-api_3.1.0.jar@4,reference\:file\:javax.ws.rs-api_2.0.1.jar@4,reference\:file\:jcifs_1.3.17.jar@4,reference\:file\:joda-time_2.2.0.jar@4,reference\:file\:junit_4.12.0.jar@4,reference\:file\:org.apache.aries.spifly.dynamic.bundle_1.0.8.jar@4,reference\:file\:org.apache.aries.util_1.0.0.jar@4,reference\:file\:org.apache.commons.beanutils_1.8.3.jar@4,reference\:file\:org.apache.commons.collections_3.2.2.jar@4,reference\:file\:org.apache.commons.digester_3.2.0.jar@4,reference\:file\:org.apache.commons.io_2.6.0.jar@4,reference\:file\:org.apache.commons.lang_2.6.0.jar@4,reference\:file\:org.apache.commons.net_3.6.0.jar@4,reference\:file\:org.apache.felix.gogo.command_0.12.0.jar@4,reference\:file\:org.apache.felix.gogo.runtime_0.12.0.jar@4,reference\:file\:org.apache.felix.gogo.shell_0.12.0.jar@4,reference\:file\:org.apache.httpcomponents.httpclient_4.2.6.jar@4,reference\:file\:org.apache.httpcomponents.httpcore_4.2.5.jar@4,reference\:file\:org.apache.logging.log4j.api_2.8.2.jar@1\:start,reference\:file\:org.apache.logging.log4j.core_2.8.2.jar@4,reference\:file\:org.apache.logging.log4j.slf4j-impl_2.8.2.jar@4,reference\:file\:org.apache.servicemix.bundles.quartz_2.3.0.2.jar@4,reference\:file\:org.eclipse.equinox.common_3.8.0.v20160509-1230.jar@2\:start,reference\:file\:org.eclipse.equinox.console_1.1.200.v20150929-1405.jar@4,reference\:file\:org.eclipse.equinox.ds_1.4.400.v20160226-2036.jar@1\:start,reference\:file\:org.eclipse.equinox.launcher_1.3.201.v20161025-1711.jar@4,reference\:file\:org.eclipse.equinox.util_1.0.500.v20130404-1337.jar@4,reference\:file\:org.eclipse.jetty.client_9.3.9.v20160517.jar@4,reference\:file\:org.eclipse.jetty.deploy_9.3.9.v20160517.jar@4,reference\:file\:org.eclipse.jetty.http_9.3.9.v20160517.jar@4,reference\:file\:org.eclipse.jetty.io_9.3.9.v20160517.jar@4,reference\:file\:org.eclipse.jetty.osgi.boot_9.3.9.v20160517.jar@4\:start,reference\:file\:org.eclipse.jetty.schemas_3.1.0.jar@4,reference\:file\:org.eclipse.jetty.security_9.3.9.v20160517.jar@4,reference\:file\:org.eclipse.jetty.server_9.3.9.v20160517.jar@4,reference\:file\:org.eclipse.jetty.servlet_9.3.9.v20160517.jar@4,reference\:file\:org.eclipse.jetty.util_9.3.9.v20160517.jar@4,reference\:file\:org.eclipse.jetty.webapp_9.3.9.v20160517.jar@4,reference\:file\:org.eclipse.jetty.websocket.api_9.3.9.v20160517.jar@4,reference\:file\:org.eclipse.jetty.websocket.client_9.3.9.v20160517.jar@4,reference\:file\:org.eclipse.jetty.websocket.common_9.3.9.v20160517.jar@4,reference\:file\:org.eclipse.jetty.websocket.server_9.3.9.v20160517.jar@4,reference\:file\:org.eclipse.jetty.websocket.servlet_9.3.9.v20160517.jar@4,reference\:file\:org.eclipse.jetty.xml_9.3.9.v20160517.jar@4,reference\:file\:org.eclipse.osgi.services_3.2.100.v20100503.jar@4,reference\:file\:org.hamcrest.core_1.3.0.jar@4,reference\:file\:org.jboss.resteasy.client_3.0.11.Final.jar@4,reference\:file\:org.jboss.resteasy.jaxb-provider_3.0.11.Final.jar@4,reference\:file\:org.jboss.resteasy.jaxrs_3.0.11.Final.jar@4,reference\:file\:org.jboss.resteasy.jaxrs-api_3.0.11.Final.jar@4,reference\:file\:org.jmock_2.6.0.jar@4,reference\:file\:org.jmock.junit4_2.6.0.jar@4,reference\:file\:org.mybatis.mybatis_3.5.2.jar@4,reference\:file\:org.objectweb.asm_5.0.1.jar@4,reference\:file\:org.objectweb.asm.commons_5.0.1.jar@4,reference\:file\:org.objectweb.asm.tree_5.0.1.jar@4,reference\:file\:org.omnifaces_3.4.1.jar@4,reference\:file\:slf4j.api_1.7.24.jar@4,reference\:file\:slf4j.simple_1.7.2.jar@4,reference\:file\:user.commons.log4j2_1.0.0.jar@4,reference\:file\:user.commons.zk_8.0.3.jar@4,reference\:file\:user.jobengine.osgi.commons_1.0.0.jar@4\:start,reference\:file\:user.jobengine.osgi.db_1.0.0.jar@4,reference\:file\:user.jobengine.osgi.server_1.0.0.jar@4,reference\:file\:user.jobengine.osgi.services_1.0.0.jar@4,reference\:file\:user.mediacube.gui_1.0.0.jar@4,reference\:file\:user.mediacube.metadata_1.0.0.jar@4,reference\:file\:user.tsm.client_1.2.0.jar@4 equinox.use.ds=true osgi.bundles.defaultStartLevel=4 diff --git a/server/-product/production/HIRTV/settings/application.yaml b/server/-product/production/HIRTV/settings/application.yaml index c8f54ce2..5fb68d8e 100644 --- a/server/-product/production/HIRTV/settings/application.yaml +++ b/server/-product/production/HIRTV/settings/application.yaml @@ -24,9 +24,9 @@ services: proxy-root: /mediacube/data/lowres/www/video nexio: host: 10.10.1.55 - collection-name: nexioclips + collection-name: test_nexioclips use-mos-gateway: true - disabled: true + disabled: false octopus: api: address: http://10.10.1.11/api/v1 diff --git a/server/-product/production/HIRTV/settings/jetty/http.xml b/server/-product/production/HIRTV/settings/jetty/http.xml deleted file mode 100644 index 95b63984..00000000 --- a/server/-product/production/HIRTV/settings/jetty/http.xml +++ /dev/null @@ -1,41 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - diff --git a/server/-product/production/HIRTV/settings/jetty/jetty-http.xml b/server/-product/production/HIRTV/settings/jetty/jetty-http.xml deleted file mode 100644 index 95b63984..00000000 --- a/server/-product/production/HIRTV/settings/jetty/jetty-http.xml +++ /dev/null @@ -1,41 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - diff --git a/server/-product/production/HIRTV/settings/jetty/jetty-https.xml b/server/-product/production/HIRTV/settings/jetty/jetty-https.xml deleted file mode 100644 index 71a08377..00000000 --- a/server/-product/production/HIRTV/settings/jetty/jetty-https.xml +++ /dev/null @@ -1,29 +0,0 @@ - - - - - - - - - - - - - - http/1.1 - - - - - - - - - - - - - - - diff --git a/server/-product/production/HIRTV/settings/jetty/jetty-ssl-context.xml b/server/-product/production/HIRTV/settings/jetty/jetty-ssl-context.xml deleted file mode 100644 index e1edaf05..00000000 --- a/server/-product/production/HIRTV/settings/jetty/jetty-ssl-context.xml +++ /dev/null @@ -1,29 +0,0 @@ - - - - - - - - - - - / - - - - - / - - - - - - - - - - diff --git a/server/-product/production/HIRTV/settings/jetty/jetty-ssl.xml b/server/-product/production/HIRTV/settings/jetty/jetty-ssl.xml deleted file mode 100644 index a079c1f6..00000000 --- a/server/-product/production/HIRTV/settings/jetty/jetty-ssl.xml +++ /dev/null @@ -1,58 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - diff --git a/server/-product/production/HIRTV/settings/jetty/jetty.xml b/server/-product/production/HIRTV/settings/jetty/jetty.xml deleted file mode 100644 index 0ba38381..00000000 --- a/server/-product/production/HIRTV/settings/jetty/jetty.xml +++ /dev/null @@ -1,129 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - false - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - 5000 - - - - diff --git a/server/-product/production/HIRTV/settings/jetty/localhost.jks b/server/-product/production/HIRTV/settings/jetty/localhost.jks deleted file mode 100644 index 95d7f4c7f49eb51400498486ead08cd5b9aad753..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 2256 zcmc(fSvb@SAIJayS*#Jpz9funVaC2iBD<_(DcNF%X3Fl&L18em=ERg`#85O$qT<+! z>@kWd*_FyxRHQ~o@ALM&?{m87{#^VnKA)@K^SSu$t?jJ=000310Q`B}gs?LKgpjbP zXaE4svK$kt0RRr7NU-lHehwZr4iE;aLIgn&4!}sTm3ka8c71bi-mc=0`{6Za{o#3? zE!-YA?K74d_sL&89hXgTcEh=)1A`VXPd&Xd3P0J_?Vf))Qhsn{Qwx1aGQ&9HSnmkV zB092Umy@}Y`QvokTsT)6hpJ@CaNHYq{^NZmBepvoGd4NH3fY==O7u1P&Qq&HZ!TQ? z>#`of2U+q~jrO6tsv)+j&ZIp>r*+W*!LLwhKi%#J9G8sB(qw^Pll!X&7@r>cgoaHD ztypxWyL>LU`_5Il>q(UTG~fBv>YaICv6&$sIrc{gR=6g6QC2cL4rw#KbI-u9dLXW? z_8=J9+Ba~4k~R_6wYKDE&6PK;(7Z2tu1hZh-0^&G3$qF>XZ8$Hyw6Qzo+UnDZZX=Y z@MDVobb@Ek*lf$eq~vC-o5ITR2daH11Pe06{WSQpH&}t!yj)Gr?S2jI4Rvys&3S4r zCOBMgca)ET4ZOA-%sY5pxpQ^kT58Wia#k*?!HZWLu9ddCm_m(vo?1{pUt8Nn#{W=~ z^)blucHOX8bM4hk$xl}n-w^E&-p)ocJKPk-J_nDP`epF&WbPqhq025x@r;C#ytR;z zT79<_FE#~O7P2%kl|D+3B2K6%FyUXtZ*cCS#|UmJ2I1T`gC0mK#<-d&3TwJZzXJQ_ zIYfy_cV`bYblnV#9nc?Ptr9^Af3 za3( zuSXp=x@n2*=lGwJu1l=Ax|ZAnVtH3^*KAJ zv}~?u$iUK;EaWA{yDpqHKvEw>AEheNs+HHJNFp-1*Df#4nyZEdKDS4-MJ}8-;koK6 zyW0HZ={mCJ>@eAMfNM6S8?uZ`6TnClpNzD+elFH($LKt=85+6fK7*TjeAi095+B&i zR;trXMsB^{$LjTRSL z;$(RYduvke@~z;S<|rO=u0evu>iOB|lC@Ijq<5ZDDw~X>*wiBEafNqDK^yfcU9P(0 zqgH%8*a7`Ih*I9D=7H9|uXei2$!O}U@+9&PlT6-HIzc?PR!Hpf8)sP-PRV!6sz+D+ zxKAvw%yt8ZYtqZ@)LaU7c(a}l@hx}Nh;4WV9XJhh<0VUU>Gcr|mI_HOA&`(n9@ zYoR)FM5&qWtmY=@Z|)QQ0sNeWlo+l>To_HP*d|}}?-kd=|J+8nmxeCN5CEWQC=xUo zMS_HuLm?mp0#mUV{|m*(!L3TRA+z~FkQWZ1BG9}jZVo60#({u@#Gr*x0{;ar#6JW3 zf5-V7$MYM<`x^)S#i0&}@u9U*dRhi(T@)H^aPlV_YU%wC|BaVOp!~04KsW#;f%^dz z3FLv0KoA(T8|T;TC=|`t@p-Me(I>soelMzUYo5OPPEF>tR9P%JGCOWSXxyUy%8eok z$%peSWESHL^d>)arpEBl+XVZ;A7*m`iUoPhCG|OZVyWmk`bUApeFgR3g*-8JdVIwi zK2?$V{Fb7xF9fF#Qem^+PXsJ^*gZz~9hfdsg-7Bq-`a=Q0A5)h_GT1rCl;t4JK+-L zq*P=>Z#nqsOI5&VAVSiLjWJKGU~^X{tJiTaEl9YWQ4sL9opxlif@=(2axdvyTUUt{ z*7-aVeJSJ}4PRnPv&0#^O?uTaf~Bh-Q4N!JvvKXVrhJ;%Q2vH7DTk6p$#6h1GW;-6nDA_7nyT|qJ*uBG&j8pv^caDem;E&&@6U)J&;~t( z@SCN*8Zq#Xxt^7zK4*?@_q3J z!Tn8X$GC){PPv?M>K;=hM_6ahX_(l^zMq8Wb9G%0EH!q)&$l*3n+aOjc6Y`mm=`DT zRNrlTLGe@(n18U-_GHVnZ+gwpH$X%eQU;CflAGoFUK_)GCzr!+;r=j$y-0bHOG&hF z(to_3;U(icdfn4|f<3uF`yuP~$vy@?TGI8>d@^P)!6{2ZK2?rW{1f@?S0sgPX^akY zhzV)fv>wm!x`j=1nrv>#o{%gm8ms?qcEuz - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - diff --git a/server/-product/production/HIRTV/settings/test-jetty/jetty-https.xml b/server/-product/production/HIRTV/settings/test-jetty/jetty-https.xml deleted file mode 100644 index 71a08377..00000000 --- a/server/-product/production/HIRTV/settings/test-jetty/jetty-https.xml +++ /dev/null @@ -1,29 +0,0 @@ - - - - - - - - - - - - - - http/1.1 - - - - - - - - - - - - - - - diff --git a/server/-product/production/HIRTV/settings/test-jetty/jetty-ssl-context.xml b/server/-product/production/HIRTV/settings/test-jetty/jetty-ssl-context.xml deleted file mode 100644 index e1edaf05..00000000 --- a/server/-product/production/HIRTV/settings/test-jetty/jetty-ssl-context.xml +++ /dev/null @@ -1,29 +0,0 @@ - - - - - - - - - - - / - - - - - / - - - - - - - - - - diff --git a/server/-product/production/HIRTV/settings/test-jetty/jetty-ssl.xml b/server/-product/production/HIRTV/settings/test-jetty/jetty-ssl.xml deleted file mode 100644 index db6e2eb4..00000000 --- a/server/-product/production/HIRTV/settings/test-jetty/jetty-ssl.xml +++ /dev/null @@ -1,58 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - diff --git a/server/-product/production/HIRTV/settings/test-jetty/jetty.xml b/server/-product/production/HIRTV/settings/test-jetty/jetty.xml deleted file mode 100644 index c9afdb5f..00000000 --- a/server/-product/production/HIRTV/settings/test-jetty/jetty.xml +++ /dev/null @@ -1,129 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - false - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - 5000 - - - - diff --git a/server/-product/production/HIRTV/settings/test-jetty/localhost.jks b/server/-product/production/HIRTV/settings/test-jetty/localhost.jks deleted file mode 100644 index 95d7f4c7f49eb51400498486ead08cd5b9aad753..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 2256 zcmc(fSvb@SAIJayS*#Jpz9funVaC2iBD<_(DcNF%X3Fl&L18em=ERg`#85O$qT<+! z>@kWd*_FyxRHQ~o@ALM&?{m87{#^VnKA)@K^SSu$t?jJ=000310Q`B}gs?LKgpjbP zXaE4svK$kt0RRr7NU-lHehwZr4iE;aLIgn&4!}sTm3ka8c71bi-mc=0`{6Za{o#3? zE!-YA?K74d_sL&89hXgTcEh=)1A`VXPd&Xd3P0J_?Vf))Qhsn{Qwx1aGQ&9HSnmkV zB092Umy@}Y`QvokTsT)6hpJ@CaNHYq{^NZmBepvoGd4NH3fY==O7u1P&Qq&HZ!TQ? z>#`of2U+q~jrO6tsv)+j&ZIp>r*+W*!LLwhKi%#J9G8sB(qw^Pll!X&7@r>cgoaHD ztypxWyL>LU`_5Il>q(UTG~fBv>YaICv6&$sIrc{gR=6g6QC2cL4rw#KbI-u9dLXW? z_8=J9+Ba~4k~R_6wYKDE&6PK;(7Z2tu1hZh-0^&G3$qF>XZ8$Hyw6Qzo+UnDZZX=Y z@MDVobb@Ek*lf$eq~vC-o5ITR2daH11Pe06{WSQpH&}t!yj)Gr?S2jI4Rvys&3S4r zCOBMgca)ET4ZOA-%sY5pxpQ^kT58Wia#k*?!HZWLu9ddCm_m(vo?1{pUt8Nn#{W=~ z^)blucHOX8bM4hk$xl}n-w^E&-p)ocJKPk-J_nDP`epF&WbPqhq025x@r;C#ytR;z zT79<_FE#~O7P2%kl|D+3B2K6%FyUXtZ*cCS#|UmJ2I1T`gC0mK#<-d&3TwJZzXJQ_ zIYfy_cV`bYblnV#9nc?Ptr9^Af3 za3( zuSXp=x@n2*=lGwJu1l=Ax|ZAnVtH3^*KAJ zv}~?u$iUK;EaWA{yDpqHKvEw>AEheNs+HHJNFp-1*Df#4nyZEdKDS4-MJ}8-;koK6 zyW0HZ={mCJ>@eAMfNM6S8?uZ`6TnClpNzD+elFH($LKt=85+6fK7*TjeAi095+B&i zR;trXMsB^{$LjTRSL z;$(RYduvke@~z;S<|rO=u0evu>iOB|lC@Ijq<5ZDDw~X>*wiBEafNqDK^yfcU9P(0 zqgH%8*a7`Ih*I9D=7H9|uXei2$!O}U@+9&PlT6-HIzc?PR!Hpf8)sP-PRV!6sz+D+ zxKAvw%yt8ZYtqZ@)LaU7c(a}l@hx}Nh;4WV9XJhh<0VUU>Gcr|mI_HOA&`(n9@ zYoR)FM5&qWtmY=@Z|)QQ0sNeWlo+l>To_HP*d|}}?-kd=|J+8nmxeCN5CEWQC=xUo zMS_HuLm?mp0#mUV{|m*(!L3TRA+z~FkQWZ1BG9}jZVo60#({u@#Gr*x0{;ar#6JW3 zf5-V7$MYM<`x^)S#i0&}@u9U*dRhi(T@)H^aPlV_YU%wC|BaVOp!~04KsW#;f%^dz z3FLv0KoA(T8|T;TC=|`t@p-Me(I>soelMzUYo5OPPEF>tR9P%JGCOWSXxyUy%8eok z$%peSWESHL^d>)arpEBl+XVZ;A7*m`iUoPhCG|OZVyWmk`bUApeFgR3g*-8JdVIwi zK2?$V{Fb7xF9fF#Qem^+PXsJ^*gZz~9hfdsg-7Bq-`a=Q0A5)h_GT1rCl;t4JK+-L zq*P=>Z#nqsOI5&VAVSiLjWJKGU~^X{tJiTaEl9YWQ4sL9opxlif@=(2axdvyTUUt{ z*7-aVeJSJ}4PRnPv&0#^O?uTaf~Bh-Q4N!JvvKXVrhJ;%Q2vH7DTk6p$#6h1GW;-6nDA_7nyT|qJ*uBG&j8pv^caDem;E&&@6U)J&;~t( z@SCN*8Zq#Xxt^7zK4*?@_q3J z!Tn8X$GC){PPv?M>K;=hM_6ahX_(l^zMq8Wb9G%0EH!q)&$l*3n+aOjc6Y`mm=`DT zRNrlTLGe@(n18U-_GHVnZ+gwpH$X%eQU;CflAGoFUK_)GCzr!+;r=j$y-0bHOG&hF z(to_3;U(icdfn4|f<3uF`yuP~$vy@?TGI8>d@^P)!6{2ZK2?rW{1f@?S0sgPX^akY zhzV)fv>wm!x`j=1nrv>#o{%gm8ms?qcEuz \ No newline at end of file diff --git a/server/-product/production/LOCAL/jobs/classes/user/jobengine/server/steps/EscortFiles.class b/server/-product/production/LOCAL/jobs/classes/user/jobengine/server/steps/EscortFiles.class new file mode 100644 index 0000000000000000000000000000000000000000..181784cfd04cd790b37b7b3bb752d45400ee9fcb GIT binary patch literal 14968 zcmbta349ynmH)p_$4KMx5j##IClH5Rz6D7LASDitk3@-&5IYG@+6LJg+X|9K%#5*3 zx1~3f-i6YJo^)wb+HM=5$Z|sn-E>R07u{ZL*_LM8ZcBH|9_{Y-SRniRzL}9nwoKUH z^2?88X5PH_zW=-5=o>G6`EdZ-*n|W|*gR>ddiaDkqGraE88vLE`mL%SHcT~Zgu9JV zO*eazDbmD5H=sQ5D?wzL^H1%X= zJW43<)G~${&zOhfsYx|}Ab(RG)%3(rGOZ4p@pP83tZ$$z))VWV=N5p%Z)=IfGDi6+k5+&^8)3M<>(-A;6#Q8yx5lpqk)oJ7H)UfPH-n6M^5^ADjT+NKm z5NaHvuI|CkgRuicF)p%>Kd>pgRAW?6X3Zp5Ah18y+uPMK)E&TO{3@ABb;V7UP}wtZ zu&-n2`26#Fg51~Fnn<+H%*@OXWJgl0D}eO`S+xbk5(LI?ZXhgk-ap+SCE=<5!k zfl#s0XxnJC-JDFN5<*Zi_rU<#2vVDMi%`)y z(BIP=>l|{PUPTDDIX4M`9iypaCTX@4JPnP93EoaEp~~2fYb9JmSpRbH_r){uaaC7v z9U_F_xN2H9A*^p`yrkiRAnZ_3gXM(E-egAYpG=Ra`cQl%rE6784Y9_>fgocKViNWeszs1Yn#okS zGoDg2iMXy{ANCW<##J*Wf@M@CtZ1;=6p;!>eG2+9K=8!5$$Sm5SmR*{HxTLsZ&RH% z!-L6mHl=bu>(TUd+>~(ugZ!LBgxZG13N{|*DjrsF1UC|-ana z!>b7XF~Oa%!R9Hn6=Wq)Ab1n;8G}GELV$aw6$r~MeTq*?kh$Ml>J!#jG$jugC9GUv z2%fw8IT7XIBzI4bsv@Tt5^f>X2x}*`aBQIPM1nzBxiAm! zo=m0^s$PakOi8%aal#@I3Z^kb2$-66Qo?e_^dj%u8J%PES_OZB*Ab*!c}^b~BW!Zx zvGn0(=osYxdxL^E;!Om9JeyTB2|}xzfh}sW1KH;ge5Pc)6?aH@n*&7AX$%E-;_U>_ zxM~ujCAhoIRV=WGQB=Y^2pa{Wg$Hrd)RQBVrW)op>u|1QybJG<@a{z#i9)L2y?7r% zwrp-0`(R09+p1kkoC8pSPIwA05{2qbPG~KpskDnGSA)MsP;9-_R7#sF9W^cocMv-q zz=sIbCgDE9s$!Hnwd~A*@K+AZM-+S%A0v22wd@RGxa7$#3B}4TL}Q-qQUC-+l{7^0 zt+I~QrzVnd5oa<8pH%Rdcz`Dj5r59ZiFq`dN~&jjWU+ocsy?mY6h33stpjmg&6utP zUQEvAe^$ZgkRep68Dmmc5A}EUXgoj(!8BLYa*i^Wjd;*>a7m{XJj5mOV!_FHj^hs> z;Su$v1%b3=BszgasBj=KB1OX-`R89NWFnG3L3HHWXQ@*K+qjx^Utu z=bsLq1jV^!5r>LFijDPaX7_Z`Fpd8QUPn1z-%#*Pe2Y+(H2Q3X9LA;URteYzo9`%i z6gh%lSB=S(NuZ;I@%x zPw=buakrE<#)80|XA*5Z8o)Dzbse|WZ{BEZu0MZ&Dy5~*-=CSOPoKYEuTPzS&{S_r z-rOc5i$Q`ks%1??Y1uN`eZ3G--Rr^79RMIPKFk5 z7K*2gU*Ml4{F2bXGud=_I-Lq<ePJe z)?ydVSHd95h5kmtKjU8r)fshafgHldSHdkS<9Ym>gnuQhyQJ<4{vH28SU#?rT?2hF z-YD{u#jSB=o!xIwZ5<6Kv~+k8Nwf^V$A3xqPr_xqrC*Xz!GGfqg%sl)jBt2KyA@fo zh_{XY&`g%sy;6^&W%v{RN5Y>8%S%01@B&^Wlv}>kZJn9WSVEa%0XAv03>S$c0-?su z-+rtR!@UH>P|ZR^5Fjt1c7C9%Z`ivxl1@hqqmeW7D~Tg~~`NAUvc_oBY{=e$1)WMQk~3w2UgqFHt3-YN1HZ-Yzu%ejW_CL-~*EEw*TlC!V5w6)-p|KDN%NY%=qQ0+$yDbFA zx-NG_?{Uw(7-WS_#z_-;yazS4nhb@e;sM*mW*j=Pk6AASaD}&D@=eXKr3%W<5$M zOSo4p{QHDmbrO7{hRpY7d}Nz{cR{u(NR_FZ z_DIy@)T-jQ6xz#I7QWGxW~eglqh5*jJEQ{8p$T1^;tLOj`lz2!9#13&C$m{yH4Gk; ztK6PgAdWU3E~5ihfg=a#| zLiLSGG(=95a;U%4F(4u4*3`~si+wC60!j!d@%qy_}%M%DL`1P5EJhKP?n{+0~@2JE6+im$=$g=a$tb`4j4> zNdFBsV8xS%J1%V{2*JVRcqVR6>M9|;V<@s%(e~2JhK8at-bHDJGNcg{U5zI?QYpLD zcbsd12FG^Gn4or9% zynV?T*kwrf>bn4~Z^RmVRur!vXR^);(la!~r$>5=LT{zF5&WiR6^SZm^ODrW?jS1D zophH(Zzm{D8sd}+o~Cz*ZR=4^@h*kl&9BRnMpsf-MS~ik_Y#8o)U@?SruWl565UN` z5sV7TA4pUUm=7FH|upZlQ3>m_4AM2re;)ArpM_#-|OGYGuYoK^fh{t_q>@g zjnMxJO4X7jelffRPudvgo%$<&iJ&C3sZ2_XC#-KVn|fb~QTTfWlL++vx|-H*Re73w zN}=!aH0MvKDb-X1^gTk&p$z9=Z))|{-TK$Qj;GciDD*?VE~_xLjuAskO`3MC^zz+0 zVVT9&*>5_6RHf=ENbfsyBcCv5W5Xq;!cs32Te^^59vwnwe&1q zkmzTG+L9y+{UiO{{w9_*Oui;;ZLmL5IW06-)Km@5G#qns;fzHdmFSm*<|5v1`W`d8 z;0>96LBCSy*YupVkI9HHz7546;vCT#)eMiB*>KO$fzA{U^{7nOpo16se^KbSyv~-H zTCX;x>YZ^z4bZ<5ylv^}F;4OC3jGKDj<=9@s1a7X>q_TDTpv_#nN%~QYLxG~Cg?vE z`Y-x#LRl(qn6XSko#tO4ybZC&7=Qdnh5kf;CIpSi5kq{1=BwD)5?_4i1%=krh5)_9 zm6}ecWCq40MuZJ6Ctvcc3iB{8p;92vGqyDkP+@`W_6fVZCAILtq*<69q7w5HE*Gg; zQ1?%!QU|nT##Hq}a~xm+{sA_gsn2L;y_Quok@^6W2~zz*HJh4=aJ?0Um2E8j6|0umGQx5pX@L?7zCyb=&2oj+ios$mHEB%9tcv!?^ekJYun@m3ji%Ii z=1`VU=sml{7rhEIuDe#!VkI`6)SOS|LF@4$JxN%(L?jS`nln)G`MZ>!oXD0mbbzqQ zor#t-63-@1r_60c@Jy#u65B+$mJ6|V4yGQ@82oc)c*yye>a6)Xwai#@d{P%HvToj6 zO00o!^(Edee$DxMC$XlY4P!~cF6tJ-1_z5n8Dlb=)pS!$bZ17jL^3n(qHHVL{>}5X z*LZ{4q&}+3td(7Cbe)HLqp&IZrgH651OR(7?-wh=a6;(+s8P_~`>yyfAS#xu%A zT&u9_ScFiKQKwvxTVMPyAvhXfJNW#hniEp_*0{ z9?-R{s+%(b)S?`ddw}gF_>-B0nlS@xAHg%D8WQVu#gY|iNB9Q| zU1s~yA+Z6%=2E&CVq?JHP~+)?VqY(_es%*l$w5Mmo&J(q*kZ>k1lz@i6n2Om=I-x! zXW_f%LcA?ITP0}WGfSZ0xgS$9+fRFBOt7O0&ah)VQ!XxCVwo3AXlaXY@!qYVO|g^l z{@5HNSnL)m;j*^vL45sIU}gPSrxbRs7r6vu%AF_NHV{vB{#%YxY)Y7be>b(xXJv>g z*H56@I1j7?!2_TYb@0Itf&_pefI7&o>oSB;?z*l(2vx4@WvJsnQ~^+jTG#y*sKYAP zbqIA>?YdrrI$Z9$UW+=cb6sD7I&5%VZ$uq7xvn>(4vntsCWO%9x^6`XVb}GQSd9R- zU@HjQ05pTZ|C92#`VmAQ5{KUjKn4SF4gBapySNJw|3!`cViK}91i6$e7c}aHM{`2 z;6*pe(1Tj+5fbVxE_gLw1Hg+q9LG%{)FOyDM!+ERwU8|}2A?=oZS`i~S$Kv$O@qVU z)p@9cd5q5svyO0s5U|2}0V((eOtnH3A&4|GV9zjMlHP>(<8A=;kZ}+07Sca} zdlyL`hgV1sH`!u@7PbN(%;EOtA|*U$aWan&yHv1DeiUsu<`N&q`|ZPr@o@o85T7V< z_#455_*G4L+~3mp=7 z1|NI?b~IJD<}u5KG~2i?voP&%aw*ZurA53UZ%Ynyp0hZc$2k|wD+I3${LoN`ELw33 zqR@pl+blUaZ$rLAcpeX_9r!BzV5q|t_#61)K`ScpHQWs^f_M^NU!?UkJR)qCd)k|t z^Z48QP}bDklE=5b?U#KXfJel+rOhN5ZiS2~RACy`xJ_u&WW8U|W_dxI<+e5q)p!cu zg$I8JaMQfNf20}2_e!AnIY@$EhlOI3SD3AtgVo>aIg9V-@xzl?bq3kt>K}VWfN~_q zXYmi`S{FiTDPJdGdOiHO9W{6ZF2@_O6>q}zc(dyO9fbpQ6b{gVp9nZ>(1xGlZuC;SmK=uXD9A8J#$( zB+XG3C-&!Q*}QwVp$x#=;Kdz~aVM7H?YIo@Km*>1EqD*w@m|F7KD-6*$GdTlK|SLo11=K`-7*t0)8yWZXq{v>IM4!<%UhT?QX6!xUXkYvD%&QdCdt zAfX+H=?aX1|2}va}@ljs!!2kW{_Kg_jfxy5MZdQ`Ze@DSRHEK@Oi4Rv5GJ zC|Kb!9Jj6T7;O{&3Oq(vi}3ehR@lP}1*e5Qc*cBC*u&2y3wubsdlAiIeEc#XE(L z7!kmFu@39#5FLiZ%|=J)M)(A&WxFNXS6+S*488@V zAPE=B5Tv7Y%!RzwWIQGMR3K)h(dIRIdR4^hu0J8KC_uiDuaH%jKa6N=4ZVhgc#{{W zP%a)8dJxNnFBdqRg&&Wj9OtkOPoNcFK|9W4FTN_UY8NRtR!FI_LQ0K^-iUXNF-njM z56ZEF#^4uNHPATjwh>amUPQa_x5*%ae607uyKDFQ8VUJ1=52yc{*UQ=(H6?h@EtL^*u!yAm#l z2Y*C`fOT;P%p;HwbJgzNqOT>72N(9)zeYKp!!rCvoYWd4stZ0{O%IATvm8G9EbazF zkUmGBw_8x2`nZ>0y2}^F&Cwma+hVS%>bI!J^Fp#UUpW5f>hS5W3i9>{&ju>(#+rLf z38sI54}Wwqtt|A>vfW1q`L7Ze`l6^BdG#Ivt_K{%)>g~3+5#>%>OJATb+Mxfcko6qdC$Xz3Pp6-thx=>j zOA&8t$eW|wNi>Rod^zH4K0}WWhkP~k#4P=F%N%{xgHtH$Z*CRg@(>6LcH;U%)!4*a zAo4;XKbDh(5Cw1r1<^z@c9Mb)D#u=0CJtm(jF>%{0WmTB^FBRBU#GtX!*VpzH|U!p zZ)~7%i7eQJ8hR9d_|O4K#EaE82f_Y;>m^)}5aiz|>@(kHJK9F)%-=b33U$3A^Zi}Z z9DSdx`9~x`ejYxxmlw_>#w@CZOe;`9E3tt>F8A0-KN4|WffYg%D~Cy7eG)FLK#-m; z=y3zjW&qZUfpY1wHcx*)A7^5iq&2vLE)!C%IcvR*SdgA6uMttX%l?3 z89{1r=`R=D+vm79{R92P<=&!YXAwxE8+e8y-qsxb^a=V!j{Yg)J44qD&(d%5w2?QP z)(WCINB>+y&*$jhPNK5e(_Gb-qu=M~553JHe-4MJB~Skokz7$yLoZtMUnCIng#tOc z$otqQ5jn$r!$Nwavpi&YM{TQAkHU=Fv3d^uKTv6y8Q=R)3)lw*}4Z;sWR!fIZ^ z&qabEFK?B+IkuuT$5w`d=ZbIetna~TJR&TgN4_xIoj@Hx&G1kwDya>tDU9`WC9b5a zu!FWBMq6>5wqb&ocrQtYd&@n6>KS|5{_Wug|c{*yUpV^E6w_>fuEt?q}=R z74Ts-J|Gr2e#9`&*3%Fq9LEieFTetL6dvKJR+lvb=_TxtSe3*eVb6;g7fsZ}MO45? zb~{VYN~}s0jV_7yT|^n=;zxY*Mf-q@JSrDF3PR>TlC(Wafsbuun^_~WtXcFZzk^5g zOE;Zit;0>TEZm!CTl!jZ?3(`88Wyc#J9DhPwT5-pux{&LdumuL$J)=Z{ll}YFXBDJ z`iDc_S#}`eYi;JPFvzWZB;x0P-5BzV7hP+xEl>dJL6G*Koc5xcVpv1_u!;8LO6tXR z)Q2wW$9@{XK{|jVWNopmY;}|9TlpgH2zwQKH3;PxWUpbz#T4JqZenqGa2+30+76~=$p8f#=(*h0o8rmay3wq;vJ;E+HNgoZ8Hwb@J|hNNUHy_N?{Gh$|B zaN0CoQo5ykX-k)M3+YDMCT-1<(m$Z5U;E$W(+UR%_>S<@9BAOD{N%L;7;n^ypu^c>SF9IBYbymF!> zXo=7z>~~dB<-BChu2^1ba3E>8)#r55t&~zlWfi>HU^QQH49~Q!YvMzNw!&Ilc;v3k$`nQW9kQab$) zv8M`7i~$CP@Qm%03{QBfK_(aLGO5+1-bK|=HbPp67aY4%PEIc@WfWrv89^^Qc22pj za58mqePthL#GfHJJEC&7V|ccc9k?LqrE2dB z$ws&8yGYP$Xt>^}l~>n%?sqn*&A_OC z{zQzEoD#y-<_@GX*f6{mKt|HDu(mkO(BX4-d z*@dz?qCA!JY$wJP7le(^R+Q`39}u2y6!U6?KhQz8h3EN{&I>|ny+Ose#EU}WWX`k8 z757xp$f;S|mq=l7{hlLC3xhQ&4W!Y`r>hS$W>M8hnNIpImQhj>^1|L3$1Wx3uU>Mh ze7NN6oSUqQU|^)~Zn9|3sbqhz+uPe8VMf?uJ9*{!>at|C=FkjuW>z?msu!!8GF=>7 z1E}BXYbl#RDEQNHMPq9wgrdfhsghElL>AMisVtgcjyqPq&yC1fr{xarRUUoXvkErJt9NeR(O zyW-^3G1LE&+fp|RR=yN>_@xx$hiuz(J;x|dC~wxzyHS45FLiz)Y+IqYY0cYnDl;B1 z(HEx~!`G~Z7{B7zLaWc?+LG|E`b29ttHzHTR=%j*?c=sRS1H$5yVgT>^?&H1FO>h^ zX4U0~->t7g&Fx{MSe!IHm2D66y0E#H(&~)i1L7y%B{6k}P5ZQNEz!Bi=JVZJ;v=>H zk7*AGO@+4JW$WZ!dROfZ`R~K`*$Mla2fMXJc6B$)J)Y1wKO7qQ^Kw{&L2#}3!L+e~ zF52j!o!!LQ&qj`rV4Q9)vLy(8H_d){WZ56&eu)D-9prz3gM6G2ZJg#A4xuql2ZuRA zGc6qDST#j?0i6hNQ~P`Cg!X{GsnF2miJ`=^i;O<0b!xBk$j~AaomyUhgAK!>p-`u` z$mz-9(8ygrxv#mq`5i94$z4gT_`ikff7s^zs-PwzG#`4${YQ`Z-04=NO>GAQgsqnKakg z%?Ki15(bT1!Nt0m-y5j7ADe(mxdBA-QzP&+-2(` zUpSvmTpvnYuU{Dqr6)tTm#NYfl)i%qo7h4pX}Z|S<{-l^nt&0$!dDR@4DvO;PE$PU zH|CmXJTx?5TRQR0dwjcz6N90V;YGfGllV|5o%m4@#i`J3&G%ldMb-N-E$pC!3_a{5 z$w-~|ZGrbDTIlA-TqYD?6hYo2snbV@p9H!5l%GKm@f(DAli%_C;OP(ikw4+LhOaYk K^B4X`_kRH2X=Eb+ literal 0 HcmV?d00001 diff --git a/server/-product/production/LOCAL/jobs/classes/user/jobengine/server/steps/MetadataType.class b/server/-product/production/LOCAL/jobs/classes/user/jobengine/server/steps/MetadataType.class new file mode 100644 index 0000000000000000000000000000000000000000..ae205ad2fb1601301b1a51bf24ae9e7290ddb146 GIT binary patch literal 1320 zcmb7?YflqF6o%i^ZMzp%DCMGD6h)A>Tq+89X(bRuLoAnMfn?*SVHse_vP*WiB#l4D zzo6v#VYvhc5F7rP@t1 zRtRPy-w1=FfS@P1_?BRULQaj8rsnTLJq)Zq#?df-$8c1Q#nKa z6ulDer6u%97|2K%l<*)cVMxLwRl=}@5lzBqF8c#yD^2uTcDji^%g!`0XxZ5&hAdlc zV%V~^CPuAOjC}==LJ@$0lr*>o1tqyF{&b@!F*^Y;2FlRg;@}gh)lUwF(Imk zj!8@<@;PLrytj!l`-4)tu&cu5sxokn^2C=b8<*MW%7Ttr%q2NWNXh(0o=WF<@e}la OT?(&d&I+nBzWD?9Yb^x; literal 0 HcmV?d00001 diff --git a/server/-product/production/LOCAL/jobs/classes/user/jobengine/server/steps/MetadataTypeDetector$MetadataType.class b/server/-product/production/LOCAL/jobs/classes/user/jobengine/server/steps/MetadataTypeDetector$MetadataType.class new file mode 100644 index 0000000000000000000000000000000000000000..488120eeabca17fe90ac9ba77367095c49b0c461 GIT binary patch literal 1554 zcmbu8TTj$L6vzLE?rtwsz~!Q=ct>H^%ZiG(U2y@?5EsxCNE)ApGC)ab$+Szd@x%BP zba}vNG(P*GjG3WAW8#C^zMT0@d(QmNzjJ>6`tlvX1I+5^Av|lvTvT_Wbslc{A+N?< z>~m3#6TTN$7kR=qSi)A?d;A4YxR*p?EW4>AN9fyO`>YzUaH~2Wwsr~qD}rrq`raZ- zxbRs(c;4*lm>R)OYrYZDZbUFsoOwa8m%Sv~YsF3yi8i5_;+ueZd^-v@xFG1Sc*uqC z5qjsBRu{+CW}2(>4x#a`Yh^~=3;fVerU}|ad5w^pi#E7{yamFbjUfyZ3ib4pk%{Js zT}~qWaI3aZUNb>BVR$H!ZjO~gkA|ZZ1kcaR}(gD zoFeo${g5xUcGtOBVe0`W6q}L9f;A?5$!gPrhCxCu+4f^XVV?yp9usCK%H5h{;V$m! zm?R8jnr_Dl-?ecc4+tg`g0;P9uT7|Atyx~*;a*Z($hh@qYeAZQ{8LdO5u#|d7K@EZfV2rolRyQEb{MWO8fNIs#HGvS z2k~3bK4@1T`q+m)^b>V;b=^tOP_RqqY4+V`pMB1~=iWd6`R#Y$CPtWm;9AR-PUeZd zA+0Ubk{MSz&!m%aJ-O><%F@#}bx&`+*p+$dNyD?9Fd@N-C;Brz)6}i4Ol9MVG(17? z!b)|auqcR^-jj9DF|Dl`!Lgiext?x$%X+gVBSZyaQV`5n8WA*sm=Yvvh55pR>Z3+2 zJ2zL%m9vdPt(Ywp^ATc#;fLjy*I%t&o4m9B^46=h?BweD%j>Vk1o4BiY+ecCN>Dpm zFmzB*tyRhur5RG1>i=shxkjbBRIfKGwH2jF2qF*tqS9l6kxu4LRJD}N6&5O`e4(ad zM+C7>?4x&*u@6d?3$=Q&vRLr@IV(sU)K0dy;)dFS-n*t{db5Jy_{6dxl(RQwA3-i^ zoa4M8Rx&NQ*xK2UPD9^lN_9xKp*NRx$5gnp81lAFS1?`r)R~wO49vHr>vj)}V0QfQ zC`^2G@|ix;Ooq8CxM(|D8GTnbwq?e!cXn*cZ4Y02zLq?*>1s@o5rj?mep9!e3Wkny zUyX8|8)2pe;~(3l#!WsKgy*Ft9n%n`KBb4-+zIo=fh*IlTH{N<;2qsFwxz2IyQ}e) zvWhhv{qbYdDC?edOuebNdm3LWPU~=0$KJ6OabF{+h-in%=GAIJW6lq+7@ob`a_gS$ zyimk~Mo|&*4pD9DhTOKBo6->k%uT_ux~CgY%ld9-ECxKMWf{6B3(q}ixu$IiE+4hI zhflkY+f+2Fd?$z)w&m%jm@QzSXvg_9&X(}hO~A=E2@-~y0AAb_6KTR!{QFKG;X z0rYSQ(Aoi9<_ZvHoQaP3F9JTdp8A#4Yr?;9Z7&r+{)SsVnAuCk2lv73UMfDY53+ly z_^Ex6-%G`l`(S=A6(4;MR7N1^7d1-^h!f@{{R}e55DA94z$q?~V2aaBlVpYw?s0|% zMk#TYCC;(J`L1Tx`G#KLTfd=jzvh2LDN*k9_LL9-U^G=q|3M-h?|;MMYhvl3|66;* zaw+}3wm2bRi~yI2ahVaW_R9_W%F@ literal 0 HcmV?d00001 diff --git a/server/-product/production/LOCAL/jobs/classes/user/jobengine/server/steps/TSMRestoreStep$1.class b/server/-product/production/LOCAL/jobs/classes/user/jobengine/server/steps/TSMRestoreStep$1.class new file mode 100644 index 0000000000000000000000000000000000000000..77dd877a91858f5559d2d1bd943f71e39fead4c5 GIT binary patch literal 1303 zcma)*Pj3=Y6vclRuvnND+G_o;t;I?KJFRhtx@p>op=wIgjjI7}=rr)AnHh**%fxDQ z;RkTxM>1vxj8q|ER&Q?3J@4Fe&-?M~+jrm@PtzoXNd+jxkhM=l2_uMsZ z_)<73PQgzaxH<(p&b@Uar1reF&LiY3(p-&BOBVBFG9+2C$Z}0c79(h`?izj5?={sw zcA8xkmg`>2>7F>g8~u-psmL2)dy0TbVhg)fA$LCVdhgUR9c>Hc(ZG{7b7?Wvt&yGqSf9MxF$*mou_-LMrsZo-%@|dxSc^L3(m9z@7AyuRPk12-~w<#LL*qxcHBebefCfonR48lHOXG*5)mWU-U%wy@a|()vML z{ZNIU|0S#4v=6oII@S8$7t;uR*K}s8)Mjd%tT?iFKYq|NW%&KrRl_%36|9;u>hE=( zAkcs{ge1Zh68U_Ta>y@783z(T77JkkctL_BkSmohSS(d8$d|sb{4oYz2QqPkkYS%y zh#Mit4Q>L$EpCsf4oJk<#nLD8<80YC4vxtzX3?YZpr`v literal 0 HcmV?d00001 diff --git a/server/-product/production/LOCAL/jobs/classes/user/jobengine/server/steps/TSMRestoreStep$2.class b/server/-product/production/LOCAL/jobs/classes/user/jobengine/server/steps/TSMRestoreStep$2.class new file mode 100644 index 0000000000000000000000000000000000000000..a4332459bdcd1338641639445e53a9694dedf417 GIT binary patch literal 1187 zcmb7DO>fgc5Pg%Fnz$|rZD`9^C=g1Lei&68kc_xQK?)?T)Koo`)5aO4TatCq2fXCRfks)lxnj`Nd?6T@h#h&J9 z#F3{BpK9-*)81!o!iYPDPs=+tEJEQ#jD$B3s_*S}PdG3(<_KFUP7n@a_xUg~A zK^A!zITQ%lN|KOoNyWSIu*=bb=nj~$*a`zNI24gg?vtREP$z6(pnrMCCoJexi2AWm zeeMyelL^ifZAvr0dgx#kH*8!d)MgjU#ZBBI%m+f%LuI6jIUV%KK?!R%?hwjF%cmU^N!6bz(wwR6{4F$J zgxyB^jr@OP?8nN;A^X)=Q?157$tSo?r8ue&gx0LF20|8uOQ1p_F)e@$fE*kU<^eoG zCXG0?jgN3^pK#@UnsXI^lfER?FR=)~1*Op?6af&fVR?cgCn&{@+6TC2Sa}OO`wo_M zhT92Vb_#D7OV~^CmQ(!V1P>_UKFT0G0I-70o{iFvgp3WcQA*BIGR}HR@F6MyEUco6 L&6F0U^Z@(>Fw8*1 literal 0 HcmV?d00001 diff --git a/server/-product/production/LOCAL/jobs/classes/user/jobengine/server/steps/TSMRestoreStep.class b/server/-product/production/LOCAL/jobs/classes/user/jobengine/server/steps/TSMRestoreStep.class new file mode 100644 index 0000000000000000000000000000000000000000..59569e0978741e115419ff97d459bd0895ba460a GIT binary patch literal 9183 zcmb_h3wRvWb^ecbrP;^mVHq1Q#&|5-thAD~*cfAL{Xmv%kdbVol?5^+WOnyzHPY;i zGBYbI5okgZ0wgJM2nhj7abMJJ(>TVKEgm7MN}#k!n!eJsX_`LMhLqAa^aaKF?w#G$ zYIpTWzWVd`?abVB&pprooO|@zyU%?Yz;-$=p^6a6+RTcMnh9p4HG@TMW=$|FYCCM) zjt<28`kC#R7K=MbRO5VNw23Gt&Hb`)IUR zki-a5Mm1DHt@M^~PbH!~Jr2wCxfC&i-%+hJbBFltYVT8usQFTI%>Z*~B#vMyD(lJ6!MzzM4kho8f#0b9KNnJBEXD`9i z+%ib;c9|&_z-3r1VHIKHVoK+9k#RZJ5d3N8xRxMnY;IZ7Tz(L)kWq~qLVd4hu*2C* zf>{Gu*0leL=g5h zFJS+2`}UN7;${mRIFV%Ij%FIRgsTZFg<~``+H<6^BB43IAUsxB;FgHUs6%}KZD^OU zg%B=@pLzNmu#=|c@Hnt#L~#vaMbcub!;T&9x>?ipZq;Fgp5}5{%fIw2?5JFn2>sk( zJ*qk*1bHlXMz=a?6R3w!=dRC%W^LS2lVg4Ac+PdT2{vq6tYGrC<(r(JpHG(l9zuO` zgeAuYME1A>t*W4D%U}Szu~)*L`B9wDv7)3o=4iStV;^oHNSeK0SB){M%=EYq`w7=} zD#NBesb*A13C~$3tRz^58O+g?an({Yv!@+qF=h4zi`nf#bmO3e1BCU98 z*3wyvrIc~Y9A}n3**0HE%^1%*u7=EEmXgtITQkz_0rcQ7PsR6x9Pq6Of$=a88=Aj+( z4a0O)fwYA10_VC#pI7h$hGR{}f=D1GAxYR!;=3HKazS(F$Y2;IG^CjmXSS`GhKqws zRJ9yf20$2*p<$E|w3*Xsq`Ej<5$^y0c1)46a>aiEYZQ5bS zFb+$?`{xn4{AJj12sOi|l~EnSt_rYQ!G$g5f=ZYmte=;=86KjJY3k7dEyI$c3ps%) z36q4&OPrQUxDhSmZhRouc`zf*Ym=wB<&You;)4=?j<9KoBQ_08jx!k_!hHnKQ4R`M z&j*Easl=9809ky5W8Fsyk>*3o+tf1X!^a5HW?O0AY`1V856XB5A1C-6)3q_-vgQf~ z;{p6h89&dlwa#WvH?xzLCK?Xzx=+dQBfw+t(=vA8dj9P*GJ*(5_-q~}=a6?^Gcmt4 z-n2Dtf+oNb$h{p1?0ic#^Pco^$dzDdQ>pB4MSPN*%S#w8d<@ zSF;^v@RoM_A`se>CqFR33@5)wMEt3r&IH>6cpAUVP4Fv(=uc$=SHEAA@eF>QutH6x z;*RQMi&WUQmi<<+g2 zyoHMe@Y^^e;fwix)zHl7u%@#p@3(9jU&2{}C(Rr}YzYu3oF$H*$0!!WIZR79PiQP( zm2m+V-Q+&1TFh_=tMjQ*&gMamThGgQ0WT6Hi;e4Qk`cO#rRy3|tvGvsmKjM_Y@m7H zDEFwwYpUW6WZhjRBl0fg~;GG60=7)UdxkJ+}GW&!*Gz9Hf3rHIjM z<~w&A-z3~p?CM<9Rjft}w%-}|ylC#+7;o9X#k=u8A{<_ZiI>f|WstM}6B&PsKO^{( zs*@bij5J|Qu?_u zCX>vz`&m*O*F-+8(D^01pE;JsCOG8?p)swSiPDe$(R?3AXevFosD;Y)dW67ep`Rjb zTx3VSFz^zXtNVP`GOsk07h&Gs>7PGUuQ?O0`UQmQ=ys;As7T+I< z5mpsNf8l}@BX}})nh;sOb0z!>K`H52%4|ef!5d65+?l&IA1o_21*ewUFqLq@z7k*s za034(I4iIR%HA){)gbx9A>Ew zzl;_TSkUz3zQ=-i8~-Qaojj1wTb1!HE)l#2JK>bJH#y{yhzM&-Y?fD@3mzblM- z6MWood_qQ8+nlRmbG=Q$E&-CrFOjce`zZFy6yWV+b(%RnDSm=1Q!NLd`ZRO;nXWpT zn-xL4P4zN0&CcQ49lZ_Gz)2`!ydg;B;Bn_~LQCl!m&Ez*j?G78gYPg^6* zdHH0!kJb^|dieanP<2HNC))CNf2DBGQ!F;jEM8&TgZL4xmr0=wggS2Hx$!5VsiNoR zjY$0<+9*SU&qtdHt({Yfno^y_?5QcuUP{<7p?x7?iv%eW*GDac>x(Bl^AUH^>5h+D z3BAjiq?9sUc$z}AMaC|4_$W%~C>E>?-ItJUE8)m;WUIh;A@PE^jjokxJ6-2G*JYne z%PX6YFPxV?x}G~z8Ho4IwE_vo+p5_wIJ2kh^2pmoxNaHp=LS)Nvyi0SkA8~Dw40B8 zJOlAQjtP5Z+DA7O9z4`-RIE*0E?nsFf(p#@c<-Q+5;duDB}o@@~4|*yn%o9(IMWe74k<(uu+ztoSHpti$1JPQH$GIW~5#|sSMAa z8r2npO)w#%qw2G#bkT{n`>2<&wOe)6gle-=I4A&?61IwPG~Di^!zHm?)`Bb%%SV?P z%N6)86w5y9C&Y>qth@(WLaG76(Pg-+BIAX`;w_P#PpQCMt3X&e|IXgdt2ROIF$`vP z>8foro8aZO0UJQ@0I0zg{7)?asyL#GUq1rkH<mujSauKbZr$>lUC$tR1i)PfLMeyG4?#b~68idAN+y-n# zEvm%RHe4(AHefqy;l*`=C)LBtCn-5$yr7&Y`@}LprTVmqGZn#$3TF~_g5Q|9VL1OP zcsZu$45Yy$q)=KfBKCQ#J&O?6tg&+jT^>A-o}tE@FQD(GIXxTU1rUXVYp@Diu^!io zb62~33g;?9L&|Hj9$t#m-XP^?!qb1VXwk1>(l0v=XnJRTj2glaF~bJKX-{qXcq#wK-h)X@dm=V&>ltzz-|Pw2WzkwE!c-$=tLa* zaR<7<&EqGU?Ljb$*2L1+riy-2741b4O@ag$|_@b-Nzeu&pmE5baAs!MnSvILTFLc+NFpYl&Xl8_M3Sxyk% zfht}Yb1Csdcm+Ckw@%~D!&_cKHJ#ZqgC7$*s-MB;Q1uM{m+%!rhd1P%rmAVGX`7~+ z&pZx4e-EAsRnJfz;qgn~CZ0=Pfl5sRY16322>j3x#3-6DhHcQX6QkGJ8Q-p!rYUPN_`pgKoQo*CL!g$^ks6*JsXM7q-zO*%^*)3oQj zNb5(6iRXZHC3Okn2 + + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/server/-product/production/LOCAL/jobs/schedules.json b/server/-product/production/LOCAL/jobs/schedules.json new file mode 100644 index 00000000..5c6e7c21 --- /dev/null +++ b/server/-product/production/LOCAL/jobs/schedules.json @@ -0,0 +1,192 @@ +{"joblist":[ + { + "template": "sync-subtitles.xml", + "parameters": [ + {"name": "sourceFolder", "value": "/windows", "type": "java.lang.String"}, + { + "name": "filter", + "value": { + "fileName": ".*\\.(sh|ini|conf|vhost|xml|php)$" + }, + "type": "com.ibm.nosql.json.api.BasicDBObject" + } + ] + }, + { + "template": "test-fork-cancelable.xml", + "parameters": [ {"name": "itemID", "value": 1, "type": "java.lang.Long"} ] + }, + { + "active": true, + "executeimmediate": true, + "name" : "Proxy generálás", + "template": "sys-recreate-lowres.xml", + "cronexpression": "0 */1 * * * ?", + "parameters": [ + {"name": "globalRetrievePath", "value": "file://10.11.1.90/data", "type": "java.lang.String"}, + {"name": "localRetrievePath", "value": "/mediacube/data", "type": "java.lang.String"}, + {"name": "localHiresPath", "value": "/mediacube/data", "type": "java.lang.String" }, + {"name": "globalHiresPath", "value": "\\\\10.11.1.90\\data", "type": "java.lang.String" }, + {"name": "localLowresPath", "value": "/mediacube/data/TRANSCODER_OUT", "type": "java.lang.String" }, + {"name": "transcoderAddress", "value": "http://10.11.1.111:65445/api/json/v1/", "type": "java.lang.String"}, + {"name": "transcoderTemplateName", "value": "MP4", "type": "java.lang.String"} + ] + }, + { + "active": false, + "executeimmediate": false, + "name" : "Rekurzív archiválás a POLC/FINISHED_SHOWS mappából", + "template": "archive-recursive.xml", + "cronexpression": "0 */1 * * * ?", + "parameters": [ + {"name": "sourcePath", "value": "/mnt/POLC/FINISHED_SHOWS", "type": "java.lang.String"}, + {"name": "killDateDays", "value": 1, "type": "java.lang.Integer"}, + {"name": "limit", "value": 1, "type": "java.lang.Integer"} + ] + }, + { + "active": false, + "executeimmediate": true, + "name" : "Limitált archiválás az PROMISE/ARCHIVE mappából", + "template": "archive-limited.xml", + "cronexpression": "0 */10 * * * ?", + "parameters": [ + {"name": "sourcePath", "value": "/mnt/PROMISE/ARCHIVE", "type": "java.lang.String"}, + {"name": "globalSourcePath", "value": "\\\\10.11.1.100\\PROMISE\\ARCHIVE", "type": "java.lang.String"}, + {"name": "transcoderTargetPath", "value": "/mnt/PROMISE/TRANSCODER/FFASTRANSCODER/Out", "type": "java.lang.String"}, + {"name": "killDateDays", "value": -1, "type": "java.lang.Integer"}, + {"name": "limit", "value": 10, "type": "java.lang.Integer"} + ] + }, + { + "active": false, + "executeimmediate": false, + "name" : "Párhuzamosított teszt folyamat", + "template": "fake-noparams.xml", + "cronexpression": "0 40 22 * * ?", + "parameters": [ {"name": "itemID", "value": 1, "type": "java.lang.Long"} ] + }, + { + "template": "cancelable.xml" + }, + { + "active": false, + "executeimmediate": false, + "name" : "HSM migrálás", + "template": "migrate-hsm.xml", + "cronexpression": "0 0 */1 * * ?", + "parameters": [ + {"name": "sourceLocation", "value": "", "type": "java.lang.String"}, + {"name": "targetLocation", "value": "/mediacube/data/ARCHIVE", "type": "java.lang.String"} + ] + }, + { + "active": false, + "executeimmediate": true, + "name" : "Limitált archiválás a /mediacube/data/ARCHIVE mappából", + "template": "archive-limited.xml", + "cronexpression": "0 */10 * * * ?", + "parameters": [ + {"name": "sourcePath", "value": "/mediacube/data/ARCHIVE", "type": "java.lang.String"}, + {"name": "globalSourcePath", "value": "\\\\10.11.1.90\\data\\ARCHIVE", "type": "java.lang.String"}, + {"name": "transcoderTargetPath", "value": "/mnt/PROMISE/TRANSCODER/FFASTRANSCODER/Out", "type": "java.lang.String"}, + {"name": "killDateDays", "value": -1, "type": "java.lang.Integer"}, + {"name": "limit", "value": 1000, "type": "java.lang.Integer"} + ] + }, + { + "active": false, + "executeimmediate": true, + "name" : "Lejárt /mediacube/data/ARCHIVE anyagok törlése", + "template": "delete-materials.xml", + "cronexpression": "0 */10 * * * ?", + "parameters": [ + {"name": "sourcePath", "value": "/mediacube/data/ARCHIVE", "type": "java.lang.String"} + ] + }, + { + "active": false, + "executeimmediate": true, + "name" : "Lejárt /mediacube/data/OMARCHIVE anyagok törlése", + "template": "delete-materials.xml", + "cronexpression": "0 */10 * * * ?", + "parameters": [ + {"name": "sourcePath", "value": "/mediacube/data/OMARCHIVE", "type": "java.lang.String"} + ] + }, + { + "active": false, + "executeimmediate": true, + "name" : "Lejárt /PROMISE/ARCHIVE anyagok törlése", + "template": "delete-materials.xml", + "cronexpression": "0 */10 * * * ?", + "parameters": [ + {"name": "sourcePath", "value": "/mnt/PROMISE/ARCHIVE", "type": "java.lang.String"} + ] + }, + { + "active": false, + "executeimmediate": false, + "name" : "Lejárt /POLC/FINISHED_SHOWS anyagok törlése", + "template": "delete-materials.xml", + "cronexpression": "0 * 11 * * ?", + "parameters": [ + {"name": "sourcePath", "value": "/mnt/POLC/FINISHED_SHOWS", "type": "java.lang.String"} + ] + }, + { + "name" : "SYS: batch-retrieve-ondemand", + "template": "batch-retrieve-ondemand.xml", + "active": false, + "executeimmediate": false + }, + { + "name" : "SYS: retrieve-ondemand", + "template": "retrieve-ondemand.xml", + "active": false, + "executeimmediate": false, + "parameters": [ + {"name": "globalRetrievePath", "value": "file://10.11.1.100", "type": "java.lang.String"}, + {"name": "localRetrievePath", "value": "/mnt/PROMISE/", "type": "java.lang.String"}, + {"name": "materialOutputFolder", "value": "PLAYOUT_NLE", "type": "java.lang.String"}, + {"name": "promoOutputFolder", "value": "PROMO_NLE", "type": "java.lang.String"}, + {"name": "advertisementOutputFolder", "value": "REKLAM_NLE", "type": "java.lang.String"}, + {"name": "octopusOutputFolder", "value": "OCTOPUS", "type": "java.lang.String"}, + {"name": "genericOutputFolder", "value": "ARCHIVE_RESTORE", "type": "java.lang.String"}, + {"name": "onlineOutputFolder", "value": "ONLINE", "type": "java.lang.String"}, + {"name": "killDateDays", "value": 7, "type": "java.lang.Integer"}, + {"name": "nexioAgency", "value": "ARCHIVE_RESTORE", "type": "java.lang.String"}, + {"name": "nexioPort", "value": 2098, "type": "java.lang.Integer"}, + {"name": "nexioUserName", "value": "administrator", "type": "java.lang.String"}, + {"name": "nexioPassword", "value": "system", "type": "java.lang.String"} + ] + }, + { + "name" : "SYS: common-copy", + "template": "common-copy.xml", + "active": false, + "executeimmediate": false + }, + { + "template": "create-lowres-ondemand.xml", + "active": false, + "executeimmediate": false, + "parameters": [ + {"name": "globalRetrievePath", "value": "file://10.11.1.90/data", "type": "java.lang.String"}, + {"name": "localRetrievePath", "value": "/mediacube/data", "type": "java.lang.String"}, + {"name": "localHiresPath", "value": "/mediacube/data", "type": "java.lang.String" }, + {"name": "globalHiresPath", "value": "\\\\10.11.1.90\\data", "type": "java.lang.String" }, + {"name": "localLowresPath", "value": "/mediacube/data/TRANSCODER_OUT", "type": "java.lang.String" }, + {"name": "transcoderAddress", "value": "http://10.11.1.111:65445/api/json/v1/", "type": "java.lang.String"}, + {"name": "transcoderTemplateName", "value": "MP4", "type": "java.lang.String"} + ] + }, + { + "active": false, + "executeimmediate": false, + "template": "cancelable.xml", + "parameters": [ + {"name": "param", "value": 1, "type": "java.lang.Integer"} + ] + } +]} diff --git a/server/-product/production/LOCAL/jobs/steps/ArchiveListBuilderStep.java b/server/-product/production/LOCAL/jobs/steps/ArchiveListBuilderStep.java new file mode 100644 index 00000000..b2321829 --- /dev/null +++ b/server/-product/production/LOCAL/jobs/steps/ArchiveListBuilderStep.java @@ -0,0 +1,167 @@ +package user.jobengine.server.steps; + +import java.io.File; +import java.io.IOException; +import java.nio.file.DirectoryStream; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.LinkedList; +import java.util.List; + +import org.apache.commons.lang.StringUtils; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.apache.logging.log4j.Marker; + +import user.jobengine.server.IJobEngine; +import user.jobengine.server.IJobRuntime; + +/** + * Az archivalhato mediak listazasa MediaFileWrapper objektumokban. A listazott media allomanyokat megjeloli .catched signal allomannyal, hogy a legkozelebbi + * listazas figyelmen kivul hagyja. + * + * @author robi + */ +public class ArchiveListBuilderStep extends JobStep { + private static final Logger logger = LogManager.getLogger(); + // private static final String UTF8 = "utf-8"; + private static final String STATUSFOLDER = ".STATUS"; + private static final String JSONEXT = ".json"; + private static final String CATCHEDEXT = ".catched"; + + public static final String ITEM_TITLE = "itemTitle"; + public static final String ITEM_HOUSEID = "itemHouseId"; + public static final String ITEM_DESCRIPTION = "itemDescription"; + public static final String MEDIA_HOUSEID = "mediaHouseId"; + public static final String MEDIA_TITLE = "mediaTitle"; + public static final String MEDIA_DESCRIPTION = "mediaDescription"; + public static final String MEDIA_TYPE = "mediaType"; + private static final String DURATION = "duration"; + private static final String EXISTING_MEDIAID = "existingMediaId"; + private static final String TAGS = "tags"; + + private Marker marker; + + private ArchiveItem createArchiveItem(Path jsonFilePath, Path mediaFilePath, Path catchedFilePath) { + ArchiveItem result = null; + try { + result = ArchiveItem.fromFile(jsonFilePath); + result.setMediaFile(mediaFilePath.toString()); + result.setCatchedFile(catchedFilePath.toString()); + } catch (Exception e) { + logger.catching(e); + } + + return result; + } + + private void createCatchedFile(Path catchedFilePath) { + try { + Files.createFile(catchedFilePath); + //Files.write(catchedFilePath, CATCHED.getBytes(UTF8), StandardOpenOption.CREATE); + } catch (Exception e) { + logger.catching(e); + } + } + + @StepEntry + public Object[] execute(String sourcePath, int limit, IJobEngine jobEngine, IJobRuntime jobRuntime) { + marker = jobRuntime.getSessionMarker(); + List archiveList = new LinkedList(); + DirectoryStream directoryStream = null; + try { + DirectoryStream stream = Files.newDirectoryStream(Paths.get(sourcePath)); + for (Path p : stream) { + processPathItem(p, archiveList); + } + } catch (Exception e) { + logger.catching(e); + logger.error(marker, "Az '{}' mappa elérése sikertelen. A rendszer hibaüzenete: {}", e.getMessage()); + } finally { + if (directoryStream != null) { + try { + directoryStream.close(); + } catch (IOException e) { + } + } + } + + if (limit > 0 && archiveList.size() > limit) { + archiveList = archiveList.subList(0, limit); + logger.info(marker, "A folyamat alkalmazza a beállított {} limitet.", limit); + } + + if (archiveList.size() == 0) + logger.info(marker, "Nincs archiválandó anyag."); + else + logger.info(marker, "Az archiváló folyamat {} új anyagot érzékelt.", archiveList == null ? 0 : archiveList.size()); + + for (ArchiveItem archiveItem : archiveList) { + createCatchedFile(Paths.get(archiveItem.getCatchedFile())); + } + + return new Object[] { archiveList }; + } + + private boolean processPathItem(Path mediaFilePath, final List archiveList) { + File mediaFile = mediaFilePath.toFile(); + + // if (mediaFile.length() > 0) + // return false; + + if (mediaFile.isDirectory()) { + return false; + } + + Path dotStorePath = Paths.get(mediaFilePath.getParent().toString(), STATUSFOLDER); + Path catchedFilePath = Paths.get(dotStorePath.toString(), mediaFile.getName() + CATCHEDEXT); + File catchedFile = catchedFilePath.toFile(); + if (catchedFile.exists()) { + logger.warn("{} file is already catched.", mediaFile.getName()); + return false; + } + + Path jsonFilePath = Paths.get(dotStorePath.toString(), mediaFile.getName() + JSONEXT); + File jsonFile = jsonFilePath.toFile(); + if (!jsonFile.exists()) { + logger.warn("{} has no json metadata.", mediaFile.getName()); + return false; + } + + ArchiveItem archiveItem = createArchiveItem(jsonFilePath, mediaFilePath, catchedFilePath); + + if (archiveItem == null) { + logger.warn("{} has no metadata specified.", mediaFile.getName()); + return false; + } + + if (StringUtils.isBlank(archiveItem.getItemHouseId())) { + logger.warn("{} has no Item HouseID specified in metadata.", mediaFile.getName()); + return false; + } + + if (StringUtils.isBlank(archiveItem.getItemTitle())) { + logger.warn("{} has no Item Title specified in metadata.", mediaFile.getName()); + return false; + } + + if (StringUtils.isBlank(archiveItem.getMediaHouseId())) { + logger.warn("{} has no Media HouseID specified in metadata.", mediaFile.getName()); + return false; + } + + if (StringUtils.isBlank(archiveItem.getMediaTitle())) { + logger.warn("{} has no Media Title specified in metadata.", mediaFile.getName()); + return false; + } + + //A tenyleges archivalast vesszuk elore + if (mediaFile.length() == 0) + archiveList.add(archiveItem); + else + archiveList.add(0, archiveItem); + //createCatchedFile(catchedFilePath); + return true; + } +} diff --git a/server/-product/production/LOCAL/jobs/steps/ArchiveMaterialSubmitStep.java b/server/-product/production/LOCAL/jobs/steps/ArchiveMaterialSubmitStep.java new file mode 100644 index 00000000..f11e3a52 --- /dev/null +++ b/server/-product/production/LOCAL/jobs/steps/ArchiveMaterialSubmitStep.java @@ -0,0 +1,53 @@ +package user.jobengine.server.steps; + +import java.io.File; +import java.util.List; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.apache.logging.log4j.Marker; + +import user.commons.ListUtils; +import user.jobengine.server.IJobEngine; +import user.jobengine.server.IJobRuntime; + +public class ArchiveMaterialSubmitStep extends JobStep { + private static final Logger logger = LogManager.getLogger(); + private static final String JOBTEMPLATE = "archive-material.xml"; + private static final String KILL_DATE_DAYS = "killDateDays"; + private static final String ARCHIVE = "Archiválás"; + private static final String ARCHIVE_ITEM = "archiveItem"; + private Marker marker; + + @StepEntry + public Object[] execute(List archiveList, int killDateDays, IJobEngine jobEngine, IJobRuntime jobRuntime) throws Exception { + marker = jobRuntime.getSessionMarker(); + if (archiveList == null || archiveList.size() == 0) + return null; + + if (jobRuntime.forkPrepare()) { + for (int i = 0; i < archiveList.size(); i++) { + ArchiveItem archiveItem = archiveList.get(i); + try { + IJobRuntime runtime = jobEngine.submit(jobRuntime, null, JOBTEMPLATE, ARCHIVE, + ListUtils.asMap(ARCHIVE_ITEM, archiveItem, KILL_DATE_DAYS, killDateDays)); + + int progress = (i + 1) * 100 / archiveList.size(); + setProgress(progress); + if (runtime == null) + throw new Exception("Submit returned null runtime"); + } catch (Exception e) { + logger.catching(e); + String fileName = new File(archiveItem.getMediaFile()).getName(); + logger.error(marker, "Az '{}' állomány archiválási kísérlete sikertelen. A rendszer üzenete: {}", fileName, e.getMessage()); + if (!archiveItem.removeCatchedFile()) + logger.error(marker, "Az '{}' állomány .catched jelző állománya nem törölhető.", fileName); + throw e; + } + } + } + jobRuntime.forkWaitComplete(); + return null; + } + +} diff --git a/server/-product/production/LOCAL/jobs/steps/ArchiveRecursive.java b/server/-product/production/LOCAL/jobs/steps/ArchiveRecursive.java new file mode 100644 index 00000000..6a65d5fa --- /dev/null +++ b/server/-product/production/LOCAL/jobs/steps/ArchiveRecursive.java @@ -0,0 +1,291 @@ +package user.jobengine.server.steps; + +import java.io.File; +import java.io.IOException; +import java.nio.file.FileVisitResult; +import java.nio.file.FileVisitor; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.nio.file.attribute.BasicFileAttributes; +import java.text.SimpleDateFormat; +import java.util.Arrays; +import java.util.List; +import java.util.Map; + +import org.apache.commons.io.FilenameUtils; +import org.apache.commons.lang.StringUtils; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + +import com.ibm.nosql.json.api.BasicDBObject; + +import user.commons.JobStatus; +import user.commons.ListUtils; +import user.commons.mediatool.MediaInfo; +import user.jobengine.server.IJobEngine; +import user.jobengine.server.IJobRuntime; +import user.mediacube.metadata.interfaces.IMetadata; +import user.mediacube.metadata.interfaces.IMetadataProvider; +import user.mediacube.metadata.interfaces.IMetadataProviderFactory; +import user.mediacube.metadata.interfaces.MetadataProviderType; +import user.mediacube.metadata.interfaces.MetadataType; +import user.mediacube.metadata.interfaces.PlanAirMetadataListOptions; + +public class ArchiveRecursive extends JobStep implements FileVisitor { + private static final Logger logger = LogManager.getLogger(); + private static final String JOBTEMPLATE = "archive-material.xml"; + private static final String ITEM_TITLE = "itemTitle"; + private static final String ITEM_HOUSEID = "itemHouseId"; + private static final String MEDIA_HOUSEID = "mediaHouseId"; + private static final String MEDIA_TITLE = "mediaTitle"; + private static final String MEDIA_DESCRIPTION = "mediaDescription"; + private static final String MEDIA_TYPE = "mediaType"; + private static final String ARCHIVE = "Archiválás"; + private static final String ARCHIVE_ITEM = "archiveItem"; + private static final String KILL_DATE_DAYS = "killDateDays"; + private SimpleDateFormat df = new SimpleDateFormat("yyyy.MM.dd"); + + private List skipPathNames = Arrays.asList("!ARCHIVALAS_ALATT", "VASAROLT", EscortFiles.STATUSFOLDER, EscortFiles.CONFLICTFOLDER); + private int limit; + private int submitted; + private int killDateDays; + + private boolean canReadMediaInfo(Path mediaFilePath) { + boolean result = false; + try { + MediaInfo mi = new MediaInfo(mediaFilePath); + mi.process(); + result = true; + } catch (Exception e) { + logger.warn(getSessionMarker(), e.getMessage()); + } + return result; + } + + private void checkArchiveItem(ArchiveItem archiveItem, Path mediaPath) throws Exception { + if (archiveItem == null) + throw new Exception(mediaPath + " has no metadata specified."); + + if (StringUtils.isBlank(archiveItem.getItemHouseId())) + throw new Exception(mediaPath + " has no Item HouseID specified in metadata."); + + if (StringUtils.isBlank(archiveItem.getItemTitle())) + throw new Exception(mediaPath + " has no Item Title specified in metadata."); + + if (StringUtils.isBlank(archiveItem.getMediaHouseId())) + throw new Exception(mediaPath + " has no Media HouseID specified in metadata."); + + if (StringUtils.isBlank(archiveItem.getMediaTitle())) + throw new Exception(mediaPath + " has no Media Title specified in metadata."); + } + + private ArchiveItem createArchiveItem(Path filePath) throws Exception { + ArchiveItem result = null; + + String fileName = filePath.getFileName().toString(); + String mediaHouseId = FilenameUtils.removeExtension(fileName); + try { + result = getPlanAirMetadata(mediaHouseId); + } catch (Exception e) { + + logger.error("PlanAir metadata error", e); + //nem latja a drivert pl. + throw e; + } + + if (result == null) { + result = new ArchiveItem(); + BasicFileAttributes attr = Files.readAttributes(filePath, BasicFileAttributes.class); + result.setItemHouseId(df.format(attr.lastModifiedTime().toMillis())); + result.setItemTitle(filePath.getParent().toString()); + result.setMediaHouseId(mediaHouseId); + result.setMediaTitle(fileName); + //TODO kesobb kivenni + //result.setMediaDescription("/OMARCHIVE"); + result.setMediaType("Generic"); + } + + result.setMediaFile(filePath.toString()); + return result; + } + + @StepEntry + public Object[] execute(String sourcePath, int killDateDays, int limit) throws Exception { + this.killDateDays = killDateDays; + this.limit = limit; + try { + if (getJobRuntime().forkPrepare()) { + Files.walkFileTree(Paths.get(sourcePath), this); + } + } catch (Exception e) { + logger.error(getSessionMarker(), "Az '{}' mappa elérése sikertelen. A rendszer hibaüzenete: {}", sourcePath, e.getMessage()); + } finally { + if (submitted > 0) + getJobRuntime().forkWaitComplete(); + else + getJobRuntime().cancelForkPrepare(); + + } + return null; + } + + private ArchiveItem getPlanAirMetadata(String mediaHouseId) throws Exception { + PlanAirMetadataListOptions opt = new PlanAirMetadataListOptions(); + opt.setSearch(mediaHouseId); + opt.setType(MetadataType.Material); + + BasicDBObject json = null; + List data = null; + + IMetadataProviderFactory factory = getService(IMetadataProviderFactory.class); + IMetadataProvider planairProvider = factory.getProvider(MetadataProviderType.PLANAIR); + + ArchiveItem result = null; + data = planairProvider.list(opt); + if (data.size() != 0) + json = data.get(0).asJSON(); + else { + opt.setType(MetadataType.Promo); + data = planairProvider.list(opt); + if (data.size() != 0) + json = data.get(0).asJSON(); + else { + opt.setType(MetadataType.AD); + data = planairProvider.list(opt); + if (data.size() != 0) + json = data.get(0).asJSON(); + } + } + if (json != null) { + result = new ArchiveItem(); + result.setItemHouseId(json.getString(ITEM_HOUSEID)); + result.setItemTitle(json.getString(ITEM_TITLE)); + result.setMediaHouseId(json.getString(MEDIA_HOUSEID)); + result.setMediaTitle(json.getString(MEDIA_TITLE)); + result.setMediaDescription(json.getString(MEDIA_DESCRIPTION)); + result.setMediaType(json.getString(MEDIA_TYPE)); + } + return result; + } + + private boolean handleArchiveConflict(Path mediaPath) throws Exception { + boolean result = false; + String sourceFileName = mediaPath.getFileName().toString(); + if (getManager().isMediaFileExists(sourceFileName)) { + EscortFiles.createMediaCatch(mediaPath); + result = true; + + // try { + // Path parent = sourcePath.getParent(); + // Path conflictPath = Paths.get(parent.toString(), CONFLICT); + // File folder = conflictPath.toFile(); + // if (!folder.exists() || !folder.isDirectory()) { + // Set perms = PosixFilePermissions.fromString("rwxrwxrwx"); + // FileAttribute> attr = PosixFilePermissions.asFileAttribute(perms); + // try { + // Files.createDirectories(conflictPath, attr); + // } catch (Exception e) { + // try { + // Files.createDirectory(conflictPath); + // } catch (Exception e1) { + // logger.catching(e); + // throw e; + // } + // } + // } + // + // Path conflictFile = Paths.get(conflictPath.toString(), sourceFileName); + // if (conflictFile.toFile().exists()) + // conflictFile = Paths.get(conflictPath.toString(), sourceFileName + (new Date()).getTime()); + // Files.move(sourcePath, conflictFile); + // result = true; + // } catch (Exception e1) { + // logger.error(getSessionMarker(), "Hiba az '{}' állomány mappába mozgatásakor. A rendszer üzenete: {}", CONFLICT, e1.getMessage()); + // } + } + + return result; + } + + @Override + public FileVisitResult postVisitDirectory(Path paramT, IOException paramIOException) throws IOException { + return FileVisitResult.CONTINUE; + } + + @Override + public FileVisitResult preVisitDirectory(Path dir, BasicFileAttributes paramBasicFileAttributes) throws IOException { + Path dirName = dir.getFileName(); + logger.info("PreVisit {}", dir); + + if (skipPathNames.contains(dirName.toString())) + return FileVisitResult.SKIP_SUBTREE; + else + logger.info("PreVisit {}", dir); + + return FileVisitResult.CONTINUE; + } + + private boolean processPathItem(Path mediaPath) throws Exception { + if (submitted == limit) + return false; + + // if (!"Transfered".equals(mediaPath.getParent().getFileName().toString())) + // return false; + + if (mediaPath.getFileName().toString().startsWith(".")) + return false; + + File mediaFile = mediaPath.toFile(); + if (mediaFile.isDirectory()) + return false; + + if (EscortFiles.isMediaCatched(mediaPath)) + return false; + + if (handleArchiveConflict(mediaPath)) + return false; + + if (!canReadMediaInfo(mediaPath)) + return false; + + ArchiveItem archiveItem = createArchiveItem(mediaPath); + + try { + checkArchiveItem(archiveItem, mediaPath); + submitted++; + + Map parameters = ListUtils.asMap(ARCHIVE_ITEM, archiveItem, KILL_DATE_DAYS, killDateDays); + IJobRuntime runtime = getEngine().submit(getJobRuntime(), e -> { + if (e.getStatus().equals(JobStatus.CANCELED) || e.getStatus().equals(JobStatus.SUSPENDED)) + EscortFiles.removeMediaCatch(mediaPath); + }, JOBTEMPLATE, ARCHIVE, 1, IJobEngine.DEFAULT_OWNER, parameters); + if (runtime == null) + throw new Exception("Submit returned null runtime"); + runtime.setRelated(mediaPath.toString()); + EscortFiles.createMediaCatch(mediaPath); + } catch (Exception e) { + String fileName = new File(archiveItem.getMediaFile()).getName(); + logger.error(getSessionMarker(), "Az '{}' állomány archiválási kísérlete sikertelen. A rendszer üzenete: {}", fileName, e.getMessage()); + } + + return true; + } + + @Override + public FileVisitResult visitFile(Path filePath, BasicFileAttributes paramBasicFileAttributes) throws IOException { + //logger.info("Will archived {}", filePath); + try { + processPathItem(filePath); + } catch (Exception e) { + logger.catching(e); + } + return FileVisitResult.CONTINUE; + } + + @Override + public FileVisitResult visitFileFailed(Path filePath, IOException paramIOException) throws IOException { + logger.info("Error archive {}", filePath); + return FileVisitResult.CONTINUE; + } +} diff --git a/server/-product/production/LOCAL/jobs/steps/BatchRetrieveForkStep.java b/server/-product/production/LOCAL/jobs/steps/BatchRetrieveForkStep.java new file mode 100644 index 00000000..15cf36c1 --- /dev/null +++ b/server/-product/production/LOCAL/jobs/steps/BatchRetrieveForkStep.java @@ -0,0 +1,76 @@ +package user.jobengine.server.steps; + +import java.util.List; +import java.util.Map; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + +import user.commons.MediaCubeFinishMarker; +import user.commons.MediaCubeMarker; +import user.jobengine.db.ArchivedMedia; +import user.jobengine.server.IJobEngine; +import user.jobengine.server.IJobRuntime; +import user.jobengine.server.scheduler.ScheduledJob; + +public class BatchRetrieveForkStep extends JobStep { + private static final String TARGET_PATH_TYPE = "targetPathType"; + private static final Logger logger = LogManager.getLogger(); + private static final String CHILD_TEMPLATE = "retrieve-ondemand.xml"; + private static final String ARCHIVEDMEDIA = "archivedMedia"; + private static final String RECIPIENT = "successRecipient"; + private static final String HOUSEID = "houseId"; + private MediaCubeMarker marker; + + @StepEntry + public Object[] execute(List basket, String houseId, String recipient, String targetPathType, IJobEngine jobEngine, IJobRuntime jobRuntime) + throws Exception { + marker = (MediaCubeMarker) jobRuntime.getSessionMarker(); + + //session szinten csak a finishMarker cimzettje az erdekes, es ezt a cimet pluszban hasznalja a konfigban megadott cimmel + //a finishMarker orokli a cim bellitast a sessionMarkertol + marker.setTo(recipient); + + ((MediaCubeMarker) jobRuntime.getFinishMarker()).setTo(recipient); + + if (basket == null || basket.size() == 0) + return null; + setProgress(10); + + MediaCubeMarker mailMarker = new MediaCubeMarker(recipient); + mailMarker.setSessionName("Archívum viszatöltés"); + mailMarker.setSessionID(houseId); + logger.info(mailMarker, "A visszatöltések elindultak az alábbi állományokra:"); + + if (jobRuntime.forkPrepare()) { + for (ArchivedMedia archivedMedia : basket) { + logger.info(mailMarker, archivedMedia.getMedia().getMediaFilesName()); + submit(archivedMedia, recipient, houseId, targetPathType, jobEngine, jobRuntime); + } + } + setProgress(50); + logger.info(new MediaCubeFinishMarker(mailMarker), "A visszatöltések végeztével megerősítő üzenetet küldünk."); + jobRuntime.forkWaitComplete(); + setProgress(100); + return null; + } + + public void submit(ArchivedMedia archivedMedia, String recipient, String houseId, String targetPathType, IJobEngine jobEngine, IJobRuntime jobRuntime) + throws Exception { + try { + ScheduledJob scheduledJob = jobEngine.getScheduledJob(CHILD_TEMPLATE); + Map parameters = scheduledJob.getJobParameters(); + parameters.put(ARCHIVEDMEDIA, archivedMedia); + parameters.put(HOUSEID, houseId); + parameters.put(RECIPIENT, recipient); + parameters.put(TARGET_PATH_TYPE, targetPathType); + IJobRuntime child = jobEngine.submit(jobRuntime, null, CHILD_TEMPLATE, String.format("Visszatöltés %s részére", recipient), parameters); + ((MediaCubeMarker) child.getSessionMarker()).setTo(recipient); + } catch (Exception e) { + logger.catching(e); + logger.error(marker, "Hiba a kötegelt visszatöltésben. A rendszer üzenete: {}", e.getMessage()); + } + + } + +} diff --git a/server/-product/production/LOCAL/jobs/steps/CancelableStep.java b/server/-product/production/LOCAL/jobs/steps/CancelableStep.java new file mode 100644 index 00000000..0ff5eca4 --- /dev/null +++ b/server/-product/production/LOCAL/jobs/steps/CancelableStep.java @@ -0,0 +1,46 @@ +package user.jobengine.server.steps; + +import org.apache.commons.net.ftp.FTPClient; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.apache.logging.log4j.Marker; +import org.apache.logging.log4j.MarkerManager; + +import user.commons.StoreUri; +import user.commons.remotestore.FtpDirectoryLister; +import user.commons.remotestore.RemoteStoreProtocol; + +public class CancelableStep extends JobStep { + private static final Logger logger = LogManager.getLogger(); + int count = 10; + + @StepEntry + public Object[] execute(int param) throws Exception { + try { + + Marker marker = MarkerManager.getMarker("MEDIAPROFILE"); + getJobRuntime().setRelated("TESZT" + param); + + for (int i = 0; i < count; i++) { + if (getJobRuntime().isWaitingCancel()) + break; + Thread.sleep(500); + /* + if (param==0) + throw new Exception(); + if (param==1) { + getJobRuntime().setDisableRetry(true); + throw new Exception(); + } + */ + int progress = (i + 1) * 100 / count; + setProgress(progress); + } + } catch (Exception e) { + logger.info(e.getMessage()); + throw e; + } + return null; + } + +} diff --git a/server/-product/production/LOCAL/jobs/steps/CleanupMountedLocationStep.java b/server/-product/production/LOCAL/jobs/steps/CleanupMountedLocationStep.java new file mode 100644 index 00000000..4e6f10d1 --- /dev/null +++ b/server/-product/production/LOCAL/jobs/steps/CleanupMountedLocationStep.java @@ -0,0 +1,316 @@ +package user.jobengine.server.steps; + +import java.io.File; +import java.io.IOException; +import java.nio.file.DirectoryStream; +import java.nio.file.FileVisitResult; +import java.nio.file.FileVisitor; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.nio.file.SimpleFileVisitor; +import java.nio.file.attribute.BasicFileAttributes; +import java.text.ParseException; +import java.text.SimpleDateFormat; +import java.util.ArrayList; +import java.util.Collections; +import java.util.Date; +import java.util.List; + +import org.apache.commons.io.FileUtils; +import org.apache.commons.lang.StringUtils; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.apache.logging.log4j.Marker; + +import user.commons.RemoteFile; +import user.commons.StoreUri; +import user.commons.remotestore.RemoteStoreProtocol; +import user.jobengine.db.Store; +import user.jobengine.server.IJobEngine; +import user.jobengine.server.IJobRuntime; + +public class CleanupMountedLocationStep extends JobStep implements FileVisitor { + + private static final Logger logger = LogManager.getLogger(); + private static final String PROJECTFOLDER = "PROJECT"; + private static final String DATEFORMAT = "yyyyMMdd"; + private static final String DOT = "."; + private static final String STATUSFOLDER = ".STATUS"; + private static final String EWC2EXT = ".ewc2"; + private static final String XMPEXT = ".xmp"; + private static final String CATCHEDEXT = ".catched"; + private static final String KILLDATEEXT = ".killdate"; + private static final String JSONEXT = ".json"; + + private static boolean isEmpty(final Path directory) throws IOException { + try (DirectoryStream dirStream = Files.newDirectoryStream(directory)) { + final int[] count = new int[] { 0 }; + final int[] specialCount = new int[] { 0 }; + dirStream.forEach(p -> { + count[0]++; + // if (p.getFileName().toString().toLowerCase().equals(PROJECTFOLDER.toLowerCase())) + // specialCount[0]++; + if (p.getFileName().toString().toLowerCase().equals(STATUSFOLDER.toLowerCase())) + specialCount[0]++; + + }); + if (specialCount[0] == count[0]) + return true; + } + return false; + } + + private Marker marker; + + final int[] allCount = new int[] { 0 }; + final int[] currentCount = new int[] { 0 }; + + private Path sourcePath; + private SimpleDateFormat dateFormat; + private StoreUri tsmStoreUri; + + private Date checkExpiration(List killDateFiles) { + Date killDate = null; + for (Path killDateFile : killDateFiles) { + Date currentKillDate = getKillDate(killDateFile); + if (currentKillDate == null) + continue; + if ((killDate != null && currentKillDate.after(killDate)) || killDate == null) + killDate = currentKillDate; + } + return new Date().after(killDate) ? killDate : null; + } + + @StepEntry + public Object[] execute(String sourceFolder, IJobEngine jobEngine, IJobRuntime jobRuntime) throws Exception { + marker = getSessionMarker(); + sourcePath = Paths.get(sourceFolder); + DirectoryStream directoryStream = null; + if (StringUtils.isBlank(sourcePath.toString())) { + logger.error(marker, "A folyamat 'sourcePath' bemeneti paramétere üres."); + throw new NullPointerException("System is not configured properly, 'sourceFolder' input parameter missing."); + } + + if (!sourcePath.toFile().exists() || !sourcePath.toFile().isDirectory()) { + logger.error(marker, "A {} mappa nem létezik.", sourceFolder); + throw new NullPointerException(String.format("Directory %s not exists.", sourceFolder)); + } + + Store tsmStore = getManager().getSystemStore(false); + if (tsmStore == null) + throw new NullPointerException("A TSM bejegyzés nem található!"); + + tsmStoreUri = tsmStore.getSourceStoreUri(RemoteStoreProtocol.TSM); + if (tsmStoreUri == null) + throw new NullPointerException("A TSM forrás elérése nem található!"); + + try { + setProgress(1); + dateFormat = new SimpleDateFormat(DATEFORMAT); + + Files.walkFileTree(sourcePath, new SimpleFileVisitor() { + @Override + public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) throws IOException { + allCount[0]++; + return super.visitFile(file, attrs); + } + }); + Files.walkFileTree(sourcePath, this); + } catch (Exception e) { + logger.catching(e); + logger.error(marker, "Hiba a '{}' mappa feldolgozásában. A rendszer hibaüzenete: {}", sourcePath, e.getMessage()); + throw e; + } finally { + if (directoryStream != null) { + try { + directoryStream.close(); + } catch (IOException e) { + } + } + } + return null; + } + + private Date getKillDate(Path killDateFile) { + String fileName = killDateFile.getFileName().toString(); + int end = fileName.lastIndexOf(DOT); + if (end < 1) + return null; + int start = fileName.lastIndexOf(DOT, end - 1); + if (start < 0) + return null; + String strKillDate = fileName.substring(start + 1, end); + Date result = null; + if (StringUtils.isNumeric(strKillDate)) { + try { + result = dateFormat.parse(strKillDate); + } catch (ParseException e) { + logger.error(marker, "A {} fájl 'killdate' állománya hibás formátumú, a {} karaktersorozat nem konvertálható dátummá.", fileName, strKillDate); + return null; + } + } else + logger.error(marker, "A {} fájl 'killdate' állománya hibás formátumú, az dátum helyett ez áll: '{}'.", fileName, strKillDate); + return result; + } + + private List getKillDateFiles(Path filePath) { + String killDateFilePattern = String.format("%s.*%s", filePath.getFileName().toString(), KILLDATEEXT); + List result = new ArrayList<>(); + Path statusPath = null; + try { + statusPath = Paths.get(filePath.getParent().toString(), STATUSFOLDER); + } catch (Exception e) { + logger.catching(e); + return null; + } + File statusPathFile = statusPath.toFile(); + if (statusPathFile.exists() && statusPathFile.isDirectory()) { + try (DirectoryStream stream = Files.newDirectoryStream(statusPath, killDateFilePattern)) { + stream.forEach(p -> result.add(p)); + } catch (Exception e) { + logger.catching(e); + } + } + Collections.sort(result); + return result; + } + + private boolean isArchived(Path filePath) { + boolean result = false; + String name = filePath.getFileName().toString(); + String[] tsmName = new String[] { null }; + String query = String.format("SELECT relativepath FROM MEDIAFILE WHERE houseid = '%s'", name); + getManager().executeQuery(query, rs -> { + tsmName[0] = rs.getString("relativepath"); + return false; + }, null); + + if (tsmName[0] != null) { + try { + RemoteFile remoteFile = tsmStoreUri.getRemoteFile(tsmName[0]); + result = remoteFile != null; + } catch (Exception e) { + logger.error(e.getMessage()); + } finally { + tsmStoreUri.cleanUp(); + } + } + return result; + } + + @Override + public FileVisitResult postVisitDirectory(Path dir, IOException exc) throws IOException { + if (!dir.equals(sourcePath) && isEmpty(dir)) { + if (!removeExistingSpecialDirectory(dir, PROJECTFOLDER)) + return FileVisitResult.CONTINUE; + if (!removeExistingSpecialDirectory(dir, STATUSFOLDER)) + return FileVisitResult.CONTINUE; + if (removeFile(dir)) + logger.info(marker, "A {} üres mappa törlése sikeres.", dir); + + } + return FileVisitResult.CONTINUE; + } + + @Override + public FileVisitResult preVisitDirectory(Path dir, BasicFileAttributes attrs) throws IOException { + + //A .-al kezdodo mappakat kihagyjuk + if (dir.getFileName().toString().startsWith(".")) + return FileVisitResult.SKIP_SUBTREE; + + return FileVisitResult.CONTINUE; + } + + private void processPathItem(Path filePath) { + currentCount[0]++; + + if (filePath.getFileName().toString().startsWith(".")) + return; + + int progress = currentCount[0] * 100 / allCount[0]; + setProgress(progress); + + logger.info("Checking {}", filePath); + List killDateFiles = getKillDateFiles(filePath); + if (killDateFiles == null || killDateFiles.size() == 0) { + logger.warn(marker, "A {} fájlhoz nem található 'killdate' állomány.", filePath); + return; + } + + if (killDateFiles.size() != 1) + logger.warn(marker, "A {} fájlhoz több 'killdate' állomány található, a legújabb dátum határozza meg a törlés időpontját.", filePath); + + Date killDate = checkExpiration(killDateFiles); + if (killDate == null) + return; + + if (!isArchived(filePath)) { + logger.error(marker, "A(z) {} anyag törlésre van kijelölve, de nem található az archívumban.", filePath); + return; + } + + if (removeFiles(filePath, killDateFiles)) + logger.info(marker, "A {} fájl és kapcsolódó állományai a {} killdate bejegyzés alapján sikeresen törlődtek.", filePath.getFileName(), + dateFormat.format(killDate)); + else + logger.warn(marker, "A {} fájl és kapcsolódó állományai a {} killdate bejegyzés alapján csak részlegesen vagy egyáltalán nem törlődtek.", + filePath.getFileName(), dateFormat.format(killDate)); + } + + private boolean removeExistingSpecialDirectory(Path dir, String folderName) throws IOException { + File projectPath = Paths.get(dir.toString(), folderName).toFile(); + if (projectPath.exists() && projectPath.isDirectory()) { + FileUtils.deleteDirectory(projectPath); + if (projectPath.exists()) { + logger.warn(marker, "A {} alatti {} mappa törlése nem sikerült.", dir, folderName); + return false; + } + } + return true; + } + + private boolean removeFile(Path filePath) { + boolean result = false; + try { + //logger.error("REMOVE {}", filePath); + File file = filePath.toFile(); + if (file.exists()) + result = file.delete(); + } catch (Exception e) { + logger.error(marker, "A {} fájl nem törölhető. A rendszer hibaüzenete: {}", filePath, e.getMessage()); + } + return result; + } + + private boolean removeFiles(Path filePath, List killDateFiles) { + if (!removeFile(filePath)) + return false; + + removeFile(Paths.get(filePath.toString() + EWC2EXT)); + removeFile(Paths.get(filePath.toString() + XMPEXT)); + removeFile(Paths.get(filePath.getParent().toString(), STATUSFOLDER, filePath.getFileName().toString() + CATCHEDEXT)); + removeFile(Paths.get(filePath.getParent().toString(), STATUSFOLDER, filePath.getFileName().toString() + JSONEXT)); + + boolean result = true; + for (Path killDateFile : killDateFiles) { + if (!removeFile(killDateFile)) + result = false; + } + + return result; + } + + @Override + public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) throws IOException { + processPathItem(file); + return FileVisitResult.CONTINUE; + } + + @Override + public FileVisitResult visitFileFailed(Path file, IOException e) throws IOException { + logger.error(marker, "A {} fájl nem érhető el. A rendszer hibaüzenete: {}", file.toString(), e.getMessage()); + return FileVisitResult.CONTINUE; + } +} diff --git a/server/-product/production/LOCAL/jobs/steps/CreateArchiveItemStep.java b/server/-product/production/LOCAL/jobs/steps/CreateArchiveItemStep.java new file mode 100644 index 00000000..b169edf5 --- /dev/null +++ b/server/-product/production/LOCAL/jobs/steps/CreateArchiveItemStep.java @@ -0,0 +1,45 @@ +package user.jobengine.server.steps; + +import java.nio.file.Paths; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + +import com.ibm.nosql.json.api.BasicDBObject; +import com.ibm.nosql.json.api.DB; +import com.ibm.nosql.json.api.DBCollection; + +import user.commons.nosql.NoSQLUtils; +import user.jobengine.db.Media; +import user.jobengine.db.Store; + +public class CreateArchiveItemStep extends JobStep { + private static final Logger logger = LogManager.getLogger(); + + @StepEntry + public Object[] execute(Media mediaCubeMedia, String localHiresPath) throws Exception { + DB db = NoSQLUtils.getNoSQLDB(); + DBCollection collection = db.getCollection("missing_lowres"); + Store highResStore = getManager().getSystemStore(false); + + ArchiveItem archiveItem = null; + try { + if (mediaCubeMedia.getMediaFilesCount() != 1) + throw new Exception("Expected media count is 1, found " + mediaCubeMedia.getMediaFilesCount()); + if (mediaCubeMedia.getMediaFiles().get(0).getStoreId() != highResStore.getId()) + throw new Exception("Expected media store is a high-res store"); + + String name = mediaCubeMedia.getMediaFileRealName(); + archiveItem = new ArchiveItem(); + archiveItem.setMediaFile(Paths.get(localHiresPath, name).toString()); + collection.save(new BasicDBObject("name", name)); + } catch (Exception e) { + logger.catching(e); + logger.info(getMarker(), e.getMessage()); + throw e; + } finally { + setProgress(100); + } + return new Object[] { archiveItem }; + } +} diff --git a/server/-product/production/LOCAL/jobs/steps/CreateMissingLowresStep.java b/server/-product/production/LOCAL/jobs/steps/CreateMissingLowresStep.java new file mode 100644 index 00000000..19bad479 --- /dev/null +++ b/server/-product/production/LOCAL/jobs/steps/CreateMissingLowresStep.java @@ -0,0 +1,84 @@ +package user.jobengine.server.steps; + +import java.nio.file.Paths; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + +import com.ibm.nosql.json.api.BasicDBObject; +import com.ibm.nosql.json.api.DB; +import com.ibm.nosql.json.api.DBCollection; + +import user.commons.MediaCubeUndoMarker; +import user.commons.nosql.NoSQLUtils; +import user.jobengine.db.IItemManager; +import user.jobengine.db.Media; +import user.jobengine.server.IJobEngine; +import user.jobengine.server.IJobRuntime; + +public class CreateMissingLowresStep extends JobStep { + private static final Logger logger = LogManager.getLogger(); + + @StepEntry + public Object[] execute(String localHiresPath, IJobEngine jobEngine, IJobRuntime jobRuntime) throws Exception { + Object[] result = new Object[] { null, null, "%s", null, 0, true }; + DB db = NoSQLUtils.getNoSQLDB(); + DBCollection collection = db.getCollection("missing_lowres"); + IItemManager manager = jobEngine.getItemManager(); + Media media = getFirstUntranscodedMedia(manager, collection); + + try { + if (media == null) { + logger.info(new MediaCubeUndoMarker(getSessionMarker().getSessionID()), "Nincs feldolgozandó hiány."); + // throw new Exception("Nincs feldolgozandó hiány."); + cancel(); + return null; + } + + String name = media.getMediaFileRealName(); + result[0] = media; + ArchiveItem archiveItem = new ArchiveItem(); + archiveItem.setMediaFile(Paths.get(localHiresPath, name).toString()); + result[1] = archiveItem; + collection.save(new BasicDBObject("name", name)); + logger.info(getSessionMarker(), "Processing mediaID: {}", media.getId()); + + } catch (Exception e) { + logger.catching(e); + logger.error(getSessionMarker(), e.getMessage()); + throw e; + } finally { + setProgress(100); + } + return result; + } + + private Media getFirstUntranscodedMedia(IItemManager manager, DBCollection collection) { + Media[] result = new Media[] { null }; + //MV + String query = "SELECT mediaid FROM VW_MISSING_PROXY_IDS WHERE HOUSEID like 'M%' or HOUSEID like 'P%' or HOUSEID like 'R%' ORDER BY modified DESC"; + + //HTV + //String query = "SELECT mediaid FROM VW_MISSING_PROXY_IDS"; + manager.executeQuery(query, rs -> { + try { + long mediaId = rs.getLong(1); + Media media = manager.getMedia(mediaId); + //a nevgeneralas miatt az eredeti MediaFilesName nem jo, a pontos nev kell nekunk + String name = media.getMediaFileRealName(); + logger.info(getSessionMarker(), "Checking {}", name); + long existing = collection.find(new BasicDBObject("name", name)).count(); + if (existing > 0) { + logger.info(getSessionMarker(), "{} is on missing_lowres list", name); + return true; + } + + result[0] = media; + } catch (Exception e) { + logger.error(e); + } + return false; + }, null); + return result[0]; + } +} diff --git a/server/-product/production/LOCAL/jobs/steps/FileCopyStep.java b/server/-product/production/LOCAL/jobs/steps/FileCopyStep.java new file mode 100644 index 00000000..c4172550 --- /dev/null +++ b/server/-product/production/LOCAL/jobs/steps/FileCopyStep.java @@ -0,0 +1,95 @@ +package user.jobengine.server.steps; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.apache.logging.log4j.Marker; + +import user.commons.StoreUri; +import user.commons.remotestore.IProgressEventListener; +import user.commons.remotestore.ProgressEvent; +import user.commons.remotestore.RemoteStoreProtocol; +import user.jobengine.db.IItemManager; +import user.jobengine.server.IJobEngine; +import user.jobengine.server.IJobRuntime; + +public class FileCopyStep extends JobStep { + private static final Logger logger = LogManager.getLogger(); + private IItemManager manager; + private Marker marker; + + private void check(String sourceProtocol, String sourcePath, String sourceFileName, String targetProtocol, String targetPath, String targetFileName, + IJobEngine jobEngine, IJobRuntime jobRuntime) { + if (jobEngine == null) { + logger.error(marker, "A folyamatkezelő réteg nem elérhető."); + throw new NullPointerException("Internal error, missing JobEngine reference."); + } + manager = jobEngine.getItemManager(); + if (manager == null) { + logger.error(marker, "Az adatbáziskezelő réteg nem elérhető."); + throw new NullPointerException("Internal error, missing ItemManager reference."); + } + if (sourceProtocol == null) { + logger.error(marker, "A forrás protokol bemeneti paramétere üres."); + throw new NullPointerException("System is not configured properly, 'sourceProtocol' input parameter missing."); + } + if (sourcePath == null) { + logger.error(marker, "A forrás fájl elérés bemeneti paramétere üres."); + throw new NullPointerException("System is not configured properly, 'sourcePath' input parameter missing."); + } + if (sourceFileName == null) { + logger.error(marker, "A forrás fájlnév bemeneti paramétere üres."); + throw new NullPointerException("System is not configured properly, 'sourceFileName' input parameter missing."); + } + if (targetProtocol == null) { + logger.error(marker, "A cél protokol bemeneti paramétere üres."); + throw new NullPointerException("System is not configured properly, 'targetProtocol' input parameter missing."); + } + if (targetPath == null) { + logger.error(marker, "A cél fájl elérés bemeneti paramétere üres."); + throw new NullPointerException("System is not configured properly, 'targetPath' input parameter missing."); + } + if (targetFileName == null) { + logger.error(marker, "A cél fájlnév bemeneti paramétere üres."); + throw new NullPointerException("System is not configured properly, 'targetFileName' input parameter missing."); + } + } + + @StepEntry + public Object[] execute(String sourceProtocol, String sourcePath, String sourceFileName, String targetProtocol, String targetPath, String targetFileName, + int killDateDays, IJobEngine jobEngine, IJobRuntime jobRuntime) throws Exception { + marker = jobRuntime.getSessionMarker(); + + check(sourceProtocol, sourcePath, sourceFileName, targetProtocol, targetPath, targetFileName, jobEngine, jobRuntime); + + StoreUri source = null; + StoreUri target = null; + + try { + source = manager.createStoreUri(Enum.valueOf(RemoteStoreProtocol.class, sourceProtocol), sourcePath); + target = manager.createStoreUri(Enum.valueOf(RemoteStoreProtocol.class, targetProtocol), targetPath); + source.addProgressListener(new IProgressEventListener() { + @Override + public void progressChanged(ProgressEvent evt) { + jobRuntime.incrementProgress(evt.getProgress()); + } + }); + + source.transferFrom(target, sourceFileName, targetFileName); + if (killDateDays > -1) + EscortFiles.createUNCKillDate(targetPath, targetFileName, killDateDays, marker); + + } catch (Exception e) { + logger.catching(e); + throw e; + } finally { + if (source != null) { + source.cleanUp(); + } + if (target != null) { + target.cleanUp(); + } + } + + return null; + } +} diff --git a/server/-product/production/LOCAL/jobs/steps/FileSearchFilterOptions.java b/server/-product/production/LOCAL/jobs/steps/FileSearchFilterOptions.java new file mode 100644 index 00000000..436c3be0 --- /dev/null +++ b/server/-product/production/LOCAL/jobs/steps/FileSearchFilterOptions.java @@ -0,0 +1,41 @@ +package user.jobengine.server.steps; + +import java.nio.file.Path; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +import com.ibm.nosql.json.api.BasicDBObject; + +public class FileSearchFilterOptions { + + private BasicDBObject filter; + + public FileSearchFilterOptions(BasicDBObject filter) { + this.filter = filter; + } + + public boolean acceptFile(Path file) { + if (filter == null) + return true; + + if (filter.containsKey("fileName")) { + //.*\.(sh|ini|conf|vhost|xml|php)$ + String fileNamePattern = filter.getString("fileName"); + if (fileNamePattern == null || fileNamePattern.trim().length() == 0) + return true; + + Pattern pattern = Pattern.compile(fileNamePattern, Pattern.CASE_INSENSITIVE); + + Matcher matcher = pattern.matcher(file.getFileName().toString()); + if (matcher.find()) + return true; + + } + + return false; + } + + public boolean preAcceptDirectory(Path file) { + return true; + } +} diff --git a/server/-product/production/LOCAL/jobs/steps/HSMMigrateStep.java b/server/-product/production/LOCAL/jobs/steps/HSMMigrateStep.java new file mode 100644 index 00000000..6b85f1d7 --- /dev/null +++ b/server/-product/production/LOCAL/jobs/steps/HSMMigrateStep.java @@ -0,0 +1,401 @@ +package user.jobengine.server.steps; + +import java.io.BufferedInputStream; +import java.io.BufferedOutputStream; +import java.io.File; +import java.io.FileInputStream; +import java.io.FileOutputStream; +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.nio.file.attribute.BasicFileAttributes; +import java.text.SimpleDateFormat; +import java.util.Arrays; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; + +import org.apache.commons.io.FilenameUtils; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.apache.logging.log4j.Marker; + +import com.ibm.nosql.json.api.BasicDBObject; +import com.ibm.nosql.json.api.DB; +import com.ibm.nosql.json.api.DBCollection; +import com.ibm.nosql.json.api.QueryBuilder; + +import user.commons.IEntityBase; +import user.commons.MediaCubeMarker; +import user.commons.nosql.NoSQLUtils; +import user.jobengine.db.IItemManager; +import user.jobengine.db.MediaFile; +import user.jobengine.db.MediaFileDAO; +import user.mediacube.metadata.interfaces.IMetadata; +import user.mediacube.metadata.interfaces.IMetadataProvider; +import user.mediacube.metadata.interfaces.IMetadataProviderFactory; +import user.mediacube.metadata.interfaces.MetadataProviderType; +import user.mediacube.metadata.interfaces.MetadataType; +import user.mediacube.metadata.interfaces.PlanAirMetadataListOptions; + +public class HSMMigrateStep extends JobStep { + private static final Logger logger = LogManager.getLogger(); + private static final String MXFEXT = ".mxf"; + private static final String MOVEXT = ".mov"; + private Marker marker = null; + private IMetadataProvider hsmProvider; + private Map tapeContents = new LinkedHashMap<>(); + private DBCollection excludes; + private DBCollection fileHistory; + private DBCollection volumeHistory; + private DB db; + private IMetadataProvider planairProvider; + private SimpleDateFormat df = new SimpleDateFormat("yyyy.MM.dd"); + + private void cleanupHistory() { + if (db == null) + db = NoSQLUtils.getNoSQLDB(); + if (fileHistory == null) + fileHistory = db.getCollection("hsm_migrate_file_history"); + if (volumeHistory == null) + volumeHistory = db.getCollection("hsm_migrate_volume_history"); + fileHistory.drop(); + volumeHistory.drop(); + } + + private BasicDBObject createMetadata(String volumeName, String fileName) throws Exception { + + Path filePath = Paths.get(fileName); + String mediaHouseId = FilenameUtils.removeExtension(filePath.getFileName().toString()); + BasicDBObject result = null; + try { + result = getPlanAirMetadata(mediaHouseId); + } catch (Exception e) { + + logger.error("PlanAir metadata error", e); + //nem latja a drivert pl. + //throw e; + } + + if (result == null) { + result = new BasicDBObject(); + BasicFileAttributes attr = Files.readAttributes(filePath, BasicFileAttributes.class); + result.put("itemHouseId", df.format(attr.lastModifiedTime().toMillis())); + result.put("itemTitle", filePath.getParent().toString()); + result.put("mediaHouseId", mediaHouseId); + result.put("mediaTitle", fileName); + result.put("mediaDescription", volumeName); + result.put("mediaType", "Generic"); + } + result.put("userName", "mediacube"); + return result; + } + + @SuppressWarnings("serial") + @StepEntry + public Object[] execute(String sourceLocation, String targetLocation) throws Exception { + marker = getJobRuntime().getSessionMarker(); + //remove from prod + //cleanupHistory(); + hsmProvider = getMetadataProvider(MetadataProviderType.HSM); + if (hsmProvider == null) + throw new NullPointerException("No HSM metadata provider available"); + planairProvider = getMetadataProvider(MetadataProviderType.PLANAIR); + if (planairProvider == null) + throw new NullPointerException("No PLANAIR metadata provider available"); + Path targetPath = Paths.get(targetLocation); + try { + List volumes = hsmProvider.list(new BasicDBObject()); + for (IMetadata volume : volumes) { + String volumeName = volume.getTitle(); + + BasicDBObject historyResult = queryVolumeHistory(volumeName); + if (historyResult != null) { + logger.info(marker, "A kazetta már feldolgozásra került: {}", volumeName); + continue; + } + logger.info(marker, "A kazetta feldolgozása elindul: {}", volumeName); + List contents = getContents(volumeName); + int p = 0; + boolean oneSuccess = false; + boolean hasError = false; + for (IMetadata content : contents) { + BasicDBObject c = content.asJSON(); + String hsmFileName = c.getString("fileName"); + long contentFileSize = NoSQLUtils.asLong(c, "fileSize"); + if (!tapeContents.containsKey(hsmFileName)) { + tapeContents.put(hsmFileName, c); + Path sourceFilePath = Paths.get(sourceLocation, hsmFileName); + if (contentFileSize < Files.getFileStore(targetPath).getUsableSpace()) { + Path targetFilePath = Paths.get(targetLocation, sourceFilePath.getFileName().toString()); + + try { + if (processHSMFile(volumeName, hsmFileName, sourceFilePath, targetFilePath)) { + oneSuccess = true; + } + } catch (Exception e) { + hasError = true; + } + } + } + setProgress(p++ * 100 / contents.size()); + + if (getJobRuntime().isWaitingCancel()) { + logger.info("Job canceled by user"); + //ne mentsuk a szalagot meg + return null; + } + + } + + if (oneSuccess && !hasError) { + saveVolumeHistory(volumeName); + String subject = "A kazetta eltávolítható a HSM rendszerből: " + volumeName; + logger.info(new MediaCubeMarker() { + { + setSubject(subject); + } + }, subject); + } + + //ha mar sikerult valamit archivalni kilepunk + if (oneSuccess) + break; + } + } catch (Exception e) { + logger.error(marker, "Hiba a migráció során. A rendszer hibaüzenete: {}", e.getMessage()); + throw e; + } + + return null; + } + + private List getContents(String volumeName) throws Exception { + List contents = null; + contents = hsmProvider.list(new BasicDBObject("volumeName", volumeName)); + return contents; + } + + protected IMetadataProvider getMetadataProvider(MetadataProviderType type) { + IMetadataProviderFactory factory = getService(IMetadataProviderFactory.class); + if (factory == null) + return null; + return factory.getProvider(type); + } + + private BasicDBObject getPlanAirMetadata(String mediaHouseId) throws Exception { + PlanAirMetadataListOptions opt = new PlanAirMetadataListOptions(); + opt.setSearch(mediaHouseId); + + List result = null; + opt.setType(MetadataType.Material); + result = planairProvider.list(opt); + if (result.size() != 0) + return result.get(0).asJSON(); + + opt.setType(MetadataType.Promo); + result = planairProvider.list(opt); + if (result.size() != 0) + return result.get(0).asJSON(); + + opt.setType(MetadataType.AD); + result = planairProvider.list(opt); + if (result.size() != 0) + return result.get(0).asJSON(); + + return null; + } + + //true if need copy + public boolean prepareCopy(String hsmFileName, Path source, Path target) throws IOException { + boolean result = true; + BasicDBObject excludeResult = queryExclude(hsmFileName); + if (excludeResult != null) { + logger.warn(marker, "Kivételként megjelölt: {}", hsmFileName); + return false; + } + + File sourceFile = source.toFile(); + File targetFile = target.toFile(); + + if (!sourceFile.exists()) { + logger.warn(marker, "A forrás nem elérhető: {}", source); + return false; + } + + // BasicDBObject historyResult = queryFileHistory(contentFileName); + // if (historyResult != null) + // return false; + + // if (!mediaFile.getName().toLowerCase().endsWith(MOVEXT.toLowerCase()) && !mediaFile.getName().toLowerCase().endsWith(MXFEXT.toLowerCase())) + // return; + // logger.info("Start copy from {} to {}", sourceFilePath, targetFilePath); + + boolean targetExists = targetFile.exists(); + + long targetLength = targetFile.length(); + long sourceLength = sourceFile.length(); + + if (targetLength == sourceLength) { + logger.warn(marker, "A fájl már fel van dolgozva: {}, {} -> {}", source, sourceLength, targetLength); + return false; + } + + if (targetLength > sourceLength) { + logger.warn(marker, "A célfájl nagyobb, törlöm: {}", target); + Files.delete(target); + targetLength = 0; + targetExists = false; + } + + if (targetExists) { + logger.warn(marker, "A fájl már létezik, a másolás folytatódik: {}, {} -> {}", target, sourceLength, targetLength); + } else + logger.warn(marker, "Migrálás: {}, {} -> {}", source, sourceLength, targetLength); + + return result; + } + + private boolean processHSMFile(String volumeName, String hsmFileName, Path sourceFilePath, Path targetFilePath) throws Exception { + int repeat = 4; + boolean successCopy = false; + + IItemManager manager = getManager(); + MediaFileDAO mfDAO = (MediaFileDAO) manager.getBaseDAO(MediaFile.class); + List mediaFiles = mfDAO.getByHouseId(sourceFilePath.getFileName().toString()); + if (mediaFiles != null && mediaFiles.size() > 0) { + logger.warn(marker, "Már archivált: {}", hsmFileName); + return false; + } + + if (prepareCopy(hsmFileName, sourceFilePath, targetFilePath)) { + while (repeat > 0) { + try { + resumeableCopy(sourceFilePath, targetFilePath); + repeat = 0; + successCopy = true; + } catch (Exception e) { + if (Files.exists(targetFilePath) && targetFilePath.toFile().length() == 0) + Files.delete(targetFilePath); + //logger.warn(marker, "Hiba a másolás során: {} ({})", sourceFilePath, e.getMessage()); + repeat--; + } + } + } + String metadataFileName = sourceFilePath.getFileName() + EscortFiles.DOT_JSON; + Path metadataPath = Paths.get(targetFilePath.getParent().toString(), EscortFiles.STATUSFOLDER, metadataFileName); + boolean createMetadata = Files.exists(targetFilePath) && !Files.exists(metadataPath); + + if (successCopy || createMetadata) { + String metadata = null; + try { + metadata = createMetadata(volumeName, hsmFileName).toPrettyString(""); + EscortFiles.createMetadata(targetFilePath.getParent().toString(), targetFilePath.getFileName().toString(), metadata); + //saveFileHistory(contentFileName); + + } catch (Exception e) { + logger.error(marker, "Metadata error", e); + return false; + } + return true; + } + + logger.error(marker, "A fájl másolása nem lehetséges: {}", sourceFilePath); + return false; + } + + public BasicDBObject queryExclude(String fileName) { + if (db == null) + db = NoSQLUtils.getNoSQLDB(); + if (excludes == null) + excludes = db.getCollection("hsm_migrate_exclude"); + Path filePath = Paths.get(fileName); + String pureFileName = FilenameUtils.removeExtension(filePath.getFileName().toString()); + QueryBuilder qb = QueryBuilder.start("name").in(Arrays.asList(fileName, pureFileName)); + BasicDBObject exceptionResult = NoSQLUtils.asSingle(excludes.find(qb.get())); + return exceptionResult; + } + + public BasicDBObject queryFileHistory(String fileName) { + if (db == null) + db = NoSQLUtils.getNoSQLDB(); + if (fileHistory == null) + fileHistory = db.getCollection("hsm_migrate_file_history"); + QueryBuilder qb = QueryBuilder.start("name").in(Arrays.asList(fileName)); + BasicDBObject historyResult = NoSQLUtils.asSingle(fileHistory.find(qb.get())); + return historyResult; + } + + public BasicDBObject queryVolumeHistory(String volumeName) { + if (db == null) + db = NoSQLUtils.getNoSQLDB(); + if (volumeHistory == null) + volumeHistory = db.getCollection("hsm_migrate_volume_history"); + QueryBuilder qb = QueryBuilder.start("name").in(Arrays.asList(volumeName)); + BasicDBObject historyResult = NoSQLUtils.asSingle(volumeHistory.find(qb.get())); + return historyResult; + } + + public void resumeableCopy(Path source, Path target) throws Exception { + File sourceFile = source.toFile(); + File targetFile = target.toFile(); + boolean targetExists = targetFile.exists(); + + long targetLength = targetFile.length(); + long sourceLength = sourceFile.length(); + + try (InputStream in = new BufferedInputStream(new FileInputStream(sourceFile)); + OutputStream out = new BufferedOutputStream(new FileOutputStream(targetFile, targetExists))) { + + byte[] buffer = new byte[128 * 1024]; + int lengthRead; + + if (targetExists) + in.skip(targetLength); + + while ((lengthRead = in.read(buffer)) > 0) { + out.write(buffer, 0, lengthRead); + out.flush(); + targetLength = targetFile.length(); + if (targetLength > sourceLength) { + throw new Exception("Hiba! A fájl túl nagy lett."); + } + + if (getJobRuntime().isWaitingCancel()) { + break; + } + } + + targetLength = targetFile.length(); + sourceLength = sourceFile.length(); + if (targetLength != sourceLength) { + throw new Exception("Hiba! A fájl mérete nem egyezik."); + } + } + } + + private void saveFileHistory(String fileName) { + if (db == null) + db = NoSQLUtils.getNoSQLDB(); + if (fileHistory == null) + fileHistory = db.getCollection("hsm_migrate_file_history"); + BasicDBObject item = new BasicDBObject(); + item.put("name", fileName); + fileHistory.save(item); + + } + + private void saveVolumeHistory(String volumeName) { + if (db == null) + db = NoSQLUtils.getNoSQLDB(); + if (volumeHistory == null) + volumeHistory = db.getCollection("hsm_migrate_volume_history"); + BasicDBObject item = new BasicDBObject(); + item.put("name", volumeName); + volumeHistory.save(item); + + } +} diff --git a/server/-product/production/LOCAL/jobs/steps/MXFCutterStep.java b/server/-product/production/LOCAL/jobs/steps/MXFCutterStep.java new file mode 100644 index 00000000..58a5944d --- /dev/null +++ b/server/-product/production/LOCAL/jobs/steps/MXFCutterStep.java @@ -0,0 +1,129 @@ +package user.jobengine.server.steps; + +import org.apache.commons.lang.StringUtils; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.apache.logging.log4j.Marker; + +import user.commons.RemoteFile; +import user.commons.StoreUri; +import user.commons.remotestore.IProgressEventListener; +import user.commons.remotestore.ProgressEvent; +import user.commons.remotestore.RemoteStoreProtocol; +//import user.jobengine.db.Media; +import user.jobengine.db.ArchivedMedia; +import user.jobengine.db.IItemManager; +import user.jobengine.db.Store; +import user.jobengine.server.IJobEngine; +import user.jobengine.server.IJobRuntime; + +public class MXFCutterStep extends JobStep { + private static final String TARGETNAMEPATTERN = "-ARCH-%s"; + private static final Logger logger = LogManager.getLogger(); + private IItemManager manager; + private StoreUri tempTargetUri; + private StoreUri tempSourceUri; + private String sourceFileName; + private Marker marker; + private int nexioPort; + private String nexioUserName, nexioPassword; + private String nexioHost; + + protected void checkTargetPath(String targetPath) { + if (StringUtils.isBlank(targetPath)) { + logger.error(marker, "A folyamat 'targetPath' bemeneti paramétere üres."); + throw new NullPointerException("System is not configured properly, 'targetPath' input parameter missing."); + } + } + + protected StoreUri createTargetUri(IItemManager manager, String targetPath) throws NullPointerException { + StoreUri result = null; + result = manager.createStoreUri(RemoteStoreProtocol.FTP, nexioHost); + result.setPortNumber(nexioPort); + result.setUserName(nexioUserName); + result.setPassword(nexioPassword); + + return result; + } + + @StepEntry + public Object[] execute(ArchivedMedia archivedMedia, String targetPath, String houseId, String successRecipient, int killDateDays, boolean useNexioTarget, + String nexioAgency, int nexioPort, String nexioUserName, String nexioPassword, IJobEngine jobEngine, IJobRuntime jobRuntime) throws Exception { + this.nexioPort = nexioPort; + this.nexioUserName = nexioUserName; + this.nexioPassword = nexioPassword; + nexioHost = System.getProperty("nexio.host"); + marker = jobRuntime.getSessionMarker(); + + if (useNexioTarget && archivedMedia.getTcIn() != null && archivedMedia.getTcOut() != null) { + setAndCheck(archivedMedia, houseId, targetPath, useNexioTarget, jobEngine); + + final IJobRuntime runtime = jobRuntime; + sourceFileName = houseId + TARGETNAMEPATTERN; + tempSourceUri.addProgressListener(new IProgressEventListener() { + @Override + public void progressChanged(ProgressEvent evt) { + runtime.incrementProgress(evt.getProgress()); + } + }); + + RemoteFile result = tempSourceUri.transferFrom(tempTargetUri, sourceFileName, sourceFileName); + + EscortFiles.setNEXIOKillDate(killDateDays, houseId, nexioAgency, tempTargetUri); + + logger.info("A {} videó kivágva {}s - {}s", sourceFileName, archivedMedia.getTcIn(), archivedMedia.getTcOut()); + } + + return null; + } + + // private String getSourceFileName(ArchivedMedia archivedMedia, Store + // store) { + // List mediaFiles = archivedMedia.getMedia().getMediaFiles(); + // if (mediaFiles == null) + // return null; + // for (MediaFile mediaFile : mediaFiles) { + // if (mediaFile.getStore().getId() == store.getId()) + // return mediaFile.getRelativePath(); + // } + // return null; + // } + + private void setAndCheck(ArchivedMedia archivedMedia, String houseId, String targetPath, boolean useNexioTarget, IJobEngine jobEngine) { + if (jobEngine == null) { + logger.error(marker, "A folyamatkezelő réteg nem elérhető."); + throw new NullPointerException("Internal error, missing JobEngine reference."); + } + manager = jobEngine.getItemManager(); + if (manager == null) { + logger.error(marker, "Az adatbáziskezelő réteg nem elérhető."); + throw new NullPointerException("Internal error, missing ItemManager reference."); + } + if (archivedMedia == null) { + logger.error(marker, "A folyamat 'mediaCubeMedia' bemeneti paramétere üres."); + throw new NullPointerException("System is not configured properly, 'mediaCubeMedia' input parameter missing."); + } + checkTargetPath(targetPath); + if (StringUtils.isBlank(houseId)) { + logger.error(marker, "A folyamat 'houseId' bemeneti paramétere üres."); + throw new NullPointerException("System is not configured properly, 'houseId' input parameter missing."); + } + Store tsmStore = manager.getSystemStore(false); + if (tsmStore == null) { + logger.error(marker, "A TSM rendszer beállítás nem elérhető."); + throw new NullPointerException("System is not configured properly, missing TSM Store."); + } + + tempSourceUri = manager.createStoreUri(RemoteStoreProtocol.LOCAL, targetPath); + if (tempSourceUri == null) { + logger.error(marker, "A TSM rendszer beállítás paraméterei nem elérhetőek."); + throw new NullPointerException("System is not configured properly, missing TSM StoreUri."); + } + tempTargetUri = createTargetUri(manager, targetPath); + // sourceFileName = getSourceFileName(archivedMedia, tsmStore); + if (sourceFileName == null) { + logger.error(marker, "Adatbázis bejegyzés hiba, a visszatöltendő fájl neve nem található."); + throw new NullPointerException("Database error, missing MediaFile 'relativePath'."); + } + } +} diff --git a/server/-product/production/LOCAL/jobs/steps/MediaToolStep.java b/server/-product/production/LOCAL/jobs/steps/MediaToolStep.java new file mode 100644 index 00000000..c0c56f68 --- /dev/null +++ b/server/-product/production/LOCAL/jobs/steps/MediaToolStep.java @@ -0,0 +1,41 @@ +package user.jobengine.server.steps; + +import java.nio.file.Path; +import java.nio.file.Paths; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.apache.logging.log4j.Marker; + +import user.commons.mediatool.MediaInfo; +import user.jobengine.db.Media; + +public class MediaToolStep extends JobStep { + private static final Logger logger = LogManager.getLogger(); + private Marker marker; + + @StepEntry + public Object[] execute(ArchiveItem archiveItem, Media mediaCubeMedia) throws Exception { + marker = getSessionMarker(); + + //TODO ez most minden esetben ujranezi + // if (mediaCubeMedia.getLength() > 0) + // return null; + + Path filePath = Paths.get(archiveItem.getMediaFile()); + MediaInfo mi = new MediaInfo(filePath); + try { + mi.process(); + long frames = mi.getFrames(); + if (frames > 0) { + mediaCubeMedia.setLength(frames); + getManager().modify(mediaCubeMedia); + } + } catch (Exception e) { + logger.catching(e); + logger.error(marker, "A '{}' média analizálása sikertelen. A rendszer üzenete: {}", filePath, e.getMessage()); + } + return null; + } + +} diff --git a/server/-product/production/LOCAL/jobs/steps/MetadataTransformStep.java b/server/-product/production/LOCAL/jobs/steps/MetadataTransformStep.java new file mode 100644 index 00000000..d1938203 --- /dev/null +++ b/server/-product/production/LOCAL/jobs/steps/MetadataTransformStep.java @@ -0,0 +1,168 @@ +package user.jobengine.server.steps; + +import java.io.File; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.nio.file.attribute.FileAttribute; +import java.nio.file.attribute.PosixFilePermission; +import java.nio.file.attribute.PosixFilePermissions; +import java.util.Date; +import java.util.Set; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.apache.logging.log4j.Marker; + +import com.ibm.nosql.json.api.BasicDBList; + +import user.jobengine.db.Item; +import user.jobengine.db.ItemManager; +import user.jobengine.db.ItemType; +import user.jobengine.db.Media; +import user.jobengine.server.IJobEngine; +import user.jobengine.server.IJobRuntime; + +/** + * Itemek es mediak krealasa a ArchiveItem objektum alapjan. + * + * @author robi + */ +public class MetadataTransformStep extends JobStep { + private static final String CONFLICT = ".CONFLICT"; + private static final Logger logger = LogManager.getLogger(); + private static final String ITEM_MANAGER_IS_NULL = "ItemManager is null"; + public static final String DEFAULT_MEDIATYPE = "Generic"; + private ItemManager itemManager; + + private Marker marker;; + + private void addTags(ArchiveItem archiveItem, Media mediaCubeMedia) { + BasicDBList tags = archiveItem.getTags(); + if (tags != null) { + for (Object tag : tags) { + + try { + String tagText = String.valueOf(tag); + itemManager.addMediaTag(tagText, mediaCubeMedia.getId()); + System.out.println(); + + } catch (Exception e) { + logger.catching(e); + } + } + } + } + + private void checkDuplicates(ArchiveItem archiveItem, String sourceFileName) throws Exception { + if (itemManager.isMediaFileExists(sourceFileName)) { + try { + Path sourcePath = Paths.get(archiveItem.getMediaFile()); + Path parent = sourcePath.getParent(); + Path conflictPath = Paths.get(parent.toString(), CONFLICT); + File folder = conflictPath.toFile(); + if (!folder.exists() || !folder.isDirectory()) { + Set perms = PosixFilePermissions.fromString("rwxrwxrwx"); + FileAttribute> attr = PosixFilePermissions.asFileAttribute(perms); + try { + Files.createDirectories(conflictPath, attr); + } catch (Exception e) { + try { + Files.createDirectory(conflictPath); + } catch (Exception e1) { + logger.catching(e); + throw e; + } + } + } + + Files.move(sourcePath, Paths.get(conflictPath.toString(), sourceFileName + (new Date()).getTime())); + } catch (Exception e1) { + logger.catching(e1); + logger.error(marker, "Hiba az '{}' állomány mappába másolásakor. A rendszer üzenete: {}", CONFLICT, e1.getMessage()); + } + throw new Exception("Az '" + sourceFileName + "' állomány már megtalálható az archívumban, archiválása nem lehetséges."); + } + } + + private Item createItem(ArchiveItem archiveItem) { + Item mediaCubeItem = getExistingItem(archiveItem.getItemHouseId(), archiveItem.getItemTitle()); + if (mediaCubeItem == null) + mediaCubeItem = itemManager.createItem(DEFAULT_MEDIATYPE, archiveItem.getItemTitle(), archiveItem.getItemDescription(), + archiveItem.getItemHouseId()); + return mediaCubeItem; + } + + private Media createMedia(ArchiveItem archiveItem, Item mediaCubeItem, String mediaType) { + Media mediaCubeMedia; + mediaCubeMedia = itemManager.createMedia(mediaType, archiveItem.getMediaTitle(), archiveItem.getMediaDescription(), archiveItem.getMediaHouseId()); + mediaCubeMedia.setLength(archiveItem.getDuration()); + mediaCubeItem.appendMedia(mediaCubeMedia); + + return mediaCubeMedia; + } + + @StepEntry + public Object[] execute(ArchiveItem archiveItem, IJobEngine jobEngine, IJobRuntime jobRuntime) throws Exception { + marker = jobRuntime.getSessionMarker(); + Media mediaCubeMedia = null; + itemManager = (ItemManager) jobEngine.getItemManager(); + if (itemManager == null) + throw new NullPointerException(ITEM_MANAGER_IS_NULL); + try { + File sourceMediaFile = new File(archiveItem.getMediaFile()); + String sourceFileName = sourceMediaFile.getName(); + checkDuplicates(archiveItem, sourceFileName); + Item mediaCubeItem = createItem(archiveItem); + jobRuntime.incrementProgress(50); + String mediaType = getCreateType(archiveItem); + mediaCubeMedia = createMedia(archiveItem, mediaCubeItem, mediaType); + //ha itemid 0 akkor merge, egyebkent media insert + + if (mediaCubeItem.getId() == 0) + itemManager.mergeItemStructure(mediaCubeItem); + else { + mediaCubeMedia.setItemId(mediaCubeItem.getId()); + mediaCubeMedia.add(); + } + + addTags(archiveItem, mediaCubeMedia); + + } catch (Exception e) { + logger.catching(e); + String fileName = new File(archiveItem.getMediaFile()).getName(); + logger.error(marker, "Az '{}' állomány nem archiválható, mert a metaadat transzformáció sikertelen. A rendszer üzenete: {}", fileName, + e.getMessage()); + if (!archiveItem.removeCatchedFile()) + logger.error(marker, "Az '{}' állomány .catched jelző állománya nem törölhető.", fileName); + throw e; + } finally { + jobRuntime.incrementProgress(100); + } + return new Object[] { mediaCubeMedia }; + } + + private String getCreateType(ArchiveItem archiveItem) { + String mediaType = archiveItem.getMediaType(); + if (mediaType == null || mediaType.length() == 0) + mediaType = DEFAULT_MEDIATYPE; + else { + ItemType mediaItemType = itemManager.getItemType(mediaType); + if (mediaItemType == null) + itemManager.createItemType(mediaType, mediaType).add(); + } + return mediaType; + } + + private Item getExistingItem(String itemHouseId, String itemTitle) { + Item[] result = new Item[] { null }; + String sql = String.format("select id from item where houseid='%s' and title='%s'", itemHouseId, itemTitle); + itemManager.executeQuery(sql, rs -> { + long id = rs.getLong("id"); + result[0] = itemManager.getItem(id); + return true; + }, null); + return result[0]; + } + +} diff --git a/server/-product/production/LOCAL/jobs/steps/OutputPathAndNameSelectorStep.java b/server/-product/production/LOCAL/jobs/steps/OutputPathAndNameSelectorStep.java new file mode 100644 index 00000000..7148df4c --- /dev/null +++ b/server/-product/production/LOCAL/jobs/steps/OutputPathAndNameSelectorStep.java @@ -0,0 +1,151 @@ +package user.jobengine.server.steps; + +import java.io.IOException; +import java.nio.file.FileVisitResult; +import java.nio.file.FileVisitor; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.nio.file.SimpleFileVisitor; +import java.nio.file.attribute.BasicFileAttributes; + +import org.apache.commons.lang.StringUtils; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.apache.logging.log4j.Marker; + +import user.jobengine.db.ArchivedMedia; +import user.jobengine.server.IJobEngine; +import user.jobengine.server.IJobRuntime; + +public class OutputPathAndNameSelectorStep extends JobStep { + + private static final String TARGETNAMEPATTERN = "-ARCH-%s"; + + private static final Logger logger = LogManager.getLogger(); + + private Marker marker; + + private void check(String localRetrievePath, String materialOutputFolder, String promoOutputFolder, String advertisementOutputFolder, + String octopusOutputFolder, String genericOutputFolder, String houseId, String targetPathType) { + if (StringUtils.isBlank(localRetrievePath)) { + logger.error(marker, "A folyamat 'localRetrievePath' bemeneti paramétere üres."); + throw new NullPointerException("System is not configured properly, 'localRetrievePath' input parameter missing."); + } + if (StringUtils.isBlank(materialOutputFolder)) { + logger.error(marker, "A folyamat 'materialOutputFolder' bemeneti paramétere üres."); + throw new NullPointerException("System is not configured properly, 'materialOutputFolder' input parameter missing."); + } + if (StringUtils.isBlank(promoOutputFolder)) { + logger.error(marker, "A folyamat 'promoOutputFolder' bemeneti paramétere üres."); + throw new NullPointerException("System is not configured properly, 'promoOutputFolder' input parameter missing."); + } + if (StringUtils.isBlank(advertisementOutputFolder)) { + logger.error(marker, "A folyamat 'advertisementOutputFolder' bemeneti paramétere üres."); + throw new NullPointerException("System is not configured properly, 'advertisementOutputFolder' input parameter missing."); + } + if (StringUtils.isBlank(octopusOutputFolder)) { + logger.error(marker, "A folyamat 'octopusOutputFolder' bemeneti paramétere üres."); + throw new NullPointerException("System is not configured properly, 'octopusOutputFolder' input parameter missing."); + } + if (StringUtils.isBlank(genericOutputFolder)) { + logger.error(marker, "A folyamat 'genericOutputFolder' bemeneti paramétere üres."); + throw new NullPointerException("System is not configured properly, 'genericOutputFolder' input parameter missing."); + } + if (StringUtils.isBlank(houseId)) { + logger.error(marker, "A folyamat 'houseId' bemeneti paramétere üres."); + throw new NullPointerException("System is not configured properly, 'houseId' input parameter missing."); + } + if (StringUtils.isBlank(targetPathType)) { + logger.error(marker, "A folyamat 'targetPathType' bemeneti paramétere üres."); + throw new NullPointerException("System is not configured properly, 'targetPathType' input parameter missing."); + } + } + + @StepEntry + public Object[] execute(String localRetrievePath, String materialOutputFolder, String promoOutputFolder, String advertisementOutputFolder, + String octopusOutputFolder, String genericOutputFolder, String onlineOutputFolder, String houseId, String targetPathType, + ArchivedMedia archivedMedia, IJobEngine jobEngine, IJobRuntime jobRuntime) throws Exception { + marker = jobRuntime.getSessionMarker(); + check(localRetrievePath, materialOutputFolder, promoOutputFolder, advertisementOutputFolder, octopusOutputFolder, genericOutputFolder, houseId, + targetPathType); + Object[] result = null; + switch (Integer.parseInt(targetPathType)) { + case 0: + String outputFolder = getFolderById(materialOutputFolder, promoOutputFolder, advertisementOutputFolder, octopusOutputFolder, genericOutputFolder, + houseId, archivedMedia); + result = localTargetInit(localRetrievePath, outputFolder, houseId, jobRuntime); + break; + case 1: + result = localTargetInit(localRetrievePath, onlineOutputFolder, houseId, jobRuntime); + break; + case 2: + + if (archivedMedia.getTcIn() != null && archivedMedia.getTcOut() != null) + result = new Object[] { genericOutputFolder, houseId, true }; + else + result = new Object[] { null, houseId, true }; + break; + } + return result; + } + + private String getFolderById(String materialOutputFolder, String promoOutputFolder, String advertisementOutputFolder, String octopusOutputFolder, + String genericOutputFolder, String houseId, ArchivedMedia archivedMedia) throws Exception { + String id = houseId.toUpperCase(); + MetadataType mdType = MetadataTypeDetector.GuessMetadataType(id); + String result = null; + + //a groovy nem latja enumnak, hanem az objektum tulajdonsaganak + switch (mdType.toString()) { + case "OctopusPlaceholder": + case "OctopusStory": + result = octopusOutputFolder; + break; + case "TrafficMaterial": + result = materialOutputFolder; + break; + case "TrafficPromo": + result = promoOutputFolder; + break; + case "TrafficAD": + result = advertisementOutputFolder; + break; + case "Generic": + result = genericOutputFolder; + break; + } + return result; + } + + private String getPossiblePath(String id, Path targetPath) throws IOException { + String[] result = new String[] { targetPath.toString() }; + FileVisitor matcherVisitor = new SimpleFileVisitor() { + @Override + public FileVisitResult preVisitDirectory(Path dir, BasicFileAttributes attrs) throws IOException { + String dirName = dir.getFileName().toString(); + if (dirName.startsWith(id + "-") || dirName.equals(id)) { + result[0] = dir.toString(); + return FileVisitResult.TERMINATE; + } + return FileVisitResult.CONTINUE; + } + }; + Files.walkFileTree(targetPath.getParent(), matcherVisitor); + return result[0]; + } + + private Object[] localTargetInit(String localRetrievePath, String outputFolder, String houseId, IJobRuntime jobRuntime) throws IOException { + String id = houseId.toUpperCase(); + String targetPath = getPossiblePath(id, Paths.get(localRetrievePath, outputFolder, id)).toString(); + String targetNamePattern = houseId + TARGETNAMEPATTERN; + try { + EscortFiles.ensureUNCFolder(Paths.get(targetPath)); + } catch (Exception e) { + logger.error(jobRuntime.getSessionMarker(), "A cél mappa '{}' nem létezik és nem hozható létre. A rendszer hibaüzenete: {}", targetPath, + e.getMessage()); + throw e; + } + return new Object[] { targetPath, targetNamePattern, false }; + } +} diff --git a/server/-product/production/LOCAL/jobs/steps/PathItemsCollectorStep.java b/server/-product/production/LOCAL/jobs/steps/PathItemsCollectorStep.java new file mode 100644 index 00000000..69ca5d02 --- /dev/null +++ b/server/-product/production/LOCAL/jobs/steps/PathItemsCollectorStep.java @@ -0,0 +1,67 @@ +package user.jobengine.server.steps; + +import java.io.IOException; +import java.nio.file.FileVisitResult; +import java.nio.file.FileVisitor; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.nio.file.SimpleFileVisitor; +import java.nio.file.attribute.BasicFileAttributes; +import java.util.ArrayList; +import java.util.List; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + +import com.ibm.nosql.json.api.BasicDBObject; + +/* DO NOT REMOVE! +import user.jobengine.server.steps.FileSearchFilterOptions; + */ + +public class PathItemsCollectorStep extends JobStep { + private static final Logger logger = LogManager.getLogger(); + + @StepEntry + public Object[] execute(String sourceFolder, BasicDBObject filter) throws Exception { + Path sourcePath = Paths.get(sourceFolder); + + List files = new ArrayList<>(); + + //teljes rekurzivitas szuressel + FileSearchFilterOptions filterOptions = new FileSearchFilterOptions(filter); + FileVisitor visitor = new SimpleFileVisitor() { + @Override + public FileVisitResult preVisitDirectory(Path dir, BasicFileAttributes attrs) throws IOException { + if (!filterOptions.preAcceptDirectory(dir)) + return FileVisitResult.SKIP_SUBTREE; + return FileVisitResult.CONTINUE; + } + + @Override + public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) throws IOException { + if (filterOptions.acceptFile(file)) + files.add(file.toString()); + logger.info(file); + return FileVisitResult.CONTINUE; + } + + @Override + public FileVisitResult visitFileFailed(Path file, IOException exc) throws IOException { + return FileVisitResult.CONTINUE; + } + + }; + + try { + Files.walkFileTree(sourcePath, visitor); + } catch (Exception e) { + logger.error(getSessionMarker(), "Hiba a '{}' mappa feldolgozásában. A rendszer hibaüzenete: {}", sourcePath, e.getMessage()); + logger.catching(e); + throw e; + } finally { + } + return new Object[] { files }; + } +} diff --git a/server/-product/production/LOCAL/jobs/steps/TSMBackupStep.java b/server/-product/production/LOCAL/jobs/steps/TSMBackupStep.java new file mode 100644 index 00000000..60adc661 --- /dev/null +++ b/server/-product/production/LOCAL/jobs/steps/TSMBackupStep.java @@ -0,0 +1,207 @@ +package user.jobengine.server.steps; + +import java.io.File; +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.attribute.BasicFileAttributes; +import java.sql.Timestamp; +import java.util.Date; +import java.util.List; + +import org.apache.commons.lang.RandomStringUtils; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.apache.logging.log4j.Marker; +import org.apache.logging.log4j.message.Message; +import org.apache.logging.log4j.message.ParameterizedMessage; + +import user.commons.RemoteFile; +import user.commons.StoreUri; +import user.commons.mediatool.Timecode; +import user.commons.mediatool.Timecode.Type; +import user.commons.remotestore.IProgressEventListener; +import user.commons.remotestore.IStatusEventListener; +import user.commons.remotestore.ProgressEvent; +import user.commons.remotestore.RemoteStoreProtocol; +import user.commons.remotestore.StatusEvent; +import user.jobengine.db.FileType; +import user.jobengine.db.IItemManager; +import user.jobengine.db.Media; +import user.jobengine.db.MediaFile; +import user.jobengine.db.Store; +import user.jobengine.server.IJobEngine; +import user.jobengine.server.IJobRuntime; +import user.jobengine.server.JobEngineException; + +public class TSMBackupStep extends JobStep { + private static final String MXFEXT = ".MXF"; + private static final Logger logger = LogManager.getLogger(); + private IItemManager manager; + private File sourceMediaFile; + private Store tsmStore; + private StoreUri targetUri; + private FileType fileType; + private Marker marker; + + @StepEntry + public Object[] execute(ArchiveItem archiveItem, Media mediaCubeMedia, int killDateDays) throws Exception { + marker = getSessionMarker(); + + File sourceMediaFile = new File(archiveItem.getMediaFile()); + String sourceFileName = sourceMediaFile.getName(); + Timecode timecode = new Timecode(mediaCubeMedia.getLength(), Type.PAL); + String details = String.format("%s (%s, %d bytes)", sourceFileName, timecode.toString(), sourceMediaFile.length()); + getJobRuntime().setDescription(details); + try { + + setAndCheck(archiveItem, mediaCubeMedia, getEngine()); + + //TODO mxf helyett az osszes kiterjesztest!!!!! + //A dupla ellenorzes a napon beluli ismetlesek miatt kell + long existingMediaId = archiveItem.getExistingMediaId(); + if (sourceMediaFile.length() == 0 && existingMediaId == 0) { + existingMediaId = ItemManagerExtensions.getExistingRundownMedia(manager, sourceFileName.replace(MXFEXT, "")); + if (existingMediaId == 0) + existingMediaId = -1; + } + + if (existingMediaId == 0) + existingMediaId = ItemManagerExtensions.getExistingRundownMedia(manager, sourceFileName.replace(MXFEXT, "")); + + boolean randomizeName = Boolean.parseBoolean(System.getProperty("jobengine.randomize.archives", "false")); + + String targetFileName; + if (randomizeName) { + // a-z, A-Z, 0-9. For example: WRMcpIk7, s57JwCVA + //veletlenszeru neveket adunk! + targetFileName = String.format("%s-%s", RandomStringUtils.randomAlphanumeric(8), sourceFileName); + } else + targetFileName = sourceFileName; + + if (existingMediaId == 0) { + StoreUri sourceUri = manager.createStoreUri(RemoteStoreProtocol.LOCAL, sourceMediaFile.getParent().toString()); + + final IJobRuntime runtime = getJobRuntime(); + sourceUri.addProgressListener(new IProgressEventListener() { + @Override + public void progressChanged(ProgressEvent evt) { + runtime.incrementProgress(evt.getProgress()); + } + }); + sourceUri.addStatusListener(new IStatusEventListener() { + @Override + public void statusChanged(StatusEvent evt) { + evt.setCancel(!canContinue()); + } + }); + + RemoteFile remoteFile = sourceUri.transferFrom(targetUri, sourceFileName, targetFileName); + } + + if (existingMediaId > 0) + logger.info(marker, "Az '{}' TSM mentése nem szükséges, mert már megtalálható az archívumban.", sourceFileName); + + //Fel kell szabadítani, hogy a kovetkezo archivalaskor is nekifusson + if (existingMediaId == -1) { + logger.info(marker, "Az '{}' mentése jelenleg nem lehetséges, mert a szükséges metaadat még nem található meg az archívumban.", sourceFileName); + if (!archiveItem.removeCatchedFile()) + logger.error(marker, + "Az '{}' állomány .catched jelző állománya nem törölhető. Az újabb archiválási kísérlethez annak kézi eltávolítása szükséges!", + sourceMediaFile.getName()); + mediaCubeMedia.remove(); + } else { + saveMetadata(mediaCubeMedia, sourceMediaFile, targetFileName, existingMediaId); + logger.info(marker, "Az '{}' archiválása sikeres.", sourceFileName); + if (killDateDays != 0) + EscortFiles.createUNCKillDate(sourceMediaFile.getParent(), sourceFileName, killDateDays, marker); + } + + } catch (Exception e) { + logger.catching(e); + Message m = new ParameterizedMessage("Az '{}' állomány archiválása sikertelen. A rendszer hibaüzenete: {}", details, e.getMessage()); + logger.error(marker, m); + if (!archiveItem.removeCatchedFile()) + logger.error(marker, + "Az '{}' állomány .catched jelző állománya nem törölhető. Az újabb archiválási kísérlethez annak kézi eltávolítása szükséges!", + sourceMediaFile.getName()); + throw new Exception(m.getFormattedMessage()); + } + return null; + } + + private void saveMetadata(Media mediaCubeMedia, File sourceFile, String targetFileName, long existingMediaId) { + + if (existingMediaId == 0) { + MediaFile mediaFile = manager.createMediaFile(targetFileName, fileType, tsmStore, mediaCubeMedia); + mediaFile.setHouseId(sourceFile.getName()); + mediaFile.add(); + } else { + Media existingMedia = manager.getMedia(existingMediaId); + List mediaFiles = existingMedia.getMediaFiles(); + if (mediaFiles != null) { + for (MediaFile mf : mediaFiles) { + mf.setPersister(manager); + mf.setId(0); + mf.setMedia(mediaCubeMedia); + mf.add(); + } + } + } + mediaCubeMedia.setPersister(manager); + + try { + BasicFileAttributes attr = Files.readAttributes(sourceFile.toPath(), BasicFileAttributes.class); + mediaCubeMedia.setArchived(new Timestamp(attr.creationTime().toMillis())); + } catch (IOException e) { + logger.catching(e); + mediaCubeMedia.setArchived(new Timestamp(new Date().getTime())); + } + + mediaCubeMedia.modify(); + } + + private void setAndCheck(ArchiveItem archiveItem, Media mediaCubeMedia, IJobEngine jobEngine) throws JobEngineException, IOException { + if (jobEngine == null) { + logger.error(marker, "Az folyamatkezelő réteg nem elérhető."); + throw new NullPointerException("Internal error, missing JobEngine reference."); + } + manager = jobEngine.getItemManager(); + if (manager == null) { + logger.error(marker, "Az adatbáziskezelő réteg nem elérhető."); + throw new NullPointerException("Internal error, missing ItemManager reference."); + } + if (archiveItem == null) { + logger.error(marker, "A folyamat 'archiveItem' bemeneti paramétere üres."); + throw new NullPointerException("System is not configured properly, missing 'archiveItem' input parameter."); + } + sourceMediaFile = new File(archiveItem.getMediaFile()); + if (sourceMediaFile == null) { + logger.error(marker, "A folyamat 'archiveItem' bemeneti paraméter 'mediaFile' értéke üres."); + throw new NullPointerException("System is not configured properly, missing 'mediaFile' value in 'archiveItem' input parameter."); + } + if (!sourceMediaFile.exists()) { + logger.error(marker, "Az {} állomány nem létezik vagy nem érhető el.", sourceMediaFile.getName()); + throw new IOException(String.format("Input file {} not exists or unreachable.", sourceMediaFile.getName())); + } + tsmStore = manager.getSystemStore(false); + if (tsmStore == null) { + logger.error(marker, "A TSM rendszer beállítás nem elérhető."); + throw new NullPointerException("System is not configured properly, missing TSM Store."); + } + targetUri = tsmStore.getSourceStoreUri(RemoteStoreProtocol.TSM); + if (targetUri == null) { + logger.error(marker, "A TSM rendszer beállítás paraméterei nem elérhetőek."); + throw new NullPointerException("System is not configured properly, missing TSM StoreUri."); + } + fileType = manager.getFileType("High-res"); + if (fileType == null) { + logger.error(marker, "Adatbázis bejegyzés hiba, a 'High-res' FileType nem található."); + throw new NullPointerException("System is not configured properly, missing 'High-res' FileType."); + } + if (mediaCubeMedia == null) { + logger.error(marker, "A folyamat 'mediaCubeMedia' bemeneti paramétere üres."); + throw new NullPointerException("System is not configured properly, 'mediaCubeMedia' input parameter missing."); + } + + } +} diff --git a/server/-product/production/LOCAL/jobs/steps/TSMExtendedRetrieveStep.java b/server/-product/production/LOCAL/jobs/steps/TSMExtendedRetrieveStep.java new file mode 100644 index 00000000..79a601ca --- /dev/null +++ b/server/-product/production/LOCAL/jobs/steps/TSMExtendedRetrieveStep.java @@ -0,0 +1,91 @@ +package user.jobengine.server.steps; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + +import user.commons.StoreUri; +import user.commons.remotestore.RemoteStoreProtocol; +import user.jobengine.db.ArchivedMedia; +import user.jobengine.db.IItemManager; +import user.jobengine.server.IJobEngine; +import user.jobengine.server.IJobRuntime; + +public class TSMExtendedRetrieveStep extends TSMRestoreStep { + private static String NEXIO_HOST = System.getProperty("nexio.host"); + // private static String NEXIO_HOST = SystemConfiguration.getInstance().value("services.nexio.host"); + private static final Logger logger = LogManager.getLogger(); + + private boolean useNexioTarget; + private int nexioPort; + private String nexioUserName, nexioPassword; + private String nexioAgency; + + @Override + protected void afterRestore(StoreUri targetUri, String targetPath, int killDateDays, String targetFileName) throws Exception { + if (useNexioTarget) { + EscortFiles.setNEXIOKillDate(killDateDays, targetFileName, nexioAgency, targetUri); + } else { + super.afterRestore(targetUri, targetPath, killDateDays, targetFileName); + } + } + + @Override + protected void beforeRestore(StoreUri targetURI, String targetName) throws Exception { + String newTargetName = targetName; + if (targetName.contains(".")) + newTargetName = targetName.substring(0, targetName.lastIndexOf('.')); + if (useNexioTarget) + if (targetURI.fileExists(newTargetName + ".mxf")) + throw new Exception(String.format("%s-The newly created file name is existed.", getClass().getSimpleName())); + } + + @Override + protected void checkTargetPath(String targetPath) { + if (!useNexioTarget) + super.checkTargetPath(targetPath); + } + + @Override + protected StoreUri createTargetUri(IItemManager manager, String targetPath) throws NullPointerException { + StoreUri result = null; + logger.info(getSessionMarker(), "Create target uri {}", targetPath); + if (useNexioTarget) { + if (NEXIO_HOST == null) { + throw new NullPointerException("Missing system property on 'nexio.host' name"); + } + result = manager.createStoreUri(RemoteStoreProtocol.FTP, NEXIO_HOST); + result.setPortNumber(nexioPort); + result.setUserName(nexioUserName); + result.setPassword(nexioPassword); + } else + result = super.createTargetUri(manager, targetPath); + return result; + } + + @StepEntry + public Object[] execute(ArchivedMedia archivedMedia, String targetPath, String targetNamePattern, String successRecipient, int killDateDays, + String localRetrievePath, String globalRetrievePath, boolean useNexioTarget, String nexioAgency, int nexioPort, String nexioUserName, + String nexioPassword, IJobEngine jobEngine, IJobRuntime jobRuntime) throws Exception { + this.useNexioTarget = useNexioTarget; + this.nexioAgency = nexioAgency; + this.nexioPort = nexioPort; + this.nexioUserName = nexioUserName; + this.nexioPassword = nexioPassword; + if (nexioPort == 0) { + throw new NullPointerException("System is not configured properly, 'nexioPort' input parameter missing."); + } + if (nexioUserName == null) { + throw new NullPointerException("System is not configured properly, 'nexioUserName' input parameter missing."); + } + if (nexioPassword == null) { + throw new NullPointerException("System is not configured properly, 'nexioPassword' input parameter missing."); + } + if (nexioAgency == null) { + throw new NullPointerException("System is not configured properly, 'nexioAgency' input parameter missing."); + } + + return super.execute(archivedMedia.getMedia(), targetPath, targetNamePattern, successRecipient, killDateDays, localRetrievePath, globalRetrievePath, + jobEngine, jobRuntime); + } + +} diff --git a/server/-product/production/LOCAL/jobs/steps/TSMRestoreStep.java b/server/-product/production/LOCAL/jobs/steps/TSMRestoreStep.java new file mode 100644 index 00000000..5b410055 --- /dev/null +++ b/server/-product/production/LOCAL/jobs/steps/TSMRestoreStep.java @@ -0,0 +1,159 @@ +package user.jobengine.server.steps; + +import java.io.IOException; +import java.nio.file.Paths; +import java.util.List; + +import org.apache.commons.lang.StringUtils; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.apache.logging.log4j.Marker; +import org.apache.logging.log4j.message.Message; + +import user.commons.LogUtils; +import user.commons.RemoteFile; +import user.commons.StoreUri; +import user.commons.mediatool.Timecode; +import user.commons.mediatool.Timecode.Type; +import user.commons.remotestore.IProgressEventListener; +import user.commons.remotestore.IStatusEventListener; +import user.commons.remotestore.ProgressEvent; +import user.commons.remotestore.RemoteStoreProtocol; +import user.commons.remotestore.StatusEvent; +import user.jobengine.db.IItemManager; +import user.jobengine.db.Media; +import user.jobengine.db.MediaFile; +import user.jobengine.db.Store; +import user.jobengine.server.IJobEngine; +import user.jobengine.server.IJobRuntime; + +public class TSMRestoreStep extends JobStep { + private static final Logger logger = LogManager.getLogger(); + private IItemManager manager; + private StoreUri targetUri; + private StoreUri sourceUri; + private String sourceFileName; + private Marker marker; + + protected void afterRestore(StoreUri targetUri, String targetPath, int killDateDays, String targetFileName) throws IOException, Exception { + if (killDateDays > 0) + EscortFiles.createUNCKillDate(targetPath, targetFileName, killDateDays, marker); + } + + protected void beforeRestore(StoreUri targetURI, String targetFileName) throws Exception { + } + + protected void checkTargetPath(String targetPath) { + if (StringUtils.isBlank(targetPath)) { + logger.error(marker, "A folyamat 'targetPath' bemeneti paramétere üres."); + throw new NullPointerException("System is not configured properly, 'targetPath' input parameter missing."); + } + } + + protected StoreUri createTargetUri(IItemManager manager, String targetPath) { + return manager.createStoreUri(RemoteStoreProtocol.LOCAL, targetPath); + } + + @StepEntry + public Object[] execute(Media mediaCubeMedia, String targetPath, String targetNamePattern, String successRecipient, int killDateDays, + String localRetrievePath, String globalRetrievePath, IJobEngine jobEngine, IJobRuntime jobRuntime) throws Exception { + marker = jobRuntime.getSessionMarker(); + setAndCheck(mediaCubeMedia, targetPath, targetNamePattern, localRetrievePath, globalRetrievePath, jobEngine); + String targetFileName = String.format(targetNamePattern, sourceFileName); + Timecode timecode = new Timecode(mediaCubeMedia.getLength(), Type.PAL); + try { + String details = String.format("%s (%s)", sourceFileName, timecode.toString()); + jobRuntime.setDescription(details); + beforeRestore(targetUri, targetNamePattern); + final IJobRuntime runtime = jobRuntime; + sourceUri.addProgressListener(new IProgressEventListener() { + @Override + public void progressChanged(ProgressEvent evt) { + runtime.incrementProgress(evt.getProgress()); + } + }); + sourceUri.addStatusListener(new IStatusEventListener() { + @Override + public void statusChanged(StatusEvent evt) { + evt.setCancel(!canContinue()); + } + }); + RemoteFile result = sourceUri.transferFrom(targetUri, sourceFileName, targetFileName); + + String globalTargetPath = Paths.get(targetPath, targetFileName).getParent().toString().replace(Paths.get(localRetrievePath).toString(), + globalRetrievePath); + + logger.info(marker, + "Az '{}' állomány visszatöltése sikeres volt '{}' néven. A célmappa a ide kattintva nyitható meg.", + sourceFileName, targetFileName, globalTargetPath); + afterRestore(targetUri, targetPath, killDateDays, targetFileName); + + } catch (Exception e) { + Message msg = LogUtils.format("Az '{}' állomány visszatöltése sikertelen. A rendszer hibaüzenete: {}", sourceFileName, e.getMessage()); + logger.error(marker, msg); + // logger.error(jobRuntime.marker, msg); + logger.catching(e); + throw e; + } + + return null; + } + + private String getSourceFileName(Media mediaCubeMedia, Store store) { + List mediaFiles = mediaCubeMedia.getMediaFiles(); + if (mediaFiles == null) + return null; + for (MediaFile mediaFile : mediaFiles) { + if (mediaFile.getStore().getId() == store.getId()) + return mediaFile.getRelativePath(); + } + return null; + } + + private void setAndCheck(Media mediaCubeMedia, String targetPath, String targetNamePattern, String localRetrievePath, String globalRetrievePath, + IJobEngine jobEngine) { + if (jobEngine == null) { + logger.error(marker, "Az folyamatkezelő réteg nem elérhető."); + throw new NullPointerException("Internal error, missing JobEngine reference."); + } + manager = jobEngine.getItemManager(); + if (manager == null) { + logger.error(marker, "Az adatbáziskezelő réteg nem elérhető."); + throw new NullPointerException("Internal error, missing ItemManager reference."); + } + if (mediaCubeMedia == null) { + logger.error(marker, "A folyamat 'mediaCubeMedia' bemeneti paramétere üres."); + throw new NullPointerException("System is not configured properly, 'mediaCubeMedia' input parameter missing."); + } + checkTargetPath(targetPath); + if (StringUtils.isBlank(targetNamePattern)) { + logger.error(marker, "A folyamat 'targetNamePattern' bemeneti paramétere üres."); + throw new NullPointerException("System is not configured properly, 'targetNamePattern' input parameter missing."); + } + Store tsmStore = manager.getSystemStore(false); + if (tsmStore == null) { + logger.error(marker, "A TSM rendszer beállítás nem elérhető."); + throw new NullPointerException("System is not configured properly, missing TSM Store."); + } + sourceUri = tsmStore.getSourceStoreUri(RemoteStoreProtocol.TSM); + if (sourceUri == null) { + logger.error(marker, "A TSM rendszer beállítás paraméterei nem elérhetőek."); + throw new NullPointerException("System is not configured properly, missing TSM StoreUri."); + } + targetUri = createTargetUri(manager, targetPath); + sourceFileName = getSourceFileName(mediaCubeMedia, tsmStore); + if (sourceFileName == null) { + logger.error(marker, "Adatbázis bejegyzés hiba, a visszatöltendő fájl neve nem található."); + throw new NullPointerException("Database error, missing MediaFile 'relativePath'."); + } + + if (StringUtils.isBlank(localRetrievePath)) { + logger.error(marker, "A folyamat 'localRetrievePath' bemeneti paramétere üres."); + throw new NullPointerException("System is not configured properly, 'localRetrievePath' input parameter missing."); + } + if (StringUtils.isBlank(globalRetrievePath)) { + logger.error(marker, "A folyamat 'globalRetrievePath' bemeneti paramétere üres."); + throw new NullPointerException("System is not configured properly, 'globalRetrievePath' input parameter missing."); + } + } +} diff --git a/server/-product/production/LOCAL/jobs/steps/TSMSimpleRestoreStep.java b/server/-product/production/LOCAL/jobs/steps/TSMSimpleRestoreStep.java new file mode 100644 index 00000000..ba63212c --- /dev/null +++ b/server/-product/production/LOCAL/jobs/steps/TSMSimpleRestoreStep.java @@ -0,0 +1,57 @@ +package user.jobengine.server.steps; + +import java.nio.file.Files; +import java.nio.file.Paths; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + +import user.commons.StoreUri; +import user.commons.mediatool.Timecode; +import user.commons.mediatool.Timecode.Type; +import user.commons.remotestore.IProgressEventListener; +import user.commons.remotestore.IStatusEventListener; +import user.commons.remotestore.ProgressEvent; +import user.commons.remotestore.RemoteStoreProtocol; +import user.commons.remotestore.StatusEvent; +import user.jobengine.db.Media; +import user.jobengine.db.MediaFile; +import user.jobengine.db.Store; + +public class TSMSimpleRestoreStep extends JobStep { + private static final Logger logger = LogManager.getLogger(); + + @StepEntry + public Object[] execute(long mediaId, String targetPath) throws Exception { + String fileName = null; + try { + Media media = getManager().getMedia(mediaId); + MediaFile mediaFile = getManager().getSystemMediaFile(media); + fileName = mediaFile.getRelativePath(); + Timecode timecode = new Timecode(media.getLength(), Type.PAL); + getJobRuntime().setDescription(String.format("%s (%s)", fileName, timecode.toString())); + Store tsmStore = getManager().getSystemStore(false); + StoreUri sourceStoreUri = getManager().getStoreUri(tsmStore.getName(), RemoteStoreProtocol.TSM); + sourceStoreUri.addProgressListener(new IProgressEventListener() { + @Override + public void progressChanged(ProgressEvent evt) { + setProgress(evt.getProgress()); + } + }); + sourceStoreUri.addStatusListener(new IStatusEventListener() { + @Override + public void statusChanged(StatusEvent evt) { + evt.setCancel(!canContinue()); + } + }); + StoreUri targetStoreUri = getManager().createStoreUri(RemoteStoreProtocol.LOCAL, targetPath); + sourceStoreUri.transferFrom(targetStoreUri, fileName, fileName + ".part"); + Files.move(Paths.get(targetPath, fileName + ".part"), Paths.get(targetPath, fileName)); + } catch (Exception e) { + logger.error("Az '{}' állomány visszatöltése sikertelen. A rendszer üzenete: {}", fileName, e.getMessage()); + throw e; + } + + return null; + } +} diff --git a/server/-product/production/LOCAL/jobs/steps/TestForkCancelableStep.java b/server/-product/production/LOCAL/jobs/steps/TestForkCancelableStep.java new file mode 100644 index 00000000..c1b49f76 --- /dev/null +++ b/server/-product/production/LOCAL/jobs/steps/TestForkCancelableStep.java @@ -0,0 +1,35 @@ +package user.jobengine.server.steps; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + +import user.commons.IJob; +import user.commons.JobStatus; +import user.commons.ListUtils; +import user.jobengine.server.IJobEngine; +import user.jobengine.server.IJobRuntime; + +public class TestForkCancelableStep extends JobStep { + private static final String CHILD_TITLE = "Párhuzamosított alfolyamat"; + // private static final String CHILD_TEMPLATE = "fake-concurrent.xml"; + private static final String CHILD_TEMPLATE = "cancelable.xml"; + private static final Logger logger = LogManager.getLogger(); + int count = 3; + + @StepEntry + public Object[] execute(IJobEngine jobEngine, IJobRuntime jobRuntime) throws Exception { + for (int i = 0; i < count; i++) { + IJobRuntime rt = getEngine().submit(null, e -> { + if (e.getStatus().equals(JobStatus.SUSPENDED)) { + IJobRuntime rt = (IJobRuntime) e.getSource(); + logger.info("Cleanup on SUSPEND {}, {}", rt.getId(), rt.isDisableRetry()); + } + }, CHILD_TEMPLATE, "JOB " + i, 0, IJobEngine.DEFAULT_OWNER, ListUtils.asMap("param", i)); + + rt.setRelated("TEST" + rt.getId()); + } + + logger.info("Done"); + return null; + } +} diff --git a/server/-product/production/LOCAL/jobs/steps/TranscodeFFAStranStep.java b/server/-product/production/LOCAL/jobs/steps/TranscodeFFAStranStep.java new file mode 100644 index 00000000..a73ff663 --- /dev/null +++ b/server/-product/production/LOCAL/jobs/steps/TranscodeFFAStranStep.java @@ -0,0 +1,151 @@ +package user.jobengine.server.steps; + +import java.io.File; +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; + +import org.apache.commons.io.FilenameUtils; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.apache.logging.log4j.Marker; +import org.apache.logging.log4j.message.Message; +import org.apache.logging.log4j.message.ParameterizedMessage; + +import user.commons.FFAStransAPI; +import user.commons.IFFAStransAPI; +import user.commons.StoreUri; +import user.commons.mediatool.Timecode; +import user.commons.mediatool.Timecode.Type; +import user.commons.remotestore.RemoteStoreProtocol; +import user.jobengine.db.FileType; +import user.jobengine.db.IItemManager; +import user.jobengine.db.Media; +import user.jobengine.db.Store; +import user.jobengine.server.IJobEngine; +import user.jobengine.server.IJobRuntime; + +public class TranscodeFFAStranStep extends JobStep { + private static final int POLL_INTERVALL = 3000; + private static final String MP4EXT = ".MP4"; + private static final String MXFEXT = ".MXF"; + private static final String LOWRES_FILETYPE = "Low-res"; + private static final Logger logger = LogManager.getLogger("TranscodeFFAStranStep"); + private IItemManager manager; + private Store store; + private FileType fileType; + private Media mediaCubeMedia; + private Marker marker; + + @StepEntry + public Object[] execute(ArchiveItem archiveItem, Media mediaCubeMedia, String transcoderAddress, String transcoderTemplateName, + String globalHiresSourcePath, String localLowresTargetPath, boolean deleteSource, IJobEngine jobEngine, IJobRuntime jobRuntime) throws Exception { + + this.marker = jobRuntime.getSessionMarker(); + this.manager = jobEngine.getItemManager(); + this.store = check(manager.getCurrentLowresStore(), "lowres Store"); + this.fileType = check(manager.getFileType(LOWRES_FILETYPE), "lowres FileType"); + this.mediaCubeMedia = check(mediaCubeMedia, "mediaCubeMedia"); + check(archiveItem, "archiveItem"); + check(transcoderAddress, "transcoderAddress"); + check(transcoderTemplateName, "transcoderTemplateName"); + check(globalHiresSourcePath, "globalHiresSourcePath"); + check(localLowresTargetPath, "localLowresTargetPath"); + + File sourceMediaFile = new File(archiveItem.getMediaFile()); + logger.info("Transcoding {}", archiveItem.getMediaFile()); + String sourceFileName = sourceMediaFile.getName(); + Timecode timecode = new Timecode(mediaCubeMedia.getLength(), Type.PAL); + + String details = String.format("%s (%s, %d bytes)", sourceFileName, timecode.toString(), sourceMediaFile.length()); + + StoreUri storeUri = store.getTargetStoreUri(RemoteStoreProtocol.LOCAL); + if (storeUri == null) + throw new Exception("Can not detect proxy folder."); + + String webPath = storeUri.toString(true); + + Path targetPath = null; + try { + String targetFileName = FilenameUtils.removeExtension(sourceFileName) + MP4EXT; + targetPath = Paths.get(localLowresTargetPath, targetFileName); + if (!targetPath.toFile().exists()) { + // jobRuntime.setDescription(String.format("%s: %s", jobRuntime.getDescription(), details)); + jobRuntime.setDescription(String.format("%s transzkódolása", details)); + String sourceFile = Paths.get(globalHiresSourcePath, sourceFileName).toString(); + IFFAStransAPI api = new FFAStransAPI(transcoderAddress, p -> { + if (p <= 100) + jobRuntime.incrementProgress(p); + }); + + api.submit(transcoderTemplateName, sourceFile); + api.monitor(POLL_INTERVALL); + } + + //a sikeres transzkod utan nem mindig van ott egybol a fajl + long started = System.currentTimeMillis(); + while (!targetPath.toFile().exists()) { + long current = System.currentTimeMillis(); + //max 5 perc varakozas + if (current - started > 5 * 60 * 1000) + throw new Exception("Transcode job target file access timed out"); + Thread.sleep(POLL_INTERVALL); + } + + postprocess(targetPath, webPath); + + } catch (Exception e) { + logger.catching(e); + Message m = new ParameterizedMessage("{} átkódolás hiba: {}", sourceFileName, e.getMessage()); + logger.error(marker, m); + throw new Exception(m.getFormattedMessage()); + } finally { + try { + if (deleteSource && sourceMediaFile != null && sourceMediaFile.exists()) + sourceMediaFile.delete(); + } catch (Exception e) { + logger.catching(e); + } + try { + if (deleteSource && targetPath != null && targetPath.toFile().exists()) + Files.delete(targetPath); + } catch (Exception e) { + logger.catching(e); + } + } + return null; + } + + private void postprocess(Path transcodedFilePath, String webPath) throws IOException { + Path lowresPath = null; + try { + String transcodedFileName = transcodedFilePath.getFileName().toString(); + String targetPath = null; + if (transcodedFileName.indexOf(".") > 2) { + Path subdir = Paths.get(transcodedFileName.substring(0, 1), transcodedFileName.substring(1, 2), transcodedFileName.substring(2, 3)); + EscortFiles.ensureUNCFolder(webPath, subdir.toString()); + targetPath = Paths.get(subdir.toString(), transcodedFileName).toString(); + } else { + targetPath = transcodedFileName; + } + lowresPath = Paths.get(webPath, targetPath); + int version = 1; + while (lowresPath.toFile().exists()) { + String fileName = transcodedFileName + version + MP4EXT; + lowresPath = Paths.get(lowresPath.toString().replace(transcodedFileName, fileName)); + targetPath = targetPath.replace(transcodedFileName, fileName); + transcodedFileName = fileName; + version++; + } + + Files.move(transcodedFilePath, lowresPath); + manager.createMediaFile(targetPath, fileType, store, mediaCubeMedia).add(); + } catch (IOException e) { + logger.catching(e); + logger.error(marker, "A '{}' állomány mozgatása a '{}' helyre nem sikerült.", transcodedFilePath, lowresPath); + throw e; + } + } + +} diff --git a/server/-product/production/LOCAL/jobs/steps/UpdateGhostMediaDataStep.java b/server/-product/production/LOCAL/jobs/steps/UpdateGhostMediaDataStep.java new file mode 100644 index 00000000..e59008d1 --- /dev/null +++ b/server/-product/production/LOCAL/jobs/steps/UpdateGhostMediaDataStep.java @@ -0,0 +1,88 @@ +package user.jobengine.server.steps; + +import java.util.List; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.apache.logging.log4j.Marker; + +import user.commons.remotestore.RemoteStoreProtocol; +import user.jobengine.db.IItemManager; +import user.jobengine.db.Media; +import user.jobengine.db.MediaFile; +import user.jobengine.db.Store; +import user.jobengine.server.IJobEngine; +import user.jobengine.server.IJobRuntime; +import user.jobengine.server.steps.MetadataTypeDetector.MetadataType; + +public class UpdateGhostMediaDataStep extends JobStep { + private static final Logger logger = LogManager.getLogger(); + private Marker marker; + + @StepEntry + public Object[] execute(Media mediaCubeMedia, IJobEngine jobEngine, IJobRuntime jobRuntime) throws Exception { + marker = jobRuntime.getSessionMarker(); + + IItemManager manager = jobEngine.getItemManager(); + //Refresh from db + List mediaFiles = manager.getMedia(mediaCubeMedia.getId()).getMediaFiles(); + if (mediaFiles != null && mediaFiles.size() == 2) { + MediaFile lowres = null; + MediaFile highres = null; + + for (MediaFile mf : mediaFiles) { + if (mf.getStore().getSourceStoreUri(RemoteStoreProtocol.HTTP) != null) + lowres = mf; + else + highres = mf; + } + + if (highres == null) { + logger.info(marker, "Nincs highres mediaId: {}", mediaCubeMedia.getId()); + return null; + } + if (lowres == null) { + logger.info(marker, "Nincs lowres mediaId: {}", mediaCubeMedia.getId()); + return null; + } + + String id = MetadataTypeDetector.truncateExtension(highres.getRelativePath()); + id = MetadataTypeDetector.truncateVersion(id); + boolean detect = MetadataTypeDetector.GuessMetadataType(id) == MetadataType.OctopusPlaceholder + || MetadataTypeDetector.GuessMetadataType(id) == MetadataType.OctopusStory; + if (!detect) { + logger.info(marker, "Nem bejátszó mediaId: {}, file: {}", mediaCubeMedia.getId(), highres.getRelativePath()); + return null; + } + + Store highresStore = manager.getSystemStore(false); + final long sourceMediaId = lowres.getId(); + final long highresMediaFileId = highres.getId(); + final String highresRealtivePath = highres.getRelativePath(); + + manager.executeQuery("SELECT mediaid FROM mediafile WHERE relativepath=? and storeid=? and id!=?", rs -> { + long mediaId = rs.getLong(1); + Media media = manager.getMedia(mediaId); + if (media.getMediaFilesCount() == 1) { + logger.info(marker, "Hiányzó szellem lowres hozzáadása {} alapján", media.getId()); + + MediaFile mf = (MediaFile) manager.get(MediaFile.class, sourceMediaId); + mf.setMedia(media); + mf.setId(0); + manager.add(mf); + media.setLength(mediaCubeMedia.getLength()); + manager.modify(media); + } + return true; + }, st -> { + st.setString(1, highresRealtivePath); + st.setLong(2, highresStore.getId()); + st.setLong(3, highresMediaFileId); + }); + + } + + return null; + } + +} diff --git a/server/-product/production/LOCAL/jobs/templates/archive-limited.xml b/server/-product/production/LOCAL/jobs/templates/archive-limited.xml new file mode 100644 index 00000000..cf1a2469 --- /dev/null +++ b/server/-product/production/LOCAL/jobs/templates/archive-limited.xml @@ -0,0 +1,41 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/server/-product/production/LOCAL/jobs/templates/archive-material.xml b/server/-product/production/LOCAL/jobs/templates/archive-material.xml new file mode 100644 index 00000000..973ac9fa --- /dev/null +++ b/server/-product/production/LOCAL/jobs/templates/archive-material.xml @@ -0,0 +1,49 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/server/-product/production/LOCAL/jobs/templates/archive-ondemand.xml b/server/-product/production/LOCAL/jobs/templates/archive-ondemand.xml new file mode 100644 index 00000000..22665b33 --- /dev/null +++ b/server/-product/production/LOCAL/jobs/templates/archive-ondemand.xml @@ -0,0 +1,41 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/server/-product/production/LOCAL/jobs/templates/archive-recursive.xml b/server/-product/production/LOCAL/jobs/templates/archive-recursive.xml new file mode 100644 index 00000000..02b97884 --- /dev/null +++ b/server/-product/production/LOCAL/jobs/templates/archive-recursive.xml @@ -0,0 +1,25 @@ + + + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/server/-product/production/LOCAL/jobs/templates/batch-retrieve-ondemand.xml b/server/-product/production/LOCAL/jobs/templates/batch-retrieve-ondemand.xml new file mode 100644 index 00000000..20332cf0 --- /dev/null +++ b/server/-product/production/LOCAL/jobs/templates/batch-retrieve-ondemand.xml @@ -0,0 +1,29 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/server/-product/production/LOCAL/jobs/templates/cancelable.xml b/server/-product/production/LOCAL/jobs/templates/cancelable.xml new file mode 100644 index 00000000..2be289b3 --- /dev/null +++ b/server/-product/production/LOCAL/jobs/templates/cancelable.xml @@ -0,0 +1,17 @@ + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/server/-product/production/LOCAL/jobs/templates/common-copy.xml b/server/-product/production/LOCAL/jobs/templates/common-copy.xml new file mode 100644 index 00000000..7abb6fcf --- /dev/null +++ b/server/-product/production/LOCAL/jobs/templates/common-copy.xml @@ -0,0 +1,41 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/server/-product/production/LOCAL/jobs/templates/create-lowres-ondemand.xml b/server/-product/production/LOCAL/jobs/templates/create-lowres-ondemand.xml new file mode 100644 index 00000000..f7922734 --- /dev/null +++ b/server/-product/production/LOCAL/jobs/templates/create-lowres-ondemand.xml @@ -0,0 +1,106 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/server/-product/production/LOCAL/jobs/templates/delete-materials.xml b/server/-product/production/LOCAL/jobs/templates/delete-materials.xml new file mode 100644 index 00000000..365fe19c --- /dev/null +++ b/server/-product/production/LOCAL/jobs/templates/delete-materials.xml @@ -0,0 +1,17 @@ + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/server/-product/production/LOCAL/jobs/templates/migrate-hsm.xml b/server/-product/production/LOCAL/jobs/templates/migrate-hsm.xml new file mode 100644 index 00000000..faec8724 --- /dev/null +++ b/server/-product/production/LOCAL/jobs/templates/migrate-hsm.xml @@ -0,0 +1,21 @@ + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/server/-product/production/LOCAL/jobs/templates/retrieve-ondemand.xml b/server/-product/production/LOCAL/jobs/templates/retrieve-ondemand.xml new file mode 100644 index 00000000..b58d8fdf --- /dev/null +++ b/server/-product/production/LOCAL/jobs/templates/retrieve-ondemand.xml @@ -0,0 +1,150 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/server/-product/production/LOCAL/jobs/templates/retrieve.xml b/server/-product/production/LOCAL/jobs/templates/retrieve.xml new file mode 100644 index 00000000..e12fd950 --- /dev/null +++ b/server/-product/production/LOCAL/jobs/templates/retrieve.xml @@ -0,0 +1,21 @@ + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/server/-product/production/LOCAL/jobs/templates/sync-subtitles.xml b/server/-product/production/LOCAL/jobs/templates/sync-subtitles.xml new file mode 100644 index 00000000..2fc761c3 --- /dev/null +++ b/server/-product/production/LOCAL/jobs/templates/sync-subtitles.xml @@ -0,0 +1,21 @@ + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/server/-product/production/LOCAL/jobs/templates/sys-recreate-lowres.xml b/server/-product/production/LOCAL/jobs/templates/sys-recreate-lowres.xml new file mode 100644 index 00000000..9ef84b31 --- /dev/null +++ b/server/-product/production/LOCAL/jobs/templates/sys-recreate-lowres.xml @@ -0,0 +1,118 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/server/-product/production/LOCAL/jobs/templates/template1.yaml b/server/-product/production/LOCAL/jobs/templates/template1.yaml new file mode 100644 index 00000000..4459b9fa --- /dev/null +++ b/server/-product/production/LOCAL/jobs/templates/template1.yaml @@ -0,0 +1,28 @@ +jobtemplate: + multiInstance: true + useSessionLog: true + declarations: + parameters: + - name: param1 + type: java.lang.Integer + - name: param1 + type: java.lang.Integer + variables: + - name: var1 + type: java.lang.Integer + - name: var1 + type: java.lang.Integer + commands: + - type: Step1.java + weight: 1 + inputs: + - name: param1 + - name: param2 + outputs: + - name: var1 + - name: var2 + - type: StepS.java + weight: 1 + inputs: + - name: var1 + - name: var2 diff --git a/server/-product/production/LOCAL/jobs/templates/test-fork-cancelable.xml b/server/-product/production/LOCAL/jobs/templates/test-fork-cancelable.xml new file mode 100644 index 00000000..8c88d84b --- /dev/null +++ b/server/-product/production/LOCAL/jobs/templates/test-fork-cancelable.xml @@ -0,0 +1,6 @@ + + + + + + \ No newline at end of file diff --git a/server/-product/production/LOCAL/mediacube.bat b/server/-product/production/LOCAL/mediacube.bat new file mode 100644 index 00000000..099f5aee --- /dev/null +++ b/server/-product/production/LOCAL/mediacube.bat @@ -0,0 +1,17 @@ +@echo off +if exist tmp rmdir tmp /s /q +mkdir tmp +java ^ +-Dorg.eclipse.epp.logging.aeri.skipReports=true ^ +-Declipse.ignoreApp=true ^ +-Dosgi.noShutdown=true ^ +-Dlog4j.configurationFile=settings/log4j2.xml ^ +-Djetty.home=settings ^ +-Djetty.etc.config.urls=jetty.xml ^ +-Djava.io.tmpdir=tmp ^ +-Dfile.encoding=UTF-8 ^ +-Dgosh.home=configuration ^ +-jar plugins/org.eclipse.equinox.launcher_1.3.201.v20161025-1711.jar ^ +-Xms512m ^ +-Xmx1024m ^ +-console diff --git a/server/-product/production/LOCAL/settings/application.yaml b/server/-product/production/LOCAL/settings/application.yaml new file mode 100644 index 00000000..d7c9e8ed --- /dev/null +++ b/server/-product/production/LOCAL/settings/application.yaml @@ -0,0 +1,51 @@ +datasource: + mediacube: +# url: jdbc:db2://192.168.100.2:50000/mc +# user: db2inst1 +# password: password + url: jdbc:db2://10.10.1.27:50000/mc + user: db2admin + password: password + external-indexer: false + simple-search: true + login-timeout: 3 + pool-size: 10 + mediacube-nosql: +# url: jdbc:db2://192.168.100.2:50000/mc +# user: db2inst1 +# password: password + url: jdbc:db2://10.10.1.27:50000/mc + user: db2admin + password: password + schema: test + login-timeout: 3 + hsm: + url: jdbc:db2://10.11.1.89:51500/tsmdb1 + user: tsminst1 + password: tsminst1 + planair: + url: jdbc:sqlserver://10.11.254.86;databaseName=PA_Vivantis; + user: MAM + password: VDani +services: + ffmpeg: + execurable-location: /opt/ffmpeg/ffmpeg + mediacube: + proxy-root: /mediacube/data/lowres + nexio: + host: 10.10.1.55 + collection-name: nexioclips + use-mos-gateway: true + disabled: true +jobs: + validate-transfers: false + copy-buffer-size: 32768 + scheduled-execution-disabled: true +tsm: + randomize-archives: false + delimiter: / + node-name: JOBENGINE + fs-name: /JOBENGINE + alternate-fs-name: /JOBENGINE + hl-name: /JOBENGINE + \ No newline at end of file diff --git a/server/-product/production/LOCAL/settings/dsm.opt b/server/-product/production/LOCAL/settings/dsm.opt new file mode 100644 index 00000000..0a0a0ded --- /dev/null +++ b/server/-product/production/LOCAL/settings/dsm.opt @@ -0,0 +1,3 @@ +SErvername MEDIACUBE +traceflags api +tracefile /opt/mediacube/log/tsm.trace \ No newline at end of file diff --git a/server/-product/production/LOCAL/settings/ffmpeg.yaml b/server/-product/production/LOCAL/settings/ffmpeg.yaml new file mode 100644 index 00000000..94a90ca5 --- /dev/null +++ b/server/-product/production/LOCAL/settings/ffmpeg.yaml @@ -0,0 +1,14 @@ +location: /Programs/ffmpeg/bin/ffmpeg.exe +default: + - -v panic + - -stats + - -y +profiles: +- name: profile1 + settings: + - setting1 + - setting2 +- name: profile2 + settings: + - setting3 + - setting4 diff --git a/server/-product/production/LOCAL/settings/jetty.xml b/server/-product/production/LOCAL/settings/jetty.xml new file mode 100644 index 00000000..5e47cacd --- /dev/null +++ b/server/-product/production/LOCAL/settings/jetty.xml @@ -0,0 +1,45 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/server/-product/production/LOCAL/settings/log4j2.xml b/server/-product/production/LOCAL/settings/log4j2.xml new file mode 100644 index 00000000..ad946223 --- /dev/null +++ b/server/-product/production/LOCAL/settings/log4j2.xml @@ -0,0 +1,103 @@ + + + + log + ${logPath}/mediacube.log + ${logPath}/movtest.log + ${logPath}/$${date:yyyy-MM}/movtest-%d{MM-dd-yyyy}-%i.log.gz + ${logPath}/delete-materials.log + ${logPath}/$${date:yyyy-MM}/delete-materials-%d{MM-dd-yyyy}-%i.log.gz + ${logPath}/$${date:yyyy-MM}/mediacube-%d{MM-dd-yyyy}-%i.log.gz + ${logPath}/markered-mediacube.log + ${logPath}/$${date:yyyy-MM}/markered-mediacube-%d{MM-dd-yyyy}-%i.log.gz + ${logPath}/mediacube-err.log + ${logPath}/$${date:yyyy-MM}/mediacube-err-%d{MM-dd-yyyy}-%i.log.gz + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/server/-product/production/LOCAL/settings/maestro.yaml b/server/-product/production/LOCAL/settings/maestro.yaml new file mode 100644 index 00000000..1d361571 --- /dev/null +++ b/server/-product/production/LOCAL/settings/maestro.yaml @@ -0,0 +1,38 @@ +sourceStoreUri: + name: Default + protocol: LOCAL + uri: "/mnt/NLE" + fileFilter: "*.mxf" + showDirectories: true +alternateSourceStoreUris: +- name: NLE1 + protocol: LOCAL + uri: "/mnt/NLE/NLE1" + fileFilter: "*.mxf" + showDirectories: true +- name: NLE2 + protocol: LOCAL + uri: "/mnt/NLE/NLE2" + fileFilter: "*.mxf" + showDirectories: true +- name: NLE3 + protocol: LOCAL + uri: "/mnt/NLE/NLE3" + fileFilter: "*.mxf" + showDirectories: true +- name: NLE4 + protocol: LOCAL + uri: "/mnt/NLE/NLE4" + fileFilter: "*.mxf" + showDirectories: true +- name: POLC + protocol: LOCAL + uri: "/mnt/POLC" + fileFilter: "*.mxf" + showDirectories: true +targets: +- name: FINISHED_SHOWS + killDateDays: 7 + storeUri: + protocol: LOCAL + uri: "/mnt/PROMISE/FINISHED_SHOWS" diff --git a/server/-product/production/LOCAL/settings/mediacube.yaml b/server/-product/production/LOCAL/settings/mediacube.yaml new file mode 100644 index 00000000..347ca896 --- /dev/null +++ b/server/-product/production/LOCAL/settings/mediacube.yaml @@ -0,0 +1,53 @@ +jobQueuePollInterval: 1000 +disableHelp: true +maestroDisabled: false +alternateRetrieveSelector: false +disableStatistics: true +disableEditor: false +topTypeFilters: +- name: Hír bejátszó +- name: Hír nyers +- name: Visszarögzített +- name: Egyéb +bottomTypeFilters: +- name: Műsor +- name: Műsor nyers +- name: Promo +- name: Promo nyers +- name: Reklám +- name: Reklám nyers +authentication: + authEnabled: true + defaultUser: root + defaultPassword: password + adHost: intra.mediavivantis.hu + adNonSecurePort: 3268 + adBaseDn: DC=intra,DC=mediavivantis,DC=hu + adAdminMap: + - G_MV_U_MUSZAK + - G_MV_U_INGEST + adSubmitterMap: + - G_ECH_U_INFORMATIKUSOK + - G_ECH_U_MUSZAKVEZETOK + - ECH-ISILON-ADMINS + adEditorMap: + - G_ECH_U_INFORMATIKUSOK + - G_ECH_U_MUSZAKVEZETOK + - ECH-ISILON-ADMINS + localAccounts: + - user: user + password: 5F4DCC3B5AA765D61D8327DEB882CF99 + email: + - user: lebony + password: 4E25B117B14D86D7DCECB4E433CF932C + email: + - user: root + password: 5F4DCC3B5AA765D61D8327DEB882CF99 + email: vasary@elgekko.net + localAdmins: + - root + - admin + localSubmitters: + - lebony + localEditors: + - editor diff --git a/server/-product/production/MEDIAVIVANTIS/configuration/config.ini b/server/-product/production/MEDIAVIVANTIS/configuration/config.ini index fd6901cd..265bf754 100644 --- a/server/-product/production/MEDIAVIVANTIS/configuration/config.ini +++ b/server/-product/production/MEDIAVIVANTIS/configuration/config.ini @@ -1,5 +1,5 @@ #This configuration file was written by: org.eclipse.equinox.internal.frameworkadmin.equinox.EquinoxFwConfigFileParser -#Thu Mar 04 15:26:35 CET 2021 +#Thu Mar 11 15:00:06 CET 2021 osgi.bundles=reference\:file\:cglib_2.2.2.jar@4,reference\:file\:com.fasterxml.jackson.core.jackson-annotations_2.4.5.jar@4,reference\:file\:com.fasterxml.jackson.core.jackson-core_2.4.5.jar@4,reference\:file\:com.fasterxml.jackson.core.jackson-databind_2.4.5.jar@4,reference\:file\:com.fasterxml.jackson.dataformat.jackson-dataformat-yaml_2.4.5.jar@4,reference\:file\:com.fasterxml.jackson.datatype.jackson-datatype-joda_2.4.5.jar@4,reference\:file\:com.fasterxml.jackson.jaxrs.jackson-jaxrs-base_2.4.5.jar@4,reference\:file\:com.fasterxml.jackson.jaxrs.jackson-jaxrs-json-provider_2.4.5.jar@4,reference\:file\:com.fasterxml.jackson.module.jackson-module-jaxb-annotations_2.4.5.jar@4,reference\:file\:com.ibm.db2.jcc_1.4.0.jar@4,reference\:file\:com.ibm.nosql_4.19.26.jar@4,reference\:file\:com.microsoft.sqlserver.sqljdbc_6.0.8112.100.jar@4,reference\:file\:com.sun.jna_4.2.0.jar@4,reference\:file\:groovy_3.0.3.jar@4,reference\:file\:io.humble.video-arch-x86_64-pc-linux-gnu6_0.2.1.jar@4,reference\:file\:io.humble.video-arch-x86_64-w64-mingw32_0.2.1.jar@4,reference\:file\:io.humble.video-noarch_0.2.1.jar@4,reference\:file\:javax.annotation-api_1.2.0.jar@4,reference\:file\:javax.mail_1.5.0.b01.jar@4,reference\:file\:javax.servlet-api_3.1.0.jar@4,reference\:file\:javax.ws.rs-api_2.0.1.jar@4,reference\:file\:jcifs_1.3.17.jar@4,reference\:file\:joda-time_2.2.0.jar@4,reference\:file\:junit_4.12.0.jar@4,reference\:file\:org.apache.aries.spifly.dynamic.bundle_1.0.8.jar@4,reference\:file\:org.apache.aries.util_1.0.0.jar@4,reference\:file\:org.apache.commons.beanutils_1.8.3.jar@4,reference\:file\:org.apache.commons.collections_3.2.2.jar@4,reference\:file\:org.apache.commons.digester_3.2.0.jar@4,reference\:file\:org.apache.commons.io_2.6.0.jar@4,reference\:file\:org.apache.commons.lang_2.6.0.jar@4,reference\:file\:org.apache.commons.net_3.6.0.jar@4,reference\:file\:org.apache.felix.gogo.command_0.12.0.jar@4,reference\:file\:org.apache.felix.gogo.runtime_0.12.0.jar@4,reference\:file\:org.apache.felix.gogo.shell_0.12.0.jar@4,reference\:file\:org.apache.httpcomponents.httpclient_4.2.6.jar@4,reference\:file\:org.apache.httpcomponents.httpcore_4.2.5.jar@4,reference\:file\:org.apache.logging.log4j.api_2.8.2.jar@1\:start,reference\:file\:org.apache.logging.log4j.core_2.8.2.jar@4,reference\:file\:org.apache.logging.log4j.slf4j-impl_2.8.2.jar@4,reference\:file\:org.apache.servicemix.bundles.quartz_2.3.0.2.jar@4,reference\:file\:org.eclipse.equinox.common_3.8.0.v20160509-1230.jar@2\:start,reference\:file\:org.eclipse.equinox.console_1.1.200.v20150929-1405.jar@4,reference\:file\:org.eclipse.equinox.ds_1.4.400.v20160226-2036.jar@1\:start,reference\:file\:org.eclipse.equinox.launcher_1.3.201.v20161025-1711.jar@4,reference\:file\:org.eclipse.equinox.util_1.0.500.v20130404-1337.jar@4,reference\:file\:org.eclipse.jetty.client_9.3.9.v20160517.jar@4,reference\:file\:org.eclipse.jetty.deploy_9.3.9.v20160517.jar@4,reference\:file\:org.eclipse.jetty.http_9.3.9.v20160517.jar@4,reference\:file\:org.eclipse.jetty.io_9.3.9.v20160517.jar@4,reference\:file\:org.eclipse.jetty.osgi.boot_9.3.9.v20160517.jar@4\:start,reference\:file\:org.eclipse.jetty.schemas_3.1.0.jar@4,reference\:file\:org.eclipse.jetty.security_9.3.9.v20160517.jar@4,reference\:file\:org.eclipse.jetty.server_9.3.9.v20160517.jar@4,reference\:file\:org.eclipse.jetty.servlet_9.3.9.v20160517.jar@4,reference\:file\:org.eclipse.jetty.util_9.3.9.v20160517.jar@4,reference\:file\:org.eclipse.jetty.webapp_9.3.9.v20160517.jar@4,reference\:file\:org.eclipse.jetty.websocket.api_9.3.9.v20160517.jar@4,reference\:file\:org.eclipse.jetty.websocket.client_9.3.9.v20160517.jar@4,reference\:file\:org.eclipse.jetty.websocket.common_9.3.9.v20160517.jar@4,reference\:file\:org.eclipse.jetty.websocket.server_9.3.9.v20160517.jar@4,reference\:file\:org.eclipse.jetty.websocket.servlet_9.3.9.v20160517.jar@4,reference\:file\:org.eclipse.jetty.xml_9.3.9.v20160517.jar@4,reference\:file\:org.eclipse.osgi.services_3.2.100.v20100503.jar@4,reference\:file\:org.hamcrest.core_1.3.0.jar@4,reference\:file\:org.jboss.resteasy.client_3.0.11.Final.jar@4,reference\:file\:org.jboss.resteasy.jaxb-provider_3.0.11.Final.jar@4,reference\:file\:org.jboss.resteasy.jaxrs_3.0.11.Final.jar@4,reference\:file\:org.jboss.resteasy.jaxrs-api_3.0.11.Final.jar@4,reference\:file\:org.jmock_2.6.0.jar@4,reference\:file\:org.jmock.junit4_2.6.0.jar@4,reference\:file\:org.mybatis.mybatis_3.5.2.jar@4,reference\:file\:org.objectweb.asm_5.0.1.jar@4,reference\:file\:org.objectweb.asm.commons_5.0.1.jar@4,reference\:file\:org.objectweb.asm.tree_5.0.1.jar@4,reference\:file\:org.omnifaces_3.4.1.jar@4,reference\:file\:slf4j.api_1.7.24.jar@4,reference\:file\:slf4j.simple_1.7.2.jar@4,reference\:file\:user.commons.log4j2_1.0.0.jar@4,reference\:file\:user.commons.zk_8.0.3.jar@4,reference\:file\:user.jobengine.osgi.commons_1.0.0.jar@4\:start,reference\:file\:user.jobengine.osgi.db_1.0.0.jar@4,reference\:file\:user.jobengine.osgi.server_1.0.0.jar@4,reference\:file\:user.jobengine.osgi.services_1.0.0.jar@4,reference\:file\:user.mediacube.gui_1.0.0.jar@4,reference\:file\:user.mediacube.metadata_1.0.0.jar@4,reference\:file\:user.tsm.client_1.2.0.jar@4 equinox.use.ds=true osgi.bundles.defaultStartLevel=4 diff --git a/server/-product/production/MEDIAVIVANTIS/jobs/dsm.opt b/server/-product/production/MEDIAVIVANTIS/jobs/dsm.opt deleted file mode 100644 index 4955eaa4..00000000 --- a/server/-product/production/MEDIAVIVANTIS/jobs/dsm.opt +++ /dev/null @@ -1,3 +0,0 @@ -SErvername MEDIACUBE -*traceflags api -*tracefile /opt/mediacube/log/tsm.trace \ No newline at end of file diff --git a/server/-product/production/MEDIAVIVANTIS/jobs/steps/CreateMissingLowresStep.java b/server/-product/production/MEDIAVIVANTIS/jobs/steps/CreateMissingLowresStep.java index 5a1ddc5f..19bad479 100644 --- a/server/-product/production/MEDIAVIVANTIS/jobs/steps/CreateMissingLowresStep.java +++ b/server/-product/production/MEDIAVIVANTIS/jobs/steps/CreateMissingLowresStep.java @@ -9,7 +9,6 @@ import com.ibm.nosql.json.api.BasicDBObject; import com.ibm.nosql.json.api.DB; import com.ibm.nosql.json.api.DBCollection; -import user.commons.MediaCubeMarker; import user.commons.MediaCubeUndoMarker; import user.commons.nosql.NoSQLUtils; import user.jobengine.db.IItemManager; @@ -19,12 +18,10 @@ import user.jobengine.server.IJobRuntime; public class CreateMissingLowresStep extends JobStep { private static final Logger logger = LogManager.getLogger(); - private MediaCubeMarker marker; @StepEntry public Object[] execute(String localHiresPath, IJobEngine jobEngine, IJobRuntime jobRuntime) throws Exception { Object[] result = new Object[] { null, null, "%s", null, 0, true }; - marker = (MediaCubeMarker) jobRuntime.getSessionMarker(); DB db = NoSQLUtils.getNoSQLDB(); DBCollection collection = db.getCollection("missing_lowres"); IItemManager manager = jobEngine.getItemManager(); @@ -32,7 +29,7 @@ public class CreateMissingLowresStep extends JobStep { try { if (media == null) { - logger.info(new MediaCubeUndoMarker(marker.getSessionID()), "Nincs feldolgozandó hiány."); + logger.info(new MediaCubeUndoMarker(getSessionMarker().getSessionID()), "Nincs feldolgozandó hiány."); // throw new Exception("Nincs feldolgozandó hiány."); cancel(); return null; @@ -44,11 +41,11 @@ public class CreateMissingLowresStep extends JobStep { archiveItem.setMediaFile(Paths.get(localHiresPath, name).toString()); result[1] = archiveItem; collection.save(new BasicDBObject("name", name)); - logger.info(marker, "Hiányzó lowres mediaId: {}", media.getId()); + logger.info(getSessionMarker(), "Processing mediaID: {}", media.getId()); } catch (Exception e) { logger.catching(e); - logger.info(marker, e.getMessage()); + logger.error(getSessionMarker(), e.getMessage()); throw e; } finally { setProgress(100); @@ -58,16 +55,23 @@ public class CreateMissingLowresStep extends JobStep { private Media getFirstUntranscodedMedia(IItemManager manager, DBCollection collection) { Media[] result = new Media[] { null }; - String query = "SELECT mediaid FROM VW_MISSING_PROXY_IDS WHERE HOUSEID like 'M%' or HOUSEID like 'P%' or HOUSEID like 'R%' ORDER BY modified DESC FETCH FIRST ROW ONLY"; + //MV + String query = "SELECT mediaid FROM VW_MISSING_PROXY_IDS WHERE HOUSEID like 'M%' or HOUSEID like 'P%' or HOUSEID like 'R%' ORDER BY modified DESC"; + + //HTV + //String query = "SELECT mediaid FROM VW_MISSING_PROXY_IDS"; manager.executeQuery(query, rs -> { try { long mediaId = rs.getLong(1); Media media = manager.getMedia(mediaId); //a nevgeneralas miatt az eredeti MediaFilesName nem jo, a pontos nev kell nekunk String name = media.getMediaFileRealName(); + logger.info(getSessionMarker(), "Checking {}", name); long existing = collection.find(new BasicDBObject("name", name)).count(); - if (existing > 0) + if (existing > 0) { + logger.info(getSessionMarker(), "{} is on missing_lowres list", name); return true; + } result[0] = media; } catch (Exception e) { diff --git a/server/-product/production/MEDIAVIVANTIS/jobs/steps/TSMExtendedRetrieveStep.java b/server/-product/production/MEDIAVIVANTIS/jobs/steps/TSMExtendedRetrieveStep.java index 0074883c..79a601ca 100644 --- a/server/-product/production/MEDIAVIVANTIS/jobs/steps/TSMExtendedRetrieveStep.java +++ b/server/-product/production/MEDIAVIVANTIS/jobs/steps/TSMExtendedRetrieveStep.java @@ -1,5 +1,8 @@ package user.jobengine.server.steps; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + import user.commons.StoreUri; import user.commons.remotestore.RemoteStoreProtocol; import user.jobengine.db.ArchivedMedia; @@ -8,10 +11,13 @@ import user.jobengine.server.IJobEngine; import user.jobengine.server.IJobRuntime; public class TSMExtendedRetrieveStep extends TSMRestoreStep { + private static String NEXIO_HOST = System.getProperty("nexio.host"); + // private static String NEXIO_HOST = SystemConfiguration.getInstance().value("services.nexio.host"); + private static final Logger logger = LogManager.getLogger(); + private boolean useNexioTarget; private int nexioPort; private String nexioUserName, nexioPassword; - private String nexioHost; private String nexioAgency; @Override @@ -42,12 +48,12 @@ public class TSMExtendedRetrieveStep extends TSMRestoreStep { @Override protected StoreUri createTargetUri(IItemManager manager, String targetPath) throws NullPointerException { StoreUri result = null; + logger.info(getSessionMarker(), "Create target uri {}", targetPath); if (useNexioTarget) { - nexioHost = System.getProperty("nexio.host"); - if (nexioHost == null) { + if (NEXIO_HOST == null) { throw new NullPointerException("Missing system property on 'nexio.host' name"); } - result = manager.createStoreUri(RemoteStoreProtocol.FTP, nexioHost); + result = manager.createStoreUri(RemoteStoreProtocol.FTP, NEXIO_HOST); result.setPortNumber(nexioPort); result.setUserName(nexioUserName); result.setPassword(nexioPassword); diff --git a/server/-product/production/MEDIAVIVANTIS/mediacube.sh b/server/-product/production/MEDIAVIVANTIS/mediacube.sh index b70ef743..24204315 100644 --- a/server/-product/production/MEDIAVIVANTIS/mediacube.sh +++ b/server/-product/production/MEDIAVIVANTIS/mediacube.sh @@ -1,12 +1,13 @@ +cd /opt/mediacube-new rm -fr tmp/* rm -fr configuration/org.eclipse.osgi -export DSMI_CONFIG=/opt/mediacube/configuration/dsm.opt +export DSMI_CONFIG=/opt/mediacube-new/settings/dsm.opt +export DSMI_LOG=/opt/mediacube-new/log export DSMI_DIR=/opt/tivoli/tsm/client/api/bin64 -export DSMI_LOG=/opt/mediacube/log export DSM_DIR=/opt/tivoli/tsm/client/api/bin64 -export LD_LIBRARY_PATH=/opt/tivoli/tsm/client/ba/bin:/opt/tivoli/tsm/client/api/bin64:/opt/mediacube -export LIBPATH=/opt/tivoli/tsm/client/ba/bin:/opt/tivoli/tsm/client/api/bin64 -export SHLIB_PATH=/opt/tivoli/tsm/client/ba/bin:/opt/tivoli/tsm/client/api/bin64 +export LD_LIBRARY_PATH=/opt/tivoli/tsm/server/bin/dbbkapi:/opt/mediacube-new +export LIBPATH=/opt/tivoli/tsm/server/bin/dbbkapi +export SHLIB_PATH=/opt/tivoli/tsm/server/bin/dbbkapi export TZ=NFT-1DFT,M3.5.0,M10.5.0 export PATH=/usr/lib/jvm/java-1.8.0-openjdk-1.8.0.102-4.b14.el7.x86_64/bin:$PATH java \ @@ -14,11 +15,14 @@ java \ -Dorg.eclipse.epp.logging.aeri.skipReports=true \ -Declipse.ignoreApp=true \ -Dosgi.noShutdown=true \ --Dlog4j.configurationFile=configuration/log4j2.xml \ --Djetty.home=configuration/jetty \ --Djetty.etc.config.urls=jetty.xml,jetty-ssl.xml,jetty-ssl-context.xml,jetty-http.xml,jetty-https.xml \ +-Dlog4j.configurationFile=settings/log4j2.xml \ +-Djetty.home=settings \ +-Djetty.etc.config.urls=jetty.xml \ +-Dgosh.home=configuration \ -Djava.io.tmpdir=tmp \ +-Dfile.encoding=UTF-8 \ -jar plugins/org.eclipse.equinox.launcher_1.3.201.v20161025-1711.jar \ +-agentlib:jdwp=transport=dt_socket,server=y,suspend=n,address=8889 \ -Xms512m \ --Xmx1024m \ --console 5555 +-Xmx4096m \ +-console diff --git a/server/-product/production/MEDIAVIVANTIS/settings/application.yaml b/server/-product/production/MEDIAVIVANTIS/settings/application.yaml index 8d4aff66..ffd0b6ff 100644 --- a/server/-product/production/MEDIAVIVANTIS/settings/application.yaml +++ b/server/-product/production/MEDIAVIVANTIS/settings/application.yaml @@ -25,7 +25,7 @@ services: ffmpeg: execurable-location: /opt/ffmpeg/ffmpeg mediacube: - proxy-root: /_workspace/data/video + proxy-root: /mediacube/data/lowres nexio: host: 10.10.1.55 collection-name: nexioclips diff --git a/server/-product/production/MEDIAVIVANTIS/settings/dsm.opt b/server/-product/production/MEDIAVIVANTIS/settings/dsm.opt new file mode 100644 index 00000000..0a0a0ded --- /dev/null +++ b/server/-product/production/MEDIAVIVANTIS/settings/dsm.opt @@ -0,0 +1,3 @@ +SErvername MEDIACUBE +traceflags api +tracefile /opt/mediacube/log/tsm.trace \ No newline at end of file diff --git a/server/RemoteSystemsTempFiles/.project b/server/RemoteSystemsTempFiles/.project new file mode 100644 index 00000000..76756293 --- /dev/null +++ b/server/RemoteSystemsTempFiles/.project @@ -0,0 +1,12 @@ + + + RemoteSystemsTempFiles + + + + + + + org.eclipse.rse.ui.remoteSystemsTempNature + + diff --git a/server/hu.user.mediacube.executors.tests/data/hu/user/mediacube/executors/tests/data/TestChild.java b/server/hu.user.mediacube.executors.tests/data/hu/user/mediacube/executors/tests/data/TestChild.java new file mode 100644 index 00000000..382ff4a3 --- /dev/null +++ b/server/hu.user.mediacube.executors.tests/data/hu/user/mediacube/executors/tests/data/TestChild.java @@ -0,0 +1,10 @@ +package hu.user.mediacube.executors.tests.data; + +public class TestChild extends TestParent { + + @Override + public void hello() { + super.hello(); + System.out.println("Hello from child"); + } +} diff --git a/server/hu.user.mediacube.executors.tests/data/hu/user/mediacube/executors/tests/data/TestParent.java b/server/hu.user.mediacube.executors.tests/data/hu/user/mediacube/executors/tests/data/TestParent.java new file mode 100644 index 00000000..d3d3ef21 --- /dev/null +++ b/server/hu.user.mediacube.executors.tests/data/hu/user/mediacube/executors/tests/data/TestParent.java @@ -0,0 +1,8 @@ +package hu.user.mediacube.executors.tests.data; + +public class TestParent { + + public void hello() { + System.out.println("Hello from parent"); + } +} diff --git a/server/hu.user.mediacube.executors.tests/jobs/ffmpeg.yaml b/server/hu.user.mediacube.executors.tests/jobs/ffmpeg.yaml new file mode 100644 index 00000000..94a90ca5 --- /dev/null +++ b/server/hu.user.mediacube.executors.tests/jobs/ffmpeg.yaml @@ -0,0 +1,14 @@ +location: /Programs/ffmpeg/bin/ffmpeg.exe +default: + - -v panic + - -stats + - -y +profiles: +- name: profile1 + settings: + - setting1 + - setting2 +- name: profile2 + settings: + - setting3 + - setting4 diff --git a/server/hu.user.mediacube.executors.tests/src/hu/user/mediacube/executors/tests/SmallTests.java b/server/hu.user.mediacube.executors.tests/src/hu/user/mediacube/executors/tests/SmallTests.java index d97cc44a..46e13d60 100644 --- a/server/hu.user.mediacube.executors.tests/src/hu/user/mediacube/executors/tests/SmallTests.java +++ b/server/hu.user.mediacube.executors.tests/src/hu/user/mediacube/executors/tests/SmallTests.java @@ -34,9 +34,11 @@ import java.util.LinkedHashMap; import java.util.LinkedHashSet; import java.util.List; import java.util.Map; +import java.util.Map.Entry; import java.util.Set; import java.util.TreeMap; import java.util.TreeSet; +import java.util.concurrent.TimeUnit; import java.util.regex.Pattern; import org.apache.commons.io.FileUtils; @@ -45,6 +47,7 @@ import org.apache.commons.lang.StringUtils; import org.apache.ibatis.jdbc.SQL; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; +import org.codehaus.groovy.control.CompilationFailedException; import org.junit.Test; import com.ibm.nosql.json.api.BasicDBList; @@ -980,17 +983,29 @@ public class SmallTests { @Test public void test9995() throws Exception { - String className = "TestStep1"; - String root = "/opt/steps"; + String file = "/opt/test/TestChild.java"; + + + GroovyClassLoader cl = new GroovyClassLoader(this.getClass().getClassLoader()) { + + @Override + public Class loadClass(String name, boolean lookupScriptFiles, boolean preferClassOverScript, boolean resolve) + throws ClassNotFoundException, CompilationFailedException { + // TODO Auto-generated method stub + return super.loadClass(name, lookupScriptFiles, preferClassOverScript, resolve); + } + + }; - try (GroovyClassLoader cl = new GroovyClassLoader(this.getClass().getClassLoader())) { + try { - Class myClass = parseClassHierarchy(cl, root, className); + Class myClass = cl.parseClass(new File(file)); System.out.println(myClass.getCanonicalName()); Method method = myClass.getMethod("execute"); method.invoke(myClass.newInstance()); - } catch (Exception e) { - throw e; + System.out.println("done"); + } catch (Throwable e) { + System.out.println(e.getClass().getSimpleName() + ": " + e.getMessage()); } } @@ -1262,7 +1277,79 @@ public class SmallTests { @Test public void test999991() throws Exception { IConfiguration configuration = SystemConfiguration.getInstance(); - String executors = configuration.getConfig("jobs/executors.yaml"); - configuration.load(executors); + String configFile = configuration.getConfig("jobs/executors.yaml"); + HashMap settings = configuration.load(configFile, HashMap.class); + assertNotNull(settings); + } + + @Test + public void test999992() throws Exception { + IConfiguration configuration = SystemConfiguration.getInstance(); + String configFile = configuration.getConfig("jobs/ffmpeg.yaml"); + HashMap settings = configuration.load(configFile, HashMap.class); + assertNotNull(settings); } + + //AMC hianyzo hangsav hibakereses + //select finished,related from job where name='Archiválás' and finished is not null order by finished + //dsmadmc -id=support -password=userkft -TABdelimited -dataonly=yes "select backup_date ||','|| replace(replace(LL_NAME, '.mxf', ''),'/','') from backups where NODE_NAME='PASANODE' and state='ACTIVE_VERSION'" + @Test + public void test999993() throws Exception { + List jobs = Files.readAllLines(Paths.get("/opt/AMC/2021/TSM-ERROR/archive-jobs")); + HashMap jobsMap = new HashMap<>(); + SimpleDateFormat df = new SimpleDateFormat("yyyy-MM-dd hh:mm:ss.SSS"); + + for (String line : jobs) { + String[] tokens = line.split(","); + Date dt = df.parse(tokens[0]); + String name = tokens[1]; + name = name.split(" / ")[0]; + Date stored = jobsMap.get(name); + if (stored == null) + jobsMap.put(name, dt); + else if (stored.before(dt)) + jobsMap.put(name, dt); + // System.out.println(name); + } + + List tsm = Files.readAllLines(Paths.get("/opt/AMC/2021/TSM-ERROR/archive-tsm")); + HashMap tsmMap = new HashMap<>(); + SimpleDateFormat df1 = new SimpleDateFormat("yyyy-MM-dd-hh.mm.ss.SSSSSS"); + + for (String line : tsm) { + String[] tokens = line.split(","); + Date dt = df1.parse(tokens[0]); + String name = tokens[1].replace(".part", ""); + tsmMap.put(name, dt); + //System.out.println(name); + } + + System.out.println("NAME, TSM DATE, ARCHIVE DATE"); + for (Entry entry : jobsMap.entrySet()) { + + if (entry.getKey().contains("CCEM047245")) { + System.out.println("x"); + } + Date dtTsm = tsmMap.get(entry.getKey()); + if (dtTsm == null) { + continue; + } + + Date dtArchive = entry.getValue(); + long diffInMillies = dtArchive.getTime() - dtTsm.getTime(); + long diff = TimeUnit.DAYS.convert(diffInMillies, TimeUnit.MILLISECONDS); + if (diff > 0) + System.out.println(entry.getKey() + ", " + df.format(dtTsm) + ", " + df.format(dtArchive)); + } + + for (Entry entry : jobsMap.entrySet()) { + + Date dtTsm = tsmMap.get(entry.getKey()); + if (dtTsm == null) { + System.out.println("Missing " + entry.getKey() + ", " + df.format(entry.getValue())); + } + + } + } + } diff --git a/server/user.jobengine.executors/amc/user/jobengine/server/steps/FileSearchFilterOptions.java b/server/user.jobengine.executors/amc/user/jobengine/server/steps/FileSearchFilterOptions.java new file mode 100644 index 00000000..436c3be0 --- /dev/null +++ b/server/user.jobengine.executors/amc/user/jobengine/server/steps/FileSearchFilterOptions.java @@ -0,0 +1,41 @@ +package user.jobengine.server.steps; + +import java.nio.file.Path; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +import com.ibm.nosql.json.api.BasicDBObject; + +public class FileSearchFilterOptions { + + private BasicDBObject filter; + + public FileSearchFilterOptions(BasicDBObject filter) { + this.filter = filter; + } + + public boolean acceptFile(Path file) { + if (filter == null) + return true; + + if (filter.containsKey("fileName")) { + //.*\.(sh|ini|conf|vhost|xml|php)$ + String fileNamePattern = filter.getString("fileName"); + if (fileNamePattern == null || fileNamePattern.trim().length() == 0) + return true; + + Pattern pattern = Pattern.compile(fileNamePattern, Pattern.CASE_INSENSITIVE); + + Matcher matcher = pattern.matcher(file.getFileName().toString()); + if (matcher.find()) + return true; + + } + + return false; + } + + public boolean preAcceptDirectory(Path file) { + return true; + } +} diff --git a/server/user.jobengine.executors/amc/user/jobengine/server/steps/PathItemsCollectorStep.java b/server/user.jobengine.executors/amc/user/jobengine/server/steps/PathItemsCollectorStep.java new file mode 100644 index 00000000..6955a8a9 --- /dev/null +++ b/server/user.jobengine.executors/amc/user/jobengine/server/steps/PathItemsCollectorStep.java @@ -0,0 +1,66 @@ +package user.jobengine.server.steps; + +import java.io.IOException; +import java.nio.file.FileVisitResult; +import java.nio.file.FileVisitor; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.nio.file.SimpleFileVisitor; +import java.nio.file.attribute.BasicFileAttributes; +import java.util.ArrayList; +import java.util.List; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + +import com.ibm.nosql.json.api.BasicDBObject; + +/* DO NOT REMOVE! +import user.jobengine.server.steps.FileSearchFilterOptions; +*/ +public class PathItemsCollectorStep extends JobStep { + private static final Logger logger = LogManager.getLogger(); + + @StepEntry + public Object[] execute(String sourceFolder, BasicDBObject filter) throws Exception { + Path sourcePath = Paths.get(sourceFolder); + + List files = new ArrayList<>(); + + //teljes rekurzivitas szuressel + FileSearchFilterOptions filterOptions = new FileSearchFilterOptions(filter); + FileVisitor visitor = new SimpleFileVisitor() { + @Override + public FileVisitResult preVisitDirectory(Path dir, BasicFileAttributes attrs) throws IOException { + if (!filterOptions.preAcceptDirectory(dir)) + return FileVisitResult.SKIP_SUBTREE; + return FileVisitResult.CONTINUE; + } + + @Override + public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) throws IOException { + if (filterOptions.acceptFile(file)) + files.add(file.toString()); + logger.info(file); + return FileVisitResult.CONTINUE; + } + + @Override + public FileVisitResult visitFileFailed(Path file, IOException exc) throws IOException { + return FileVisitResult.CONTINUE; + } + + }; + + try { + Files.walkFileTree(sourcePath, visitor); + } catch (Exception e) { + logger.error(getSessionMarker(), "Hiba a '{}' mappa feldolgozásában. A rendszer hibaüzenete: {}", sourcePath, e.getMessage()); + logger.catching(e); + throw e; + } finally { + } + return new Object[] { files }; + } +} diff --git a/server/user.jobengine.executors/mediavivantis/user/jobengine/server/steps/CreateMissingLowresStep.java b/server/user.jobengine.executors/mediavivantis/user/jobengine/server/steps/CreateMissingLowresStep.java index 568b23c1..19bad479 100644 --- a/server/user.jobengine.executors/mediavivantis/user/jobengine/server/steps/CreateMissingLowresStep.java +++ b/server/user.jobengine.executors/mediavivantis/user/jobengine/server/steps/CreateMissingLowresStep.java @@ -9,7 +9,6 @@ import com.ibm.nosql.json.api.BasicDBObject; import com.ibm.nosql.json.api.DB; import com.ibm.nosql.json.api.DBCollection; -import user.commons.MediaCubeMarker; import user.commons.MediaCubeUndoMarker; import user.commons.nosql.NoSQLUtils; import user.jobengine.db.IItemManager; @@ -19,12 +18,10 @@ import user.jobengine.server.IJobRuntime; public class CreateMissingLowresStep extends JobStep { private static final Logger logger = LogManager.getLogger(); - private MediaCubeMarker marker; @StepEntry public Object[] execute(String localHiresPath, IJobEngine jobEngine, IJobRuntime jobRuntime) throws Exception { Object[] result = new Object[] { null, null, "%s", null, 0, true }; - marker = (MediaCubeMarker) jobRuntime.getSessionMarker(); DB db = NoSQLUtils.getNoSQLDB(); DBCollection collection = db.getCollection("missing_lowres"); IItemManager manager = jobEngine.getItemManager(); @@ -32,7 +29,7 @@ public class CreateMissingLowresStep extends JobStep { try { if (media == null) { - logger.info(new MediaCubeUndoMarker(marker.getSessionID()), "Nincs feldolgozandó hiány."); + logger.info(new MediaCubeUndoMarker(getSessionMarker().getSessionID()), "Nincs feldolgozandó hiány."); // throw new Exception("Nincs feldolgozandó hiány."); cancel(); return null; @@ -44,11 +41,11 @@ public class CreateMissingLowresStep extends JobStep { archiveItem.setMediaFile(Paths.get(localHiresPath, name).toString()); result[1] = archiveItem; collection.save(new BasicDBObject("name", name)); - logger.info(marker, "Hiányzó lowres mediaId: {}", media.getId()); + logger.info(getSessionMarker(), "Processing mediaID: {}", media.getId()); } catch (Exception e) { logger.catching(e); - logger.info(marker, e.getMessage()); + logger.error(getSessionMarker(), e.getMessage()); throw e; } finally { setProgress(100); @@ -58,17 +55,23 @@ public class CreateMissingLowresStep extends JobStep { private Media getFirstUntranscodedMedia(IItemManager manager, DBCollection collection) { Media[] result = new Media[] { null }; - //String query = "SELECT mediaid FROM VW_MISSING_PROXY_IDS WHERE HOUSEID like 'M%' or HOUSEID like 'P%' or HOUSEID like 'R%' ORDER BY modified DESC FETCH FIRST ROW ONLY"; - String query = "SELECT mediaid FROM VW_MISSING_PROXY_IDS"; + //MV + String query = "SELECT mediaid FROM VW_MISSING_PROXY_IDS WHERE HOUSEID like 'M%' or HOUSEID like 'P%' or HOUSEID like 'R%' ORDER BY modified DESC"; + + //HTV + //String query = "SELECT mediaid FROM VW_MISSING_PROXY_IDS"; manager.executeQuery(query, rs -> { try { long mediaId = rs.getLong(1); Media media = manager.getMedia(mediaId); //a nevgeneralas miatt az eredeti MediaFilesName nem jo, a pontos nev kell nekunk String name = media.getMediaFileRealName(); + logger.info(getSessionMarker(), "Checking {}", name); long existing = collection.find(new BasicDBObject("name", name)).count(); - if (existing > 0) + if (existing > 0) { + logger.info(getSessionMarker(), "{} is on missing_lowres list", name); return true; + } result[0] = media; } catch (Exception e) { diff --git a/server/user.jobengine.executors/src/user/jobengine/server/steps/FFMpeg.java b/server/user.jobengine.executors/src/user/jobengine/server/steps/FFMpeg.java index d3e8eec7..dfe0b04f 100644 --- a/server/user.jobengine.executors/src/user/jobengine/server/steps/FFMpeg.java +++ b/server/user.jobengine.executors/src/user/jobengine/server/steps/FFMpeg.java @@ -10,6 +10,9 @@ import org.apache.commons.lang.StringUtils; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; +import com.ibm.nosql.json.api.BasicDBObject; + +import user.commons.configuration.SystemConfiguration; import user.commons.mediatool.MediaInfo; public class FFMpeg { @@ -133,4 +136,90 @@ public class FFMpeg { Files.write(Paths.get(output, "index.m3u8"), indexLines); } + + static public void withProfile(String input, String output, String profile, IProgressChanged progressCallback) throws Exception { + BasicDBObject config = SystemConfiguration.getInstance().load("ffmpeg.yaml", BasicDBObject.class); + + StringBuilder sb = new StringBuilder(); + sb.append("-v panic -stats -y") + .append("\r\n") + .append(String.format("-i %s", input)) + .append("\r\n") + .append("-map 0:0 -c:v h264 -an -hls_time 100000000000 -hls_list_size 0") + .append("\r\n") + .append(String.format("-f hls %s/video.m3u8", output)) + .append("\r\n"); + + for (int i = 0; i < 4; i++) { + sb.append(String.format("-map 0:%d -f segment -segment_time 100000000000 -segment_list_size 0", i+1)) + .append("\r\n") + .append(String.format("-segment_list %s/audio%d.m3u8 -segment_format mpegts %s/audio%d%%d.aac", output, i, output, i)) + .append("\r\n"); + } + System.out.println(sb); + ProcessBuilder processBuilder = Cmd.create(Cmd.getFFMpegExecutable(), sb); + long allFrames[] = {0}; + + try { + MediaInfo mi = new MediaInfo(Paths.get(input)); + mi.process(); + allFrames[0] = mi.getFrames(); + } catch (Exception e ){ + System.err.println(e); + + } + + Cmd.execute(processBuilder, false, l -> { + if (allFrames[0] == 0) { + logger.debug(l); + System.out.println(l); + return; + } + + if (progressCallback == null) + return; + if (l.contains(FRAME) && l.contains(FPS)) { + String p = StringUtils.substringBetween(l, FRAME, FPS); + if (p != null) { + p = p.trim(); + try { + int currentFrames = Integer.parseInt(p); + progressCallback.onProgressChanged((long)currentFrames * 100 / allFrames[0]); + } catch (Exception e){} + } + } + }); + +// #EXTM3U +// #EXT-X-MEDIA:TYPE=AUDIO,GROUP-ID="audio",NAME="CH1",URI="audio0.m3u8" +// #EXT-X-MEDIA:TYPE=AUDIO,GROUP-ID="audio",NAME="CH2",URI="audio1.m3u8" +// #EXT-X-MEDIA:TYPE=AUDIO,GROUP-ID="audio",NAME="CH3",URI="audio2.m3u8" +// #EXT-X-MEDIA:TYPE=AUDIO,GROUP-ID="audio",NAME="CH4",URI="audio3.m3u8" +// #EXT-X-STREAM-INF:PROGRAM-ID=1,AUDIO="audio" +// video.m3u8 + + if (!Paths.get(output, "video.m3u8").toFile().exists()) + throw new FileNotFoundException("video.m3u8"); + if (!Paths.get(output, "video0.ts").toFile().exists()) + throw new FileNotFoundException("video0.ts"); + for (int i = 0; i < 4; i++) { + String file = String.format("audio%d.m3u8", i); + if (!Paths.get(output, file).toFile().exists()) + throw new FileNotFoundException(file); + file = String.format("audio%d0.aac", i); + if (!Paths.get(output, file).toFile().exists()) + throw new FileNotFoundException(file); + } + + List indexLines = new ArrayList<>(); + indexLines.add("#EXTM3U"); + for (int i = 0; i < 4; i++) { + indexLines.add(String.format("#EXT-X-MEDIA:TYPE=AUDIO,GROUP-ID=\"audio\",NAME=\"CH%d\",URI=\"audio%d.m3u8\"", i+1, i)); + } + indexLines.add("#EXT-X-STREAM-INF:PROGRAM-ID=1,AUDIO=\"audio\""); + indexLines.add("video.m3u8"); + + Files.write(Paths.get(output, "index.m3u8"), indexLines); + } + } diff --git a/server/user.jobengine.executors/src/user/jobengine/server/steps/TSMExtendedRetrieveStep.java b/server/user.jobengine.executors/src/user/jobengine/server/steps/TSMExtendedRetrieveStep.java index 49368387..79a601ca 100644 --- a/server/user.jobengine.executors/src/user/jobengine/server/steps/TSMExtendedRetrieveStep.java +++ b/server/user.jobengine.executors/src/user/jobengine/server/steps/TSMExtendedRetrieveStep.java @@ -1,7 +1,9 @@ package user.jobengine.server.steps; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + import user.commons.StoreUri; -import user.commons.configuration.SystemConfiguration; import user.commons.remotestore.RemoteStoreProtocol; import user.jobengine.db.ArchivedMedia; import user.jobengine.db.IItemManager; @@ -9,12 +11,13 @@ import user.jobengine.server.IJobEngine; import user.jobengine.server.IJobRuntime; public class TSMExtendedRetrieveStep extends TSMRestoreStep { - private static String NEXIO_HOST = SystemConfiguration.getInstance().value("services.nexio.host"); + private static String NEXIO_HOST = System.getProperty("nexio.host"); + // private static String NEXIO_HOST = SystemConfiguration.getInstance().value("services.nexio.host"); + private static final Logger logger = LogManager.getLogger(); private boolean useNexioTarget; private int nexioPort; private String nexioUserName, nexioPassword; - private String nexioHost; private String nexioAgency; @Override @@ -45,6 +48,7 @@ public class TSMExtendedRetrieveStep extends TSMRestoreStep { @Override protected StoreUri createTargetUri(IItemManager manager, String targetPath) throws NullPointerException { StoreUri result = null; + logger.info(getSessionMarker(), "Create target uri {}", targetPath); if (useNexioTarget) { if (NEXIO_HOST == null) { throw new NullPointerException("Missing system property on 'nexio.host' name"); diff --git a/server/user.jobengine.osgi.commons/src/user/commons/configuration/IConfiguration.java b/server/user.jobengine.osgi.commons/src/user/commons/configuration/IConfiguration.java index 67eff03d..04b2e33f 100644 --- a/server/user.jobengine.osgi.commons/src/user/commons/configuration/IConfiguration.java +++ b/server/user.jobengine.osgi.commons/src/user/commons/configuration/IConfiguration.java @@ -6,8 +6,6 @@ public interface IConfiguration { String getConfig(String relativeConfigName) throws FileNotFoundException; - void load(String file) throws Exception; - T load(String file, Class configClass) throws Exception; T value(String fqn); diff --git a/server/user.jobengine.osgi.commons/src/user/commons/configuration/SystemConfiguration.java b/server/user.jobengine.osgi.commons/src/user/commons/configuration/SystemConfiguration.java index df2d02f2..cd575a5c 100644 --- a/server/user.jobengine.osgi.commons/src/user/commons/configuration/SystemConfiguration.java +++ b/server/user.jobengine.osgi.commons/src/user/commons/configuration/SystemConfiguration.java @@ -29,12 +29,18 @@ public class SystemConfiguration implements IConfiguration { private Map settings; public SystemConfiguration() { + currentInstance = this; + } + + @SuppressWarnings("unchecked") + private void ensureSettingsLoaded() { + if (settings != null) + return; try { - load(getConfig("settings/application.yaml")); + settings = load(getConfig("settings/application.yaml"), HashMap.class); } catch (Exception e) { logger.catching(e); } - currentInstance = this; } @Override @@ -48,16 +54,6 @@ public class SystemConfiguration implements IConfiguration { throw new FileNotFoundException("Cant find configuration file " + relativeConfigName); } - @SuppressWarnings("unchecked") - @Override - public void load(String file) throws Exception { - logger.info("Loading system configuration {}", file); - if (file != null) { - String yaml = new String(Files.readAllBytes(Paths.get(file))); - settings = OBJECT_MAPPER.readValue(yaml, HashMap.class); - } - } - @Override public T load(String file, Class configClass) throws Exception { T result = null; @@ -74,6 +70,9 @@ public class SystemConfiguration implements IConfiguration { public T value(String fqn) { T result = null; String[] tokens = fqn.split("\\."); + + ensureSettingsLoaded(); + Map parent = settings; for (int i = 0; i < tokens.length - 1; i++) { parent = (Map) parent.get(tokens[i]); diff --git a/server/user.jobengine.osgi.commons/src/user/commons/nexio/NexioDataMiner.java b/server/user.jobengine.osgi.commons/src/user/commons/nexio/NexioDataMiner.java index 42280e75..2e175ae9 100644 --- a/server/user.jobengine.osgi.commons/src/user/commons/nexio/NexioDataMiner.java +++ b/server/user.jobengine.osgi.commons/src/user/commons/nexio/NexioDataMiner.java @@ -289,25 +289,32 @@ public class NexioDataMiner implements Runnable, INexioChangeListener { int count = 0; try (PreparedStatement stmt = con.prepareStatement(querySQL); ResultSet rs = stmt.executeQuery();) { while (rs.next()) { - BasicDBObject clip = new BasicDBObject(); - clip.put(ID, rs.getString(ID)); - clip.put(LONGNAMEID, rs.getString(LONGNAMEID)); - clip.put(EXTAGENCY, rs.getString(EXTAGENCY)); - clip.put(VIDEOBITRATE, rs.getString(VIDEOBITRATE)); - clip.put(VIDEOFORMAT, rs.getString(VIDEOFORMAT)); - clip.put(DURATION, rs.getLong(DURATION)); - Timestamp record = rs.getTimestamp(RECORDTIMESTAMP); - clip.put(RECORDDATE, record); - //NEXIO GW modified date nem jo a regi klippeknel - Timestamp modified = rs.getTimestamp(MODIFIEDTIMESTAMP); - if (modified.before(record)) - clip.put(MODIFIEDTIMESTAMP, record); - else - clip.put(MODIFIEDTIMESTAMP, modified); - clip.put(KILLDATE, rs.getTimestamp(KILLDATE)); - collection.insert(clip); - count++; + try { + BasicDBObject clip = new BasicDBObject(); + clip.put(ID, rs.getString(ID)); + clip.put(LONGNAMEID, rs.getString(LONGNAMEID)); + clip.put(EXTAGENCY, rs.getString(EXTAGENCY)); + clip.put(VIDEOBITRATE, rs.getString(VIDEOBITRATE)); + clip.put(VIDEOFORMAT, rs.getString(VIDEOFORMAT)); + clip.put(DURATION, rs.getLong(DURATION)); + + Timestamp record = rs.getTimestamp(RECORDTIMESTAMP); + clip.put(RECORDDATE, record); + //NEXIO GW modified date nem jo a regi klippeknel + Timestamp modified = rs.getTimestamp(MODIFIEDTIMESTAMP); + + //neha elrontja, es ez is null + if (modified != null && modified.before(record)) + clip.put(MODIFIEDTIMESTAMP, record); + else + clip.put(MODIFIEDTIMESTAMP, modified); + clip.put(KILLDATE, rs.getTimestamp(KILLDATE)); + collection.insert(clip); + count++; + } catch (Exception e1) { + logger.error("Inner exception", e1); + } } logger.info("Transfered {} clips", count); } catch (Exception e) { diff --git a/server/user.jobengine.osgi.db/migrations/environments/vm.properties b/server/user.jobengine.osgi.db/migrations/environments/vm.properties new file mode 100644 index 00000000..769ba9e1 --- /dev/null +++ b/server/user.jobengine.osgi.db/migrations/environments/vm.properties @@ -0,0 +1,83 @@ +# +# Copyright 2010-2017 the original author or authors. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +## Base time zone to ensure times are consistent across machines +time_zone=GMT+0:00 + +## The character set that scripts are encoded with +# script_char_set=UTF-8 + +## JDBC connection properties. +driver=com.ibm.db2.jcc.DB2Driver +url=jdbc:db2://192.168.100.2:50000/mc:retrieveMessagesFromServerOnGetMessage=true; +username=db2inst1 +password=password + +# +# A NOTE ON STORED PROCEDURES AND DELIMITERS +# +# Stored procedures and functions commonly have nested delimiters +# that conflict with the schema migration parsing. If you tend +# to use procs, functions, triggers or anything that could create +# this situation, then you may want to experiment with +# send_full_script=true (preferred), or if you can't use +# send_full_script, then you may have to resort to a full +# line delimiter such as "GO" or "/" or "!RUN!". +# +# Also play with the autocommit settings, as some drivers +# or databases don't support creating procs, functions or +# even tables in a transaction, and others require it. +# + +# This ignores the line delimiters and +# simply sends the entire script at once. +# Use with JDBC drivers that can accept large +# blocks of delimited text at once. +send_full_script=false + +# This controls how statements are delimited. +# By default statements are delimited by an +# end of line semicolon. Some databases may +# (e.g. MS SQL Server) may require a full line +# delimiter such as GO. +# These are ignored if send_full_script is true. +delimiter=@ +full_line_delimiter=false + +# If set to true, each statement is isolated +# in its own transaction. Otherwise the entire +# script is executed in one transaction. +# Few databases should need this set to true, +# but some do. +auto_commit=false + +# If set to false, warnings from the database will interrupt migrations. +ignore_warnings=false + +# Custom driver path to allow you to centralize your driver files +# Default requires the drivers to be in the drivers directory of your +# initialized migration directory (created with "migrate init") +# driver_path= + +# Name of the table that tracks changes to the database +changelog=CHANGELOG + +# Migrations support variable substitutions in the form of ${variable} +# in the migration scripts. All of the above properties will be ignored though, +# with the exception of changelog. +# Example: The following would be referenced in a migration file as ${ip_address} +# ip_address=192.168.0.1 + diff --git a/server/user.jobengine.osgi.db/migrations/scripts/!006_first_missing_proxy.sql b/server/user.jobengine.osgi.db/migrations/scripts/!006_first_missing_proxy.sql deleted file mode 100644 index c7d8c42e..00000000 --- a/server/user.jobengine.osgi.db/migrations/scripts/!006_first_missing_proxy.sql +++ /dev/null @@ -1,42 +0,0 @@ --- --- Copyright 2010-2016 the original author or authors. --- --- Licensed under the Apache License, Version 2.0 (the "License"); --- you may not use this file except in compliance with the License. --- You may obtain a copy of the License at --- --- http://www.apache.org/licenses/LICENSE-2.0 --- --- Unless required by applicable law or agreed to in writing, software --- distributed under the License is distributed on an "AS IS" BASIS, --- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. --- See the License for the specific language governing permissions and --- limitations under the License. --- - --- // First missing proxy --- Migration SQL that makes the change goes here. - - - -CREATE FUNCTION FIRST_MISSING_PROXY_ID () - RETURNS BIGINT - LANGUAGE SQL - READS SQL DATA - NO EXTERNAL ACTION - DETERMINISTIC - RETURN - select m.id as mediaid from vw_mediafiles v, mediafile mf, filetype ft, media m where - m.id=mf.mediaid and - mf.mediaid = v.mediaid and - v.mediafilecount=1 and - mf.filetypeid=ft.id and - ft.name='High-res' - order by m.archived desc - fetch first row only -@ - --- //@UNDO --- SQL to undo the change goes here. -DROP FUNCTION FIRST_MISSING_PROXY_ID -@ diff --git a/server/user.jobengine.osgi.db/migrations/scripts/001_create_structure.sql b/server/user.jobengine.osgi.db/migrations/scripts/001_create_structure.sql index b0b4ac35..c6c2d1c8 100644 --- a/server/user.jobengine.osgi.db/migrations/scripts/001_create_structure.sql +++ b/server/user.jobengine.osgi.db/migrations/scripts/001_create_structure.sql @@ -394,24 +394,24 @@ CREATE TABLE MEDIADESCRIPTION ( ) @ -ALTER TABLE DB2ADMIN.MEDIADESCRIPTION +ALTER TABLE MEDIADESCRIPTION ADD CONSTRAINT FK_MEDIADESCRIPTION_MEDIAID FOREIGN KEY(MEDIAID) - REFERENCES DB2ADMIN.MEDIA(ID) + REFERENCES MEDIA(ID) ON DELETE CASCADE ON UPDATE NO ACTION @ -ALTER TABLE DB2ADMIN.MEDIADESCRIPTION +ALTER TABLE MEDIADESCRIPTION ADD CONSTRAINT FK_MEDIADESCRIPTION_MEDIAFILEID FOREIGN KEY(MEDIAFILEID) - REFERENCES DB2ADMIN.MEDIAFILE(ID) + REFERENCES MEDIAFILE(ID) ON DELETE CASCADE ON UPDATE NO ACTION @ -ALTER TABLE DB2ADMIN.MEDIADESCRIPTION +ALTER TABLE MEDIADESCRIPTION ADD CONSTRAINT FK_MEDIADESCRIPTION_ITEMID FOREIGN KEY(ITEMID) - REFERENCES DB2ADMIN.ITEM(ID) + REFERENCES ITEM(ID) ON DELETE CASCADE ON UPDATE NO ACTION @ diff --git a/server/user.jobengine.osgi.db/migrations/scripts/009_add_related_to_job.sql b/server/user.jobengine.osgi.db/migrations/scripts/009_add_related_to_job.sql index fddbea58..1a0c7e31 100644 --- a/server/user.jobengine.osgi.db/migrations/scripts/009_add_related_to_job.sql +++ b/server/user.jobengine.osgi.db/migrations/scripts/009_add_related_to_job.sql @@ -23,7 +23,7 @@ ALTER TABLE JOB ADD COLUMN RELATED VARCHAR(255) CREATE INDEX IDX_JOB_RELATED ON JOB ("RELATED") @ -CALL SYSPROC.ADMIN_CMD('REORG TABLE DB2ADMIN.JOB') +CALL SYSPROC.ADMIN_CMD('REORG TABLE JOB') @ -- //@UNDO @@ -35,5 +35,5 @@ DROP INDEX IDX_JOB_RELATED ALTER TABLE JOB DROP COLUMN RELATED @ -CALL SYSPROC.ADMIN_CMD('REORG TABLE DB2ADMIN.JOB') +CALL SYSPROC.ADMIN_CMD('REORG TABLE JOB') @ diff --git a/server/user.jobengine.osgi.db/migrations/scripts/027_create_vod_store.sql b/server/user.jobengine.osgi.db/migrations/scripts/027_create_vod_store.sql index afeacf1e..9da22962 100644 --- a/server/user.jobengine.osgi.db/migrations/scripts/027_create_vod_store.sql +++ b/server/user.jobengine.osgi.db/migrations/scripts/027_create_vod_store.sql @@ -4,9 +4,11 @@ INSERT INTO STORE (NAME, ISSYSTEM, ISLOWRES) VALUES ('VOD', 'N', 'N') @ +-- @DELIMITER ; INSERT INTO STOREURI(STOREID,PROTOCOL,DELIVERY,URI,ISSTREAM,ISSOURCE,ISTARGET,USERNAME,PASSWORD,ROOTPATH,PORTNUMBER) VALUES ((SELECT ID FROM STORE WHERE NAME='VOD'),'FTP',null,'192.168.0.101','N','Y','Y','ftpuser','FtPVs3r@1',null,null) -@ +; +-- @DELIMITER @ -- //@UNDO -- SQL to undo the change goes here. diff --git a/server/user.jobengine.osgi.db/migrations/scripts/028_alter_storeuri_incrase_username.sql b/server/user.jobengine.osgi.db/migrations/scripts/028_alter_storeuri_incrase_username.sql index 5b00ff9e..b96046e7 100644 --- a/server/user.jobengine.osgi.db/migrations/scripts/028_alter_storeuri_incrase_username.sql +++ b/server/user.jobengine.osgi.db/migrations/scripts/028_alter_storeuri_incrase_username.sql @@ -19,7 +19,7 @@ ALTER TABLE STOREURI ALTER COLUMN USERNAME SET DATA TYPE VARCHAR(50) @ -CALL SYSPROC.ADMIN_CMD ( 'REORG TABLE DB2ADMIN.STOREURI' ) +CALL SYSPROC.ADMIN_CMD ( 'REORG TABLE STOREURI' ) @ -- //@UNDO @@ -27,5 +27,5 @@ CALL SYSPROC.ADMIN_CMD ( 'REORG TABLE DB2ADMIN.STOREURI' ) ALTER TABLE STOREURI ALTER COLUMN USERNAME SET DATA TYPE VARCHAR(20) @ -CALL SYSPROC.ADMIN_CMD ( 'REORG TABLE DB2ADMIN.STOREURI' ) +CALL SYSPROC.ADMIN_CMD ( 'REORG TABLE STOREURI' ) @ diff --git a/server/user.jobengine.osgi.server/src/user/jobengine/server/IJobRuntime.java b/server/user.jobengine.osgi.server/src/user/jobengine/server/IJobRuntime.java index a7c660fc..c63033d8 100644 --- a/server/user.jobengine.osgi.server/src/user/jobengine/server/IJobRuntime.java +++ b/server/user.jobengine.osgi.server/src/user/jobengine/server/IJobRuntime.java @@ -86,6 +86,8 @@ public interface IJobRuntime extends IJob { boolean isService(); + boolean isTerminated(); + boolean isWaitFinish(); boolean isWaitingCancel(); diff --git a/server/user.jobengine.osgi.server/src/user/jobengine/server/JobEngine.java b/server/user.jobengine.osgi.server/src/user/jobengine/server/JobEngine.java index 716c7f5c..9b79734c 100644 --- a/server/user.jobengine.osgi.server/src/user/jobengine/server/JobEngine.java +++ b/server/user.jobengine.osgi.server/src/user/jobengine/server/JobEngine.java @@ -931,6 +931,7 @@ public class JobEngine implements IJobEngine { isRunning = true; TSMClient.SetUpMultithread(); } catch (Exception e) { + logger.error(e.getMessage()); logger.error(e); } logger.info("JobEngine started"); @@ -1026,7 +1027,7 @@ public class JobEngine implements IJobEngine { @Override public void suspendExecutingJob(Throwable t, IJobRuntime jobRuntime) { - String description = t.getMessage(); + String description = t.getClass().getSimpleName() + " : " + t.getMessage(); jobRuntime.setDescription(description); logger.error(description); //TODO itt miert FINISH a kovetkezo allapot, miert nem SUSPEND diff --git a/server/user.jobengine.osgi.server/src/user/jobengine/server/JobEngineConfiguration.java b/server/user.jobengine.osgi.server/src/user/jobengine/server/JobEngineConfiguration.java index de5335c8..1bcc931f 100644 --- a/server/user.jobengine.osgi.server/src/user/jobengine/server/JobEngineConfiguration.java +++ b/server/user.jobengine.osgi.server/src/user/jobengine/server/JobEngineConfiguration.java @@ -26,6 +26,7 @@ import com.ibm.nosql.json.api.BasicDBList; import com.ibm.nosql.json.api.BasicDBObject; import groovy.lang.GroovyClassLoader; +import groovy.lang.GroovyCodeSource; import user.commons.configuration.IConfiguration; import user.commons.nosql.NoSQLUtils; import user.jobengine.server.ast.Encoder; @@ -114,26 +115,32 @@ public class JobEngineConfiguration implements IJobEngineConfiguration { BasicDBObject jsParam = (BasicDBObject) jobParams.get(p); String n = jsParam.getString("name"); Object v = jsParam.get("value"); + + if (v == null) + continue; + if (v instanceof BasicDBList) { - ArrayList list = (BasicDBList) v; - sj.setJobParameter(n, list); - } else { - if (v == null) - continue; - - String sv = String.valueOf(v); - - if ("null".equals(v)) - continue; - - Class clazz = null; - try { - clazz = Class.forName(jsParam.getString("type")); - Object value = clazz.getConstructor(new Class[] { String.class }).newInstance(sv); - sj.setJobParameter(n, value); - } catch (Exception e) { - logger.error(e); - } + sj.setJobParameter(n, v); + continue; + } + + if (v instanceof BasicDBObject) { + sj.setJobParameter(n, v); + continue; + } + + String sv = String.valueOf(v); + + if ("null".equals(v)) + continue; + + Class clazz = null; + try { + clazz = Class.forName(jsParam.getString("type")); + Object value = clazz.getConstructor(new Class[] { String.class }).newInstance(sv); + sj.setJobParameter(n, value); + } catch (Exception e) { + logger.error(e); } } @@ -279,25 +286,34 @@ public class JobEngineConfiguration implements IJobEngineConfiguration { private Class parseClassHierarchy(GroovyClassLoader cl, String root, String className) throws IOException { Path path = Paths.get(root, className); - //logger.info("Loading class {} from {}", className, path.toString()); Class result = null; if (!path.toFile().exists()) return result; List lines = Files.readAllLines(path); for (String line : lines) { - if (line.contains("extends")) { + String trimmedLine = line.trim(); + + if (trimmedLine.startsWith("import")) { + String[] tokens = trimmedLine.split("\\."); + String name = tokens[tokens.length - 1]; + name = name.replace(";", ""); + logger.info("Try loading imported file {}", name); + parseClassHierarchy(cl, root, name + ".java"); + } - String[] tokens = line.split("extends"); + if (trimmedLine.contains("extends")) { + String[] tokens = trimmedLine.split("extends"); String parent = tokens[tokens.length - 1]; parent = parent.replace("{", "").trim(); //logger.info("Class {} needs parent class {}", className, parent); - + logger.info("Try loading parent file {}", parent); parseClassHierarchy(cl, root, parent + ".java"); break; } } - result = cl.parseClass(path.toFile()); + GroovyCodeSource x = new GroovyCodeSource(path.toFile()); + result = cl.parseClass(x, true); return result; } } diff --git a/server/user.jobengine.osgi.server/src/user/jobengine/server/JobRuntime.java b/server/user.jobengine.osgi.server/src/user/jobengine/server/JobRuntime.java index f05de1ec..bc7312e5 100644 --- a/server/user.jobengine.osgi.server/src/user/jobengine/server/JobRuntime.java +++ b/server/user.jobengine.osgi.server/src/user/jobengine/server/JobRuntime.java @@ -593,6 +593,11 @@ public class JobRuntime extends Job implements IJobRuntime { return isService; } + @Override + public boolean isTerminated() { + return getStatus() == JobStatus.SUSPENDED || getStatus() == JobStatus.FINISHED; + } + public boolean isTouchedByExecutor() { if (!getStatus().equals(JobStatus.WAIT_EXECUTOR)) return false; @@ -728,7 +733,7 @@ public class JobRuntime extends Job implements IJobRuntime { private Semaphore forkSempahore; private IJobChangedListener jobChangedListener; private List childrenIDs; - + * */ @Override diff --git a/server/user.jobengine.osgi.server/src/user/jobengine/server/scheduler/ScheduledJob.java b/server/user.jobengine.osgi.server/src/user/jobengine/server/scheduler/ScheduledJob.java index a202ebf2..8a1fcef0 100644 --- a/server/user.jobengine.osgi.server/src/user/jobengine/server/scheduler/ScheduledJob.java +++ b/server/user.jobengine.osgi.server/src/user/jobengine/server/scheduler/ScheduledJob.java @@ -35,7 +35,7 @@ public class ScheduledJob extends NativeJob { //implements InterruptableJob{ public IJobRuntime doJob() throws JobEngineException { if (jobEngine.isScheduledExecutionDisabled()) { - logger.warn("Scheduled execution is temporarly disabled, skipping execution of '{}'", jobName); + //logger.warn("Scheduled execution is temporarly disabled, skipping execution of '{}'", jobName); return null; } return jobEngine.submit(templateName, jobName, getJobParameters()); @@ -43,7 +43,7 @@ public class ScheduledJob extends NativeJob { //implements InterruptableJob{ public IJobRuntime doJob(IJobStatusChangedListener statusListener) throws JobEngineException { if (jobEngine.isScheduledExecutionDisabled()) { - logger.warn("Scheduled execution is temporarly disabled, skipping execution of '{}'", jobName); + //logger.warn("Scheduled execution is temporarly disabled, skipping execution of '{}'", jobName); return null; } return jobEngine.submit(null, statusListener, templateName, jobName, getJobParameters()); diff --git a/server/user.jobengine.osgi.services/META-INF/MANIFEST.MF b/server/user.jobengine.osgi.services/META-INF/MANIFEST.MF index f88665fc..ff67dc8e 100644 --- a/server/user.jobengine.osgi.services/META-INF/MANIFEST.MF +++ b/server/user.jobengine.osgi.services/META-INF/MANIFEST.MF @@ -4,7 +4,7 @@ Bundle-Name: Services Bundle-SymbolicName: user.jobengine.osgi.services;singleton:=true Bundle-Version: 1.0.0 Bundle-RequiredExecutionEnvironment: JavaSE-1.8 -Service-Component: OSGI-INF/componentBinder.xml, OSGI-INF/HaltCommandProvider.xml +Service-Component: OSGI-INF/*.xml Bundle-ActivationPolicy: lazy Web-ContextPath: /services Import-Package: com.fasterxml.jackson.annotation;version="2.4.5", diff --git a/server/user.jobengine.osgi.services/OSGI-INF/TestJobCommandProvider.xml b/server/user.jobengine.osgi.services/OSGI-INF/TestJobCommandProvider.xml new file mode 100644 index 00000000..6f962f35 --- /dev/null +++ b/server/user.jobengine.osgi.services/OSGI-INF/TestJobCommandProvider.xml @@ -0,0 +1,8 @@ + + + + + + + + \ No newline at end of file diff --git a/server/user.jobengine.osgi.services/src/user/jobengine/osgi/console/TestJobCommandProvider.java b/server/user.jobengine.osgi.services/src/user/jobengine/osgi/console/TestJobCommandProvider.java new file mode 100644 index 00000000..957bf7ee --- /dev/null +++ b/server/user.jobengine.osgi.services/src/user/jobengine/osgi/console/TestJobCommandProvider.java @@ -0,0 +1,75 @@ +package user.jobengine.osgi.console; + +import java.util.HashMap; +import java.util.Map; +import java.util.concurrent.CountDownLatch; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.eclipse.osgi.framework.console.CommandInterpreter; +import org.eclipse.osgi.framework.console.CommandProvider; + +import user.jobengine.server.IJobEngine; +import user.jobengine.server.IJobRuntime; +import user.jobengine.server.IJobStatusChangedListener; +import user.jobengine.server.JobStatusChangedEvent; + +public class TestJobCommandProvider implements CommandProvider { + private static final Logger logger = LogManager.getLogger(); + private IJobEngine jobEngine; + + public Object _test(CommandInterpreter intp) { + if (jobEngine == null) + return "nothing to do"; + + CountDownLatch finishLatch = new CountDownLatch(1); + + final Map parameters = new HashMap<>(); + parameters.put("param", 100); + StringBuilder result = new StringBuilder(); + try { + IJobRuntime runtime = jobEngine.submit("cancelable.xml", "Test", parameters); + runtime.addEventListener(new IJobStatusChangedListener() { + @Override + public void statusChanged(JobStatusChangedEvent event) { + IJobRuntime rt = (IJobRuntime) event.getSource(); + if (rt.isTerminated()) { + finishLatch.countDown(); + result.append("completed"); + return; + } + //result.append(String.format("%d %s : %d", rt.getId(), rt.getName(), rt.getProgress())); + } + }); + + finishLatch.await(); + } catch (Exception e1) { + logger.error(e1.getMessage()); + } + + return result.toString(); + } + + synchronized public void activate() { + logger.info("{} activated", getClass().getSimpleName()); + } + + synchronized public void bindJobEngine(IJobEngine jobEngine) { + this.jobEngine = jobEngine; + logger.info("JobEngine binded"); + } + + synchronized public void deactivate() { + logger.info("{} deactivated", getClass().getSimpleName()); + } + + @Override + public String getHelp() { + return "test - Execute Test job\n"; + } + + synchronized public void unbindJobEngine(IJobEngine jobEngine) { + logger.info("JobEngine unbinded"); + jobEngine = null; + } +} \ No newline at end of file diff --git a/server/user.mediacube.gui/img/outline_theaters_black_18dp.png b/server/user.mediacube.gui/img/outline_theaters_black_18dp.png new file mode 100644 index 0000000000000000000000000000000000000000..607e28c4b5a66c008e951cee3675591a5a424ab8 GIT binary patch literal 126 zcmeAS@N?(olHy`uVBq!ia0y~yU=RXf4i*LmhCj#M?l3SgxO%!chG?8mPFP@eAj`Gp%C7}>VUq>AHq3IFvMBI%N2k~# er5QyG46Ew18)s|FzhhuvVDNPHb6Mw<&;$U`wkZ<; literal 0 HcmV?d00001 diff --git a/server/user.mediacube.gui/pages/alternate_retrievebatchselector.zul b/server/user.mediacube.gui/pages/alternate_retrievebatchselector.zul index 04f29770..835960ec 100644 --- a/server/user.mediacube.gui/pages/alternate_retrievebatchselector.zul +++ b/server/user.mediacube.gui/pages/alternate_retrievebatchselector.zul @@ -1,5 +1,5 @@ - @@ -27,6 +27,7 @@ emptyMessage="Üres" selectedItems="@save(jlm.selectedItems)"> + @@ -34,6 +35,7 @@