From 39126a305164cfb98ff4cef1ccc74efa258ccc31 Mon Sep 17 00:00:00 2001 From: Omar Sweidan Date: Tue, 10 May 2022 12:51:47 +0200 Subject: [PATCH] =?utf8?q?OMAR=20mappa=20hozz=C3=A1ad=C3=A1sa=20git-hez?= MIME-Version: 1.0 Content-Type: text/plain; charset=utf8 Content-Transfer-Encoding: 8bit --- .../run-mediacube-server-omar.launch | 8 +- .../OMAR/configuration/etc/gosh_profile | 1 + .../production/OMAR/jobs/executors.xml | 43 ++ .../production/OMAR/jobs/schedules.json | 181 ++++++++ .../jobs/steps/AnalyzeMediaFilesStep.java | 28 ++ .../jobs/steps/ArchiveListBuilderStep.java | 167 +++++++ .../jobs/steps/ArchiveMaterialSubmitStep.java | 52 +++ .../OMAR/jobs/steps/ArchiveRecursive.java | 324 ++++++++++++++ .../jobs/steps/BatchRetrieveForkStep.java | 76 ++++ .../OMAR/jobs/steps/CalculateMD5Step.java | 34 ++ .../OMAR/jobs/steps/CancelableStep.java | 42 ++ .../steps/CleanupMountedLocationStep.java | 283 ++++++++++++ .../jobs/steps/CreateArchiveItemStep.java | 47 ++ .../jobs/steps/CreateMissingLowresStep.java | 95 ++++ .../OMAR/jobs/steps/DeleteFile.java | 22 + .../OMAR/jobs/steps/DummyTestStep1.java | 23 + .../OMAR/jobs/steps/DummyTestStep2.java | 20 + .../OMAR/jobs/steps/DummyTestStep3.java | 17 + .../OMAR/jobs/steps/FileCopyStep.java | 95 ++++ .../jobs/steps/GenerateJSONMetadataStep.java | 65 +++ .../OMAR/jobs/steps/HSMMigrateStep.java | 401 +++++++++++++++++ .../OMAR/jobs/steps/IntegrationTestStep.java | 128 ++++++ .../OMAR/jobs/steps/MXFCutterStep.java | 129 ++++++ .../OMAR/jobs/steps/MediaToolStep.java | 29 ++ .../jobs/steps/MetadataTransformStep.java | 168 +++++++ .../OMAR/jobs/steps/MetadataUpdater.java | 297 +++++++++++++ .../OMAR/jobs/steps/MoveJpegToIsilonStep.java | 165 +++++++ .../steps/OutputPathAndNameSelectorStep.java | 154 +++++++ .../jobs/steps/PrepareMediaRestoreStep.java | 37 ++ .../steps/PrepareRemoteTranscodeStep.java | 48 ++ .../steps/QueryMissingProxyMediaStep.java | 74 ++++ .../OMAR/jobs/steps/RemoteJobStep.java | 36 ++ .../jobs/steps/SafeDeleteRecursiveStep.java | 416 ++++++++++++++++++ .../OMAR/jobs/steps/SaveMediaProxy.java | 22 + .../OMAR/jobs/steps/TSMBackupStep.java | 239 ++++++++++ .../jobs/steps/TSMExtendedRetrieveStep.java | 95 ++++ .../OMAR/jobs/steps/TSMRestoreStep.java | 187 ++++++++ .../OMAR/jobs/steps/TSMSimpleRestoreStep.java | 57 +++ .../jobs/steps/TestForkCancelableStep.java | 35 ++ .../jobs/steps/TranscodeFFAStranStep.java | 151 +++++++ .../OMAR/jobs/steps/TransferStep.java | 110 +++++ .../jobs/steps/UpdateGhostMediaDataStep.java | 89 ++++ .../OMAR/jobs/steps/ValidateProResStep.java | 32 ++ .../OMAR/jobs/steps/shared/EscortFiles.java | 348 +++++++++++++++ .../jobs/steps/shared/ExternalCommand.java | 79 ++++ .../steps/shared/ExternalCommandExecutor.java | 32 ++ .../jobs/steps/shared/ExternalProfile.java | 33 ++ .../steps/shared/ExternalProfilesConfig.java | 15 + .../steps/shared/FileSearchFilterOptions.java | 41 ++ .../jobs/steps/shared/IExternalCallback.java | 5 + .../steps/shared/ItemManagerExtensions.java | 94 ++++ .../jobs/steps/shared/MediaCubeClient.java | 77 ++++ .../shared/MediaFileSearchFilterOptions.java | 42 ++ .../OMAR/jobs/steps/shared/MetadataType.java | 5 + .../steps/shared/MetadataTypeDetector.java | 49 +++ .../OMAR/jobs/steps/shared/TestLib.java | 9 + .../OMAR/jobs/steps/shared/TestLib1.java | 8 + .../OMAR/jobs/templates/archive-limited.xml | 43 ++ .../OMAR/jobs/templates/archive-material.xml | 49 +++ .../OMAR/jobs/templates/archive-ondemand.xml | 43 ++ .../OMAR/jobs/templates/archive-recursive.xml | 25 ++ .../templates/batch-retrieve-ondemand.xml | 29 ++ .../OMAR/jobs/templates/calculatemd5.xml | 19 + .../OMAR/jobs/templates/cancelable.xml | 17 + .../OMAR/jobs/templates/common-copy.xml | 41 ++ .../jobs/templates/create-lowres-ondemand.xml | 106 +++++ .../jobs/templates/create-proxy-ffmpeg.xml | 25 ++ .../OMAR/jobs/templates/delete-materials.xml | 17 + .../jobs/templates/dummy-test-job-copy.xml | 47 ++ .../OMAR/jobs/templates/dummy-test-job.xml | 47 ++ .../OMAR/jobs/templates/integration-test.xml | 6 + .../OMAR/jobs/templates/metadata-updater.xml | 6 + .../OMAR/jobs/templates/migrate-hsm.xml | 21 + .../jobs/templates/move-jpeg-to-isilon.xml | 51 +++ .../OMAR/jobs/templates/prores-archive.xml | 76 ++++ .../OMAR/jobs/templates/remote-transcode.xml | 111 +++++ .../OMAR/jobs/templates/retrieve-ondemand.xml | 150 +++++++ .../OMAR/jobs/templates/retrieve.xml | 21 + .../jobs/templates/safe-delete-recursive.xml | 17 + .../OMAR/jobs/templates/sync-subtitles.xml | 23 + .../jobs/templates/sys-recreate-lowres.xml | 118 +++++ .../jobs/templates/test-fork-cancelable.xml | 6 + .../-product/production/OMAR/log/.gitignore | 5 + .../production/OMAR/log/2022-05/.gitignore | 6 + .../markered-mediacube-05-09-2022-1.log.gz | Bin 0 -> 600 bytes .../log/2022-05/mediacube-05-09-2022-1.log.gz | Bin 0 -> 58862 bytes .../2022-05/mediacube-err-05-09-2022-1.log.gz | Bin 0 -> 308 bytes server/-product/production/OMAR/mediacube.bat | 17 + .../production/OMAR/settings/application.yaml | 45 ++ .../-product/production/OMAR/settings/dsm.opt | 1 + .../production/OMAR/settings/dsmopt.lock | 0 .../OMAR/settings/external-commands.yaml | 27 ++ .../production/OMAR/settings/jetty.xml | 45 ++ .../production/OMAR/settings/log4j2.xml | 105 +++++ .../production/OMAR/settings/maestro.yaml | 38 ++ .../production/OMAR/settings/mediacube.yaml | 53 +++ 96 files changed, 7011 insertions(+), 4 deletions(-) create mode 100644 server/-product/production/OMAR/configuration/etc/gosh_profile create mode 100644 server/-product/production/OMAR/jobs/executors.xml create mode 100644 server/-product/production/OMAR/jobs/schedules.json create mode 100644 server/-product/production/OMAR/jobs/steps/AnalyzeMediaFilesStep.java create mode 100644 server/-product/production/OMAR/jobs/steps/ArchiveListBuilderStep.java create mode 100644 server/-product/production/OMAR/jobs/steps/ArchiveMaterialSubmitStep.java create mode 100644 server/-product/production/OMAR/jobs/steps/ArchiveRecursive.java create mode 100644 server/-product/production/OMAR/jobs/steps/BatchRetrieveForkStep.java create mode 100644 server/-product/production/OMAR/jobs/steps/CalculateMD5Step.java create mode 100644 server/-product/production/OMAR/jobs/steps/CancelableStep.java create mode 100644 server/-product/production/OMAR/jobs/steps/CleanupMountedLocationStep.java create mode 100644 server/-product/production/OMAR/jobs/steps/CreateArchiveItemStep.java create mode 100644 server/-product/production/OMAR/jobs/steps/CreateMissingLowresStep.java create mode 100644 server/-product/production/OMAR/jobs/steps/DeleteFile.java create mode 100644 server/-product/production/OMAR/jobs/steps/DummyTestStep1.java create mode 100644 server/-product/production/OMAR/jobs/steps/DummyTestStep2.java create mode 100644 server/-product/production/OMAR/jobs/steps/DummyTestStep3.java create mode 100644 server/-product/production/OMAR/jobs/steps/FileCopyStep.java create mode 100644 server/-product/production/OMAR/jobs/steps/GenerateJSONMetadataStep.java create mode 100644 server/-product/production/OMAR/jobs/steps/HSMMigrateStep.java create mode 100644 server/-product/production/OMAR/jobs/steps/IntegrationTestStep.java create mode 100644 server/-product/production/OMAR/jobs/steps/MXFCutterStep.java create mode 100644 server/-product/production/OMAR/jobs/steps/MediaToolStep.java create mode 100644 server/-product/production/OMAR/jobs/steps/MetadataTransformStep.java create mode 100644 server/-product/production/OMAR/jobs/steps/MetadataUpdater.java create mode 100644 server/-product/production/OMAR/jobs/steps/MoveJpegToIsilonStep.java create mode 100644 server/-product/production/OMAR/jobs/steps/OutputPathAndNameSelectorStep.java create mode 100644 server/-product/production/OMAR/jobs/steps/PrepareMediaRestoreStep.java create mode 100644 server/-product/production/OMAR/jobs/steps/PrepareRemoteTranscodeStep.java create mode 100644 server/-product/production/OMAR/jobs/steps/QueryMissingProxyMediaStep.java create mode 100644 server/-product/production/OMAR/jobs/steps/RemoteJobStep.java create mode 100644 server/-product/production/OMAR/jobs/steps/SafeDeleteRecursiveStep.java create mode 100644 server/-product/production/OMAR/jobs/steps/SaveMediaProxy.java create mode 100644 server/-product/production/OMAR/jobs/steps/TSMBackupStep.java create mode 100644 server/-product/production/OMAR/jobs/steps/TSMExtendedRetrieveStep.java create mode 100644 server/-product/production/OMAR/jobs/steps/TSMRestoreStep.java create mode 100644 server/-product/production/OMAR/jobs/steps/TSMSimpleRestoreStep.java create mode 100644 server/-product/production/OMAR/jobs/steps/TestForkCancelableStep.java create mode 100644 server/-product/production/OMAR/jobs/steps/TranscodeFFAStranStep.java create mode 100644 server/-product/production/OMAR/jobs/steps/TransferStep.java create mode 100644 server/-product/production/OMAR/jobs/steps/UpdateGhostMediaDataStep.java create mode 100644 server/-product/production/OMAR/jobs/steps/ValidateProResStep.java create mode 100644 server/-product/production/OMAR/jobs/steps/shared/EscortFiles.java create mode 100644 server/-product/production/OMAR/jobs/steps/shared/ExternalCommand.java create mode 100644 server/-product/production/OMAR/jobs/steps/shared/ExternalCommandExecutor.java create mode 100644 server/-product/production/OMAR/jobs/steps/shared/ExternalProfile.java create mode 100644 server/-product/production/OMAR/jobs/steps/shared/ExternalProfilesConfig.java create mode 100644 server/-product/production/OMAR/jobs/steps/shared/FileSearchFilterOptions.java create mode 100644 server/-product/production/OMAR/jobs/steps/shared/IExternalCallback.java create mode 100644 server/-product/production/OMAR/jobs/steps/shared/ItemManagerExtensions.java create mode 100644 server/-product/production/OMAR/jobs/steps/shared/MediaCubeClient.java create mode 100644 server/-product/production/OMAR/jobs/steps/shared/MediaFileSearchFilterOptions.java create mode 100644 server/-product/production/OMAR/jobs/steps/shared/MetadataType.java create mode 100644 server/-product/production/OMAR/jobs/steps/shared/MetadataTypeDetector.java create mode 100644 server/-product/production/OMAR/jobs/steps/shared/TestLib.java create mode 100644 server/-product/production/OMAR/jobs/steps/shared/TestLib1.java create mode 100644 server/-product/production/OMAR/jobs/templates/archive-limited.xml create mode 100644 server/-product/production/OMAR/jobs/templates/archive-material.xml create mode 100644 server/-product/production/OMAR/jobs/templates/archive-ondemand.xml create mode 100644 server/-product/production/OMAR/jobs/templates/archive-recursive.xml create mode 100644 server/-product/production/OMAR/jobs/templates/batch-retrieve-ondemand.xml create mode 100644 server/-product/production/OMAR/jobs/templates/calculatemd5.xml create mode 100644 server/-product/production/OMAR/jobs/templates/cancelable.xml create mode 100644 server/-product/production/OMAR/jobs/templates/common-copy.xml create mode 100644 server/-product/production/OMAR/jobs/templates/create-lowres-ondemand.xml create mode 100644 server/-product/production/OMAR/jobs/templates/create-proxy-ffmpeg.xml create mode 100644 server/-product/production/OMAR/jobs/templates/delete-materials.xml create mode 100644 server/-product/production/OMAR/jobs/templates/dummy-test-job-copy.xml create mode 100644 server/-product/production/OMAR/jobs/templates/dummy-test-job.xml create mode 100644 server/-product/production/OMAR/jobs/templates/integration-test.xml create mode 100644 server/-product/production/OMAR/jobs/templates/metadata-updater.xml create mode 100644 server/-product/production/OMAR/jobs/templates/migrate-hsm.xml create mode 100644 server/-product/production/OMAR/jobs/templates/move-jpeg-to-isilon.xml create mode 100644 server/-product/production/OMAR/jobs/templates/prores-archive.xml create mode 100644 server/-product/production/OMAR/jobs/templates/remote-transcode.xml create mode 100644 server/-product/production/OMAR/jobs/templates/retrieve-ondemand.xml create mode 100644 server/-product/production/OMAR/jobs/templates/retrieve.xml create mode 100644 server/-product/production/OMAR/jobs/templates/safe-delete-recursive.xml create mode 100644 server/-product/production/OMAR/jobs/templates/sync-subtitles.xml create mode 100644 server/-product/production/OMAR/jobs/templates/sys-recreate-lowres.xml create mode 100644 server/-product/production/OMAR/jobs/templates/test-fork-cancelable.xml create mode 100644 server/-product/production/OMAR/log/.gitignore create mode 100644 server/-product/production/OMAR/log/2022-05/.gitignore create mode 100644 server/-product/production/OMAR/log/2022-05/markered-mediacube-05-09-2022-1.log.gz create mode 100644 server/-product/production/OMAR/log/2022-05/mediacube-05-09-2022-1.log.gz create mode 100644 server/-product/production/OMAR/log/2022-05/mediacube-err-05-09-2022-1.log.gz create mode 100644 server/-product/production/OMAR/mediacube.bat create mode 100644 server/-product/production/OMAR/settings/application.yaml create mode 100644 server/-product/production/OMAR/settings/dsm.opt create mode 100644 server/-product/production/OMAR/settings/dsmopt.lock create mode 100644 server/-product/production/OMAR/settings/external-commands.yaml create mode 100644 server/-product/production/OMAR/settings/jetty.xml create mode 100644 server/-product/production/OMAR/settings/log4j2.xml create mode 100644 server/-product/production/OMAR/settings/maestro.yaml create mode 100644 server/-product/production/OMAR/settings/mediacube.yaml diff --git a/server/-configuration/run-mediacube-server-omar.launch b/server/-configuration/run-mediacube-server-omar.launch index 3d55a3c3..7e9aae74 100644 --- a/server/-configuration/run-mediacube-server-omar.launch +++ b/server/-configuration/run-mediacube-server-omar.launch @@ -6,17 +6,17 @@ - + - + - + @@ -31,7 +31,7 @@ - + diff --git a/server/-product/production/OMAR/configuration/etc/gosh_profile b/server/-product/production/OMAR/configuration/etc/gosh_profile new file mode 100644 index 00000000..1c659f49 --- /dev/null +++ b/server/-product/production/OMAR/configuration/etc/gosh_profile @@ -0,0 +1 @@ +prompt=mc> \ No newline at end of file diff --git a/server/-product/production/OMAR/jobs/executors.xml b/server/-product/production/OMAR/jobs/executors.xml new file mode 100644 index 00000000..0fd7a2f2 --- /dev/null +++ b/server/-product/production/OMAR/jobs/executors.xml @@ -0,0 +1,43 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/server/-product/production/OMAR/jobs/schedules.json b/server/-product/production/OMAR/jobs/schedules.json new file mode 100644 index 00000000..0134aa6b --- /dev/null +++ b/server/-product/production/OMAR/jobs/schedules.json @@ -0,0 +1,181 @@ +{ +"joblist":[ + + { + "name":"n1", + "template":"cancelable.xml", + "active":true, + "executeimmediate":true, + "parameters":[{"name":"param","value":1000,"type":"java.lang.Integer"}] + }, + + { + "name":"n2", + "template":"cancelable.xml", + "active":true, + "executeimmediate":true, + "parameters":[{"name":"param","value":1000,"type":"java.lang.Integer"}] + }, + + { + "name":"Limitált archiválás", + "template":"archive-limited.xml", + "cronexpression":"0 */10 * * * ?", + "parameters":[ + + { + "name":"sourcePath", + "value":"/opt/mediacube/ARCHIVE", + "type":"java.lang.String" + }, + + { + "name":"globalSourcePath", + "value":"\\\\10.11.1.90\\data\\ARCHIVE", + "type":"java.lang.String" + }, + + { + "name":"transcoderTargetPath", + "value":"/mnt/PROMISE/TRANSCODER/FFASTRANSCODER/Out", + "type":"java.lang.String" + }, + + { + "name":"killDateDays", + "value":-1, + "type":"java.lang.Integer" + }, + + { + "name":"limit", + "value":1000, + "type":"java.lang.Integer" + } + ] + }, + + { + "name":"SYS: batch-retrieve-ondemand", + "template":"batch-retrieve-ondemand.xml" + }, + + { + "template":"calculatemd5.xml", + "name":"MD5 kiszámítása", + "parameters":[ + + { + "name":"fileName", + "value":"C:\\Users\\machine\\Downloads\\sample.mxf", + "type":"java.lang.String" + } + ] + }, + + { + "template":"dummy-test-job.xml", + "name":"dummy test job", + "parameters":[{"name":"param1","value":"Jozsi","type":"java.lang.String"}], + "cronexpression":"0/3 * * * * ?" + }, + + { + "template":"move-jpeg-to-isilon.xml", + "name":"JPEG másolása ISILON-ba", + "parameters":[ + + { + "name":"sourceUri", + "value":"localhost", + "type":"java.lang.String" + }, + + { + "name":"sourceProtocol", + "value":"LOCAL", + "type":"java.lang.String" + }, + + { + "name":"sourceFolder", + "value":"c:\\data\\video", + "type":"java.lang.String" + }, + + { + "name":"targetUri", + "value":"localhost", + "type":"java.lang.String" + }, + + { + "name":"targetProtocol", + "value":"LOCAL", + "type":"java.lang.String" + }, + + { + "name":"targetFolder", + "value":"c:\\data\\video2", + "type":"java.lang.String" + }, + + { + "name":"userName", + "value":"dani", + "type":"java.lang.String" + }, + + { + "name":"password", + "value":"dani", + "type":"java.lang.String" + }, + + { + "name":"port", + "value":"21", + "type":"java.lang.Integer" + } + ] + }, + + { + "template":"prores-archive.xml", + "name":"ProRes archiválás", + "parameters":[ + + { + "name":"sourceStoreFolder", + "value":"c:\\data", + "type":"java.lang.String" + }, + + { + "name":"sourceStoreName", + "value":"AMC_LOCAL", + "type":"java.lang.String" + }, + + { + "name":"filter", + "value":{"fileName":".*\\.(mov)$"}, + "type":"java.lang.String" + }, + + { + "name":"targetStoreName", + "value":"AMC_LOCAL", + "type":"java.lang.String" + }, + + { + "name":"escortStoreFolder", + "value":"AMC_LOCAL", + "type":"java.lang.String" + } + ] + } + ] +} \ No newline at end of file diff --git a/server/-product/production/OMAR/jobs/steps/AnalyzeMediaFilesStep.java b/server/-product/production/OMAR/jobs/steps/AnalyzeMediaFilesStep.java new file mode 100644 index 00000000..98cf7fa9 --- /dev/null +++ b/server/-product/production/OMAR/jobs/steps/AnalyzeMediaFilesStep.java @@ -0,0 +1,28 @@ +package user.jobengine.server.steps; + +import java.nio.file.Paths; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + +import user.commons.mediaarea.MediaArea; + +public class AnalyzeMediaFilesStep extends JobStep { + private static final Logger logger = LogManager.getLogger(); + + @StepEntry + public Object[] execute(List foundFiles) { + Map proResFiles = new HashMap(foundFiles.size()); + + for (int i = 0; i < foundFiles.size(); i++) { + MediaArea mediaArea = new MediaArea(Paths.get(foundFiles.get(i))); + mediaArea.process(); + proResFiles.put(Paths.get(foundFiles.get(i)).toAbsolutePath().toString(), mediaArea); + } + + return new Object[] { proResFiles }; + } +} diff --git a/server/-product/production/OMAR/jobs/steps/ArchiveListBuilderStep.java b/server/-product/production/OMAR/jobs/steps/ArchiveListBuilderStep.java new file mode 100644 index 00000000..b2321829 --- /dev/null +++ b/server/-product/production/OMAR/jobs/steps/ArchiveListBuilderStep.java @@ -0,0 +1,167 @@ +package user.jobengine.server.steps; + +import java.io.File; +import java.io.IOException; +import java.nio.file.DirectoryStream; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.LinkedList; +import java.util.List; + +import org.apache.commons.lang.StringUtils; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.apache.logging.log4j.Marker; + +import user.jobengine.server.IJobEngine; +import user.jobengine.server.IJobRuntime; + +/** + * Az archivalhato mediak listazasa MediaFileWrapper objektumokban. A listazott media allomanyokat megjeloli .catched signal allomannyal, hogy a legkozelebbi + * listazas figyelmen kivul hagyja. + * + * @author robi + */ +public class ArchiveListBuilderStep extends JobStep { + private static final Logger logger = LogManager.getLogger(); + // private static final String UTF8 = "utf-8"; + private static final String STATUSFOLDER = ".STATUS"; + private static final String JSONEXT = ".json"; + private static final String CATCHEDEXT = ".catched"; + + public static final String ITEM_TITLE = "itemTitle"; + public static final String ITEM_HOUSEID = "itemHouseId"; + public static final String ITEM_DESCRIPTION = "itemDescription"; + public static final String MEDIA_HOUSEID = "mediaHouseId"; + public static final String MEDIA_TITLE = "mediaTitle"; + public static final String MEDIA_DESCRIPTION = "mediaDescription"; + public static final String MEDIA_TYPE = "mediaType"; + private static final String DURATION = "duration"; + private static final String EXISTING_MEDIAID = "existingMediaId"; + private static final String TAGS = "tags"; + + private Marker marker; + + private ArchiveItem createArchiveItem(Path jsonFilePath, Path mediaFilePath, Path catchedFilePath) { + ArchiveItem result = null; + try { + result = ArchiveItem.fromFile(jsonFilePath); + result.setMediaFile(mediaFilePath.toString()); + result.setCatchedFile(catchedFilePath.toString()); + } catch (Exception e) { + logger.catching(e); + } + + return result; + } + + private void createCatchedFile(Path catchedFilePath) { + try { + Files.createFile(catchedFilePath); + //Files.write(catchedFilePath, CATCHED.getBytes(UTF8), StandardOpenOption.CREATE); + } catch (Exception e) { + logger.catching(e); + } + } + + @StepEntry + public Object[] execute(String sourcePath, int limit, IJobEngine jobEngine, IJobRuntime jobRuntime) { + marker = jobRuntime.getSessionMarker(); + List archiveList = new LinkedList(); + DirectoryStream directoryStream = null; + try { + DirectoryStream stream = Files.newDirectoryStream(Paths.get(sourcePath)); + for (Path p : stream) { + processPathItem(p, archiveList); + } + } catch (Exception e) { + logger.catching(e); + logger.error(marker, "Az '{}' mappa elérése sikertelen. A rendszer hibaüzenete: {}", e.getMessage()); + } finally { + if (directoryStream != null) { + try { + directoryStream.close(); + } catch (IOException e) { + } + } + } + + if (limit > 0 && archiveList.size() > limit) { + archiveList = archiveList.subList(0, limit); + logger.info(marker, "A folyamat alkalmazza a beállított {} limitet.", limit); + } + + if (archiveList.size() == 0) + logger.info(marker, "Nincs archiválandó anyag."); + else + logger.info(marker, "Az archiváló folyamat {} új anyagot érzékelt.", archiveList == null ? 0 : archiveList.size()); + + for (ArchiveItem archiveItem : archiveList) { + createCatchedFile(Paths.get(archiveItem.getCatchedFile())); + } + + return new Object[] { archiveList }; + } + + private boolean processPathItem(Path mediaFilePath, final List archiveList) { + File mediaFile = mediaFilePath.toFile(); + + // if (mediaFile.length() > 0) + // return false; + + if (mediaFile.isDirectory()) { + return false; + } + + Path dotStorePath = Paths.get(mediaFilePath.getParent().toString(), STATUSFOLDER); + Path catchedFilePath = Paths.get(dotStorePath.toString(), mediaFile.getName() + CATCHEDEXT); + File catchedFile = catchedFilePath.toFile(); + if (catchedFile.exists()) { + logger.warn("{} file is already catched.", mediaFile.getName()); + return false; + } + + Path jsonFilePath = Paths.get(dotStorePath.toString(), mediaFile.getName() + JSONEXT); + File jsonFile = jsonFilePath.toFile(); + if (!jsonFile.exists()) { + logger.warn("{} has no json metadata.", mediaFile.getName()); + return false; + } + + ArchiveItem archiveItem = createArchiveItem(jsonFilePath, mediaFilePath, catchedFilePath); + + if (archiveItem == null) { + logger.warn("{} has no metadata specified.", mediaFile.getName()); + return false; + } + + if (StringUtils.isBlank(archiveItem.getItemHouseId())) { + logger.warn("{} has no Item HouseID specified in metadata.", mediaFile.getName()); + return false; + } + + if (StringUtils.isBlank(archiveItem.getItemTitle())) { + logger.warn("{} has no Item Title specified in metadata.", mediaFile.getName()); + return false; + } + + if (StringUtils.isBlank(archiveItem.getMediaHouseId())) { + logger.warn("{} has no Media HouseID specified in metadata.", mediaFile.getName()); + return false; + } + + if (StringUtils.isBlank(archiveItem.getMediaTitle())) { + logger.warn("{} has no Media Title specified in metadata.", mediaFile.getName()); + return false; + } + + //A tenyleges archivalast vesszuk elore + if (mediaFile.length() == 0) + archiveList.add(archiveItem); + else + archiveList.add(0, archiveItem); + //createCatchedFile(catchedFilePath); + return true; + } +} diff --git a/server/-product/production/OMAR/jobs/steps/ArchiveMaterialSubmitStep.java b/server/-product/production/OMAR/jobs/steps/ArchiveMaterialSubmitStep.java new file mode 100644 index 00000000..0aedbbe8 --- /dev/null +++ b/server/-product/production/OMAR/jobs/steps/ArchiveMaterialSubmitStep.java @@ -0,0 +1,52 @@ +package user.jobengine.server.steps; + +import java.io.File; +import java.util.List; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.apache.logging.log4j.Marker; + +import user.commons.ListUtils; +import user.jobengine.server.IJobEngine; +import user.jobengine.server.IJobRuntime; + +public class ArchiveMaterialSubmitStep extends JobStep { + private static final Logger logger = LogManager.getLogger(); + private static final String JOBTEMPLATE = "archive-material.xml"; + private static final String KILL_DATE_DAYS = "killDateDays"; + private static final String ARCHIVE = "Archiválás"; + private static final String ARCHIVE_ITEM = "archiveItem"; + private Marker marker; + + @StepEntry + public Object[] execute(List archiveList, int killDateDays, IJobEngine jobEngine, IJobRuntime jobRuntime) throws Exception { + marker = jobRuntime.getSessionMarker(); + if (archiveList == null || archiveList.size() == 0) + return null; + + if (jobRuntime.forkPrepare()) { + for (int i = 0; i < archiveList.size(); i++) { + ArchiveItem archiveItem = archiveList.get(i); + try { + IJobRuntime runtime = jobEngine.submit(jobRuntime, null, JOBTEMPLATE, ARCHIVE, + ListUtils.asMap(ARCHIVE_ITEM, archiveItem, KILL_DATE_DAYS, killDateDays)); + int progress = (i + 1) * 100 / archiveList.size(); + setProgress(progress); + //TODO kivezetni a submit hibaüzenetet + if (runtime == null) + throw new Exception("Submit returned null runtime"); + } catch (Exception e) { + logger.catching(e); + String fileName = new File(archiveItem.getMediaFile()).getName(); + logger.error(marker, "A(z) '{}' állomány archiválási kísérlete sikertelen. A rendszer üzenete: {}", fileName, e.getMessage()); + if (!archiveItem.removeCatchedFile()) + logger.error(marker, "A(z) '{}' állomány .catched jelző állománya nem törölhető.", fileName); + throw e; + } + } + } + jobRuntime.forkWaitComplete(); + return null; + } +} diff --git a/server/-product/production/OMAR/jobs/steps/ArchiveRecursive.java b/server/-product/production/OMAR/jobs/steps/ArchiveRecursive.java new file mode 100644 index 00000000..3f01f32b --- /dev/null +++ b/server/-product/production/OMAR/jobs/steps/ArchiveRecursive.java @@ -0,0 +1,324 @@ +package user.jobengine.server.steps; + +import java.io.File; +import java.io.IOException; +import java.nio.file.FileVisitResult; +import java.nio.file.FileVisitor; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.nio.file.attribute.BasicFileAttributes; +import java.text.SimpleDateFormat; +import java.util.Arrays; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; + +import org.apache.commons.io.FilenameUtils; +import org.apache.commons.lang.StringUtils; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.apache.logging.log4j.message.Message; +import org.apache.logging.log4j.message.ParameterizedMessage; + +import com.ibm.nosql.json.api.BasicDBObject; + +import user.commons.JobStatus; +import user.commons.ListUtils; +import user.commons.log4j2.marker.MediaCubeMarker; +import user.commons.mediatool.MediaInfo; +import user.jobengine.server.IJobEngine; +import user.jobengine.server.IJobRuntime; +import user.jobengine.server.steps.shared.EscortFiles; +import user.mediacube.metadata.interfaces.IMetadata; +import user.mediacube.metadata.interfaces.IMetadataProvider; +import user.mediacube.metadata.interfaces.IMetadataProviderFactory; +import user.mediacube.metadata.interfaces.MetadataProviderType; +import user.mediacube.metadata.interfaces.MetadataType; +import user.mediacube.metadata.interfaces.PlanAirMetadataListOptions; + +public class ArchiveRecursive extends JobStep implements FileVisitor { + private static final Logger logger = LogManager.getLogger(); + private static final String JOBTEMPLATE = "archive-material.xml"; + private static final String ITEM_TITLE = "itemTitle"; + private static final String ITEM_HOUSEID = "itemHouseId"; + private static final String MEDIA_HOUSEID = "mediaHouseId"; + private static final String MEDIA_TITLE = "mediaTitle"; + private static final String MEDIA_DESCRIPTION = "mediaDescription"; + private static final String MEDIA_TYPE = "mediaType"; + private static final String ARCHIVE = "Archiválás"; + private static final String ARCHIVE_ITEM = "archiveItem"; + private static final String KILL_DATE_DAYS = "killDateDays"; + private SimpleDateFormat df = new SimpleDateFormat("yyyy.MM.dd"); + + private List skipPathNames = Arrays.asList("!ARCHIVALAS_ALATT", EscortFiles.STATUSFOLDER, + EscortFiles.CONFLICTFOLDER); + private int limit; + private int submitted; + private int killDateDays; + private boolean disableProxy; + + private boolean canReadMediaInfo(Path mediaFilePath) { + boolean result = false; + try { + MediaInfo mi = new MediaInfo(mediaFilePath); + mi.process(); + result = true; + } catch (Exception e) { + logger.warn(getSessionMarker(), e.getMessage()); + } + return result; + } + + private void checkArchiveItem(ArchiveItem archiveItem) throws Exception { + if (archiveItem == null) + throw new Exception("No metadata specified."); + + if (StringUtils.isBlank(archiveItem.getItemHouseId())) + throw new Exception("No Item HouseID specified in metadata."); + + if (StringUtils.isBlank(archiveItem.getItemTitle())) + throw new Exception("No Item Title specified in metadata."); + + if (StringUtils.isBlank(archiveItem.getMediaHouseId())) + throw new Exception("No Media HouseID specified in metadata."); + + if (StringUtils.isBlank(archiveItem.getMediaTitle())) + throw new Exception("No Media Title specified in metadata."); + } + + private String archiveItemJSON(ArchiveItem result) { + BasicDBObject obj = new BasicDBObject(); + obj.put("itemHouseId", result.getItemHouseId()); + obj.put("itemTitle", result.getItemTitle()); + obj.put("mediaHouseId", result.getMediaHouseId()); + obj.put("mediaTitle", result.getMediaTitle()); + obj.put("mediaDescription", result.getMediaDescription()); + obj.put("mediaType", result.getMediaType()); + return obj.toPrettyString(""); + } + + private ArchiveItem createArchiveItem(Path filePath) throws Exception { + ArchiveItem result = null; + + String fileName = filePath.getFileName().toString(); + String mediaHouseId = FilenameUtils.removeExtension(fileName); + try { + result = getPlanAirMetadata(mediaHouseId); + + if (result != null) { + result.setMediaFile(filePath.toString()); + // 210617 proxy keszites tiltasa + result.setDisableProxy(disableProxy); + } + + logger.info(getSessionMarker(), "PlanAir query done for {}", filePath); + } catch (Exception e) { + logger.error(getSessionMarker(), "PlanAir metadata error", e); + // nem latja a drivert pl. + throw e; + } + +//210616 Ha nincs metaadat, nem archivalunk + if (result == null) { + result = new ArchiveItem(); + BasicFileAttributes attr = Files.readAttributes(filePath, BasicFileAttributes.class); + result.setItemHouseId(df.format(attr.lastModifiedTime().toMillis())); + result.setItemTitle(filePath.getParent().toString()); + result.setMediaHouseId(mediaHouseId); + result.setMediaTitle(fileName); + result.setMediaDescription("/ARCHIVE-TEST"); + result.setMediaType("Generic"); + result.setMediaFile(filePath.toString()); + } + + return result; + } + + @StepEntry + public Object[] execute(String sourcePath, int killDateDays, int limit, boolean disableProxy) throws Exception { + this.killDateDays = killDateDays; + this.limit = limit; + this.disableProxy = disableProxy; + logger.info(getSessionMarker(), "Starting in {}", sourcePath); + try { + if (getJobRuntime().forkPrepare()) { + Files.walkFileTree(Paths.get(sourcePath), this); + } + } catch (Exception e) { + logger.error(getSessionMarker(), "Az '{}' mappa elérése sikertelen. A rendszer hibaüzenete: {}", sourcePath, + e.getMessage()); + } finally { + if (submitted > 0) + getJobRuntime().forkWaitComplete(); + else + getJobRuntime().cancelForkPrepare(); + } + return null; + } + + private ArchiveItem getPlanAirMetadata(String mediaHouseId) throws Exception { + PlanAirMetadataListOptions opt = new PlanAirMetadataListOptions(); + opt.setSearch(mediaHouseId); + opt.setType(MetadataType.Material); + + BasicDBObject json = null; + List data = null; + + IMetadataProviderFactory factory = getService(IMetadataProviderFactory.class); + if (factory == null) + logger.info(getSessionMarker(), "IMetadataProviderFactory is null"); + + IMetadataProvider planairProvider = factory.getProvider(MetadataProviderType.PLANAIR); + if (planairProvider == null) + logger.info(getSessionMarker(), "IMetadataProvider is null"); + + ArchiveItem result = null; + data = planairProvider.list(opt); + if (data.size() != 0) + json = data.get(0).asJSON(); + else { + opt.setType(MetadataType.Promo); + data = planairProvider.list(opt); + if (data.size() != 0) + json = data.get(0).asJSON(); + else { + opt.setType(MetadataType.AD); + data = planairProvider.list(opt); + if (data.size() != 0) + json = data.get(0).asJSON(); + } + } + if (json != null) { + result = new ArchiveItem(); + result.setItemHouseId(json.getString(ITEM_HOUSEID)); + result.setItemTitle(json.getString(ITEM_TITLE)); + result.setMediaHouseId(json.getString(MEDIA_HOUSEID)); + result.setMediaTitle(json.getString(MEDIA_TITLE)); + result.setMediaDescription(json.getString(MEDIA_DESCRIPTION)); + result.setMediaType(json.getString(MEDIA_TYPE)); + } + return result; + } + + private boolean handleArchiveConflict(Path mediaPath) throws Exception { + boolean result = false; + String sourceFileName = mediaPath.getFileName().toString(); + if (getManager().isMediaFileExists(sourceFileName)) { + EscortFiles.createMediaCatch(mediaPath); + result = true; + } + return result; + } + + @Override + public FileVisitResult postVisitDirectory(Path paramT, IOException paramIOException) throws IOException { + return FileVisitResult.CONTINUE; + } + + @Override + public FileVisitResult preVisitDirectory(Path dir, BasicFileAttributes paramBasicFileAttributes) + throws IOException { + Path dirName = dir.getFileName(); + + if (skipPathNames.contains(dirName.toString())) { + logger.info(getSessionMarker(), "PreVisit skip {}", dir); + return FileVisitResult.SKIP_SUBTREE; + } else + logger.info(getSessionMarker(), "PreVisit {}", dir); + + return FileVisitResult.CONTINUE; + } + + private boolean processPathItem(Path mediaPath) throws Exception { + if (limit != 0 && submitted == limit) { + logger.info(getSessionMarker(), "Limit reached {}, canceling", limit); + return false; + } + + String fileName = mediaPath.getFileName().toString(); + if (fileName.startsWith(".") || fileName.endsWith(".nomd")) + return false; + + logger.info(getSessionMarker(), "Processing {}", mediaPath); + + File mediaFile = mediaPath.toFile(); + if (mediaFile.isDirectory()) { + //logger.info(getSessionMarker(), "Skipping directory {}", mediaPath); + return false; + } + + if (EscortFiles.isMediaCatched(mediaPath)) { + //logger.info(getSessionMarker(), "Skipping already catched {}", mediaPath); + return false; + } + + Path nomdFile = Paths.get(mediaPath.toString() + ".nomd"); + + if (Files.exists(nomdFile)) { + //logger.info(getSessionMarker(), "Skipping nomd file exists {}", nomdFile); + return false; + } + + if (handleArchiveConflict(mediaPath)) { + logger.info(getSessionMarker(), "Skipping archive db already contains {}", mediaPath); + return false; + } + + if (!canReadMediaInfo(mediaPath)) { + logger.info(getSessionMarker(), "Skipping cant read mediainfo {}", mediaPath); + return false; + } + + ArchiveItem archiveItem = createArchiveItem(mediaPath); + + if (archiveItem == null) { + Message msg = new ParameterizedMessage("Nincs metaadat!"); + logger.info(new MediaCubeMarker("vasary@elgekko.net,muszak@mediavivantis.hu", + "Értesítés problémás " + mediaPath.getFileName().toString() + " archiválásról"), msg); + Files.createFile(nomdFile); + return false; + } + + try { + checkArchiveItem(archiveItem); + + Map parameters = ListUtils.asMap(ARCHIVE_ITEM, archiveItem, KILL_DATE_DAYS, killDateDays); + IJobRuntime runtime = getEngine().submit(getJobRuntime(), e -> { + if (e.getStatus().equals(JobStatus.CANCELED) || e.getStatus().equals(JobStatus.SUSPENDED)) + EscortFiles.removeMediaCatch(mediaPath); + }, JOBTEMPLATE, ARCHIVE, 1, IJobEngine.DEFAULT_OWNER, parameters); + if (runtime == null) + throw new Exception("Submit returned null runtime"); + runtime.setRelated(mediaPath.toString()); + EscortFiles.createMediaCatch(mediaPath); + String metadata = archiveItemJSON(archiveItem); + EscortFiles.createMetadata(mediaPath.getParent().toString(), mediaPath.getFileName().toString(), metadata); + submitted++; + } catch (Exception e) { + logger.error(getSessionMarker(), + "A(z) '{}' állomány archiválási kísérlete sikertelen. A rendszer üzenete: {}", mediaPath, + e.getMessage()); + } + + return true; + } + + @Override + public FileVisitResult visitFile(Path filePath, BasicFileAttributes paramBasicFileAttributes) throws IOException { + logger.info(getSessionMarker(), "Will checked {}", filePath); + try { + processPathItem(filePath); + } catch (Exception e) { + logger.catching(e); + } + return FileVisitResult.CONTINUE; + } + + @Override + public FileVisitResult visitFileFailed(Path filePath, IOException paramIOException) throws IOException { + logger.info("Error archive {}", filePath); + return FileVisitResult.CONTINUE; + } +} diff --git a/server/-product/production/OMAR/jobs/steps/BatchRetrieveForkStep.java b/server/-product/production/OMAR/jobs/steps/BatchRetrieveForkStep.java new file mode 100644 index 00000000..4f6b130b --- /dev/null +++ b/server/-product/production/OMAR/jobs/steps/BatchRetrieveForkStep.java @@ -0,0 +1,76 @@ +package user.jobengine.server.steps; + +import java.util.List; +import java.util.Map; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + +import user.commons.log4j2.marker.MediaCubeFinishMarker; +import user.commons.log4j2.marker.MediaCubeMarker; +import user.jobengine.db.ArchivedMedia; +import user.jobengine.server.IJobEngine; +import user.jobengine.server.IJobRuntime; +import user.jobengine.server.scheduler.ScheduledJob; + +public class BatchRetrieveForkStep extends JobStep { + private static final String TARGET_PATH_TYPE = "targetPathType"; + private static final Logger logger = LogManager.getLogger(); + private static final String CHILD_TEMPLATE = "retrieve-ondemand.xml"; + private static final String ARCHIVEDMEDIA = "archivedMedia"; + private static final String RECIPIENT = "successRecipient"; + private static final String HOUSEID = "houseId"; + private MediaCubeMarker marker; + + @StepEntry + public Object[] execute(List basket, String houseId, String recipient, String targetPathType, IJobEngine jobEngine, IJobRuntime jobRuntime) + throws Exception { + marker = (MediaCubeMarker) jobRuntime.getSessionMarker(); + + //session szinten csak a finishMarker cimzettje az erdekes, es ezt a cimet pluszban hasznalja a konfigban megadott cimmel + //a finishMarker orokli a cim bellitast a sessionMarkertol + marker.setTo(recipient); + + ((MediaCubeMarker) jobRuntime.getFinishMarker()).setTo(recipient); + + if (basket == null || basket.size() == 0) + return null; + setProgress(10); + + MediaCubeMarker mailMarker = new MediaCubeMarker(recipient); + mailMarker.setSessionName("Archívum viszatöltés"); + mailMarker.setSessionID(houseId); + logger.info(mailMarker, "A visszatöltések elindultak az alábbi állományokra:"); + + if (jobRuntime.forkPrepare()) { + for (ArchivedMedia archivedMedia : basket) { + logger.info(mailMarker, archivedMedia.getMedia().getMediaFilesName()); + submit(archivedMedia, recipient, houseId, targetPathType, jobEngine, jobRuntime); + } + } + setProgress(50); + logger.info(new MediaCubeFinishMarker(mailMarker), "A visszatöltések végeztével megerősítő üzenetet küldünk."); + jobRuntime.forkWaitComplete(); + setProgress(100); + return null; + } + + public void submit(ArchivedMedia archivedMedia, String recipient, String houseId, String targetPathType, IJobEngine jobEngine, IJobRuntime jobRuntime) + throws Exception { + try { + ScheduledJob scheduledJob = jobEngine.getScheduledJob(CHILD_TEMPLATE); + Map parameters = scheduledJob.getJobParameters(); + parameters.put(ARCHIVEDMEDIA, archivedMedia); + parameters.put(HOUSEID, houseId); + parameters.put(RECIPIENT, recipient); + parameters.put(TARGET_PATH_TYPE, targetPathType); + IJobRuntime child = jobEngine.submit(jobRuntime, null, CHILD_TEMPLATE, String.format("Visszatöltés %s részére", recipient), parameters); + ((MediaCubeMarker) child.getSessionMarker()).setTo(recipient); + } catch (Exception e) { + logger.catching(e); + logger.error(marker, "Hiba a kötegelt visszatöltésben. A rendszer üzenete: {}", e.getMessage()); + } + + } + +} diff --git a/server/-product/production/OMAR/jobs/steps/CalculateMD5Step.java b/server/-product/production/OMAR/jobs/steps/CalculateMD5Step.java new file mode 100644 index 00000000..9db442c7 --- /dev/null +++ b/server/-product/production/OMAR/jobs/steps/CalculateMD5Step.java @@ -0,0 +1,34 @@ +package user.jobengine.server.steps; + +import java.io.InputStream; +import java.nio.file.Files; +import java.nio.file.Paths; +import java.security.DigestInputStream; +import java.security.MessageDigest; + +import javax.xml.bind.DatatypeConverter; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + +public class CalculateMD5Step extends JobStep { + private static final Logger logger = LogManager.getLogger(CalculateMD5Step.class); + + @StepEntry + public Object[] execute(String fileName) throws Exception { + logger.info(getMarker(), "Executing"); + + MessageDigest messageDigest = MessageDigest.getInstance("MD5"); + InputStream is = Files.newInputStream(Paths.get(fileName)); + DigestInputStream dis = new DigestInputStream(is, messageDigest); + byte[] digest = new byte[32768]; //32k buffer + + while ((dis.read(digest)) != -1) { + messageDigest.update(digest); + } + + String md5String = DatatypeConverter.printHexBinary(digest).toUpperCase(); + logger.info("calculated MD5 hash= {}", md5String); + return new Object[] { md5String }; + } +} \ No newline at end of file diff --git a/server/-product/production/OMAR/jobs/steps/CancelableStep.java b/server/-product/production/OMAR/jobs/steps/CancelableStep.java new file mode 100644 index 00000000..9293427a --- /dev/null +++ b/server/-product/production/OMAR/jobs/steps/CancelableStep.java @@ -0,0 +1,42 @@ +package user.jobengine.server.steps; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + +public class CancelableStep extends JobStep { + private static final Logger logger = LogManager.getLogger(); + int count = 10; + + @StepEntry + public Object[] execute(int param) throws Exception { + + getJobRuntime().setRelated("TESZT" + param); + for (int i = 0; i < count; i++) { + if (getJobRuntime().isWaitingCancel()) + break; +// if (i == 1) +// throw new Exception("AAAAAAAAAA"); + + Thread.sleep(1000); + int progress = (i + 1) * 100 / count; + setProgress(progress); + } + +// try { +// +// logger.warn("Ez a fo logba megy"); +// +// //marker="MEDIACUBE" +// logger.warn(getMarker(), "Ez a markered logba megy"); +// +// //marker="MEDIACUBE | folyamat_nev" +// logger.warn(getJobRuntime().getSessionMarker(), "Ez a markered logba megy es a nevesitett logba"); +// +// } catch (Exception e) { +// e.printStackTrace(); +// throw e; +// } + return new Object[] { param }; + } + +} diff --git a/server/-product/production/OMAR/jobs/steps/CleanupMountedLocationStep.java b/server/-product/production/OMAR/jobs/steps/CleanupMountedLocationStep.java new file mode 100644 index 00000000..95fff73f --- /dev/null +++ b/server/-product/production/OMAR/jobs/steps/CleanupMountedLocationStep.java @@ -0,0 +1,283 @@ +package user.jobengine.server.steps; + +import java.io.File; +import java.io.IOException; +import java.nio.file.DirectoryStream; +import java.nio.file.FileVisitResult; +import java.nio.file.FileVisitor; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.nio.file.SimpleFileVisitor; +import java.nio.file.attribute.BasicFileAttributes; +import java.text.ParseException; +import java.text.SimpleDateFormat; +import java.util.ArrayList; +import java.util.Collections; +import java.util.Date; +import java.util.List; + +import org.apache.commons.io.FileUtils; +import org.apache.commons.lang.StringUtils; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.apache.logging.log4j.Marker; + +import user.jobengine.server.steps.shared.ItemManagerExtensions; + +public class CleanupMountedLocationStep extends JobStep implements FileVisitor { + + private static final Logger logger = LogManager.getLogger(); + private static final String PROJECTFOLDER = "PROJECT"; + private static final String DATEFORMAT = "yyyyMMdd"; + private static final String DOT = "."; + private static final String STATUSFOLDER = ".STATUS"; + private static final String EWC2EXT = ".ewc2"; + private static final String XMPEXT = ".xmp"; + private static final String CATCHEDEXT = ".catched"; + private static final String KILLDATEEXT = ".killdate"; + private static final String JSONEXT = ".json"; + + private static boolean isEmpty(final Path directory) throws IOException { + try (DirectoryStream dirStream = Files.newDirectoryStream(directory)) { + final int[] count = new int[] { 0 }; + final int[] specialCount = new int[] { 0 }; + dirStream.forEach(p -> { + count[0]++; + // if (p.getFileName().toString().toLowerCase().equals(PROJECTFOLDER.toLowerCase())) + // specialCount[0]++; + if (p.getFileName().toString().toLowerCase().equals(STATUSFOLDER.toLowerCase())) + specialCount[0]++; + + }); + if (specialCount[0] == count[0]) + return true; + } + return false; + } + + private Marker marker; + + final int[] allCount = new int[] { 0 }; + final int[] currentCount = new int[] { 0 }; + + private Path sourcePath; + private SimpleDateFormat dateFormat; + private boolean skipArchiveCheck; + + private Date checkExpiration(List killDateFiles) { + Date killDate = null; + for (Path killDateFile : killDateFiles) { + Date currentKillDate = getKillDate(killDateFile); + if (currentKillDate == null) + continue; + if ((killDate != null && currentKillDate.after(killDate)) || killDate == null) + killDate = currentKillDate; + } + return new Date().after(killDate) ? killDate : null; + } + + @StepEntry + public Object[] execute(String sourceFolder, boolean skipArchiveCheck) throws Exception { + this.skipArchiveCheck = skipArchiveCheck; + marker = getSessionMarker(); + sourcePath = Paths.get(sourceFolder); + DirectoryStream directoryStream = null; + if (StringUtils.isBlank(sourcePath.toString())) { + logger.error(marker, "A folyamat 'sourcePath' bemeneti paramétere üres."); + throw new NullPointerException("System is not configured properly, 'sourceFolder' input parameter missing."); + } + + if (!Files.exists(sourcePath) || !Files.isDirectory(sourcePath)) { + logger.error(marker, "A {} mappa nem létezik.", sourceFolder); + throw new NullPointerException(String.format("Directory %s does not exist.", sourceFolder)); + } + + try { + setProgress(1); + dateFormat = new SimpleDateFormat(DATEFORMAT); + + Files.walkFileTree(sourcePath, new SimpleFileVisitor() { + @Override + public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) throws IOException { + allCount[0]++; + return super.visitFile(file, attrs); + } + }); + Files.walkFileTree(sourcePath, this); + } catch (Exception e) { + logger.catching(e); + logger.error(marker, "Hiba a '{}' mappa feldolgozásában. A rendszer hibaüzenete: {}", sourcePath, e.getMessage()); + throw e; + } finally { + if (directoryStream != null) { + try { + directoryStream.close(); + } catch (IOException e) { + } + } + } + return null; + } + + private Date getKillDate(Path killDateFile) { + String fileName = killDateFile.getFileName().toString(); + int end = fileName.lastIndexOf(DOT); + if (end < 1) + return null; + int start = fileName.lastIndexOf(DOT, end - 1); + if (start < 0) + return null; + String strKillDate = fileName.substring(start + 1, end); + Date result = null; + if (StringUtils.isNumeric(strKillDate)) { + try { + result = dateFormat.parse(strKillDate); + } catch (ParseException e) { + logger.error(marker, "A {} fájl 'killdate' állománya hibás formátumú, a {} karaktersorozat nem konvertálható dátummá.", fileName, strKillDate); + return null; + } + } else + logger.error(marker, "A {} fájl 'killdate' állománya hibás formátumú, az dátum helyett ez áll: '{}'.", fileName, strKillDate); + return result; + } + + private List getKillDateFiles(Path filePath) { + String killDateFilePattern = String.format("%s.*%s", filePath.getFileName().toString(), KILLDATEEXT); + List result = new ArrayList<>(); + Path statusPath = null; + try { + statusPath = Paths.get(filePath.getParent().toString(), STATUSFOLDER); + } catch (Exception e) { + logger.catching(e); + return null; + } + File statusPathFile = statusPath.toFile(); + if (statusPathFile.exists() && statusPathFile.isDirectory()) { + try (DirectoryStream stream = Files.newDirectoryStream(statusPath, killDateFilePattern)) { + stream.forEach(p -> result.add(p)); + } catch (Exception e) { + logger.catching(e); + } + } + Collections.sort(result); + return result; + } + + @Override + public FileVisitResult postVisitDirectory(Path dir, IOException exc) throws IOException { + if (!dir.equals(sourcePath) && isEmpty(dir)) { + if (!removeExistingSpecialDirectory(dir, PROJECTFOLDER)) + return FileVisitResult.CONTINUE; + if (!removeExistingSpecialDirectory(dir, STATUSFOLDER)) + return FileVisitResult.CONTINUE; + if (removeFile(dir)) + logger.info(marker, "A {} üres mappa törlése sikeres.", dir); + + } + return FileVisitResult.CONTINUE; + } + + @Override + public FileVisitResult preVisitDirectory(Path dir, BasicFileAttributes attrs) throws IOException { + + //A .-al kezdodo mappakat kihagyjuk + if (dir.getFileName().toString().startsWith(".")) + return FileVisitResult.SKIP_SUBTREE; + + return FileVisitResult.CONTINUE; + } + + private void processPathItem(Path filePath) { + currentCount[0]++; + + if (filePath.getFileName().toString().startsWith(".")) + return; + + int progress = currentCount[0] * 100 / allCount[0]; + setProgress(progress); + + logger.info("Checking {}", filePath); + List killDateFiles = getKillDateFiles(filePath); + if (killDateFiles == null || killDateFiles.size() == 0) { + logger.warn(marker, "A {} fájlhoz nem található 'killdate' állomány.", filePath); + return; + } + + if (killDateFiles.size() != 1) + logger.warn(marker, "A {} fájlhoz több 'killdate' állomány található, a legújabb dátum határozza meg a törlés időpontját.", filePath); + + Date killDate = checkExpiration(killDateFiles); + if (killDate == null) + return; + + if (!skipArchiveCheck && filePath.toFile().length() > 0) { + if (!ItemManagerExtensions.isArchived(getManager(), filePath)) { + logger.error(marker, "A(z) {} anyag törlésre van kijelölve, de nem található az archívumban.", filePath); + return; + } + } + + if (removeFiles(filePath, killDateFiles)) + logger.info(marker, "A {} fájl és kapcsolódó állományai a {} killdate bejegyzés alapján sikeresen törlődtek.", filePath.getFileName(), + dateFormat.format(killDate)); + else + logger.warn(marker, "A {} fájl és kapcsolódó állományai a {} killdate bejegyzés alapján csak részlegesen vagy egyáltalán nem törlődtek.", + filePath.getFileName(), dateFormat.format(killDate)); + } + + private boolean removeExistingSpecialDirectory(Path dir, String folderName) throws IOException { + File projectPath = Paths.get(dir.toString(), folderName).toFile(); + if (projectPath.exists() && projectPath.isDirectory()) { + FileUtils.deleteDirectory(projectPath); + if (projectPath.exists()) { + logger.warn(marker, "A {} alatti {} mappa törlése nem sikerült.", dir, folderName); + return false; + } + } + return true; + } + + private boolean removeFile(Path filePath) { + boolean result = false; + try { + //logger.error("REMOVE {}", filePath); + File file = filePath.toFile(); + if (file.exists()) + result = file.delete(); + } catch (Exception e) { + logger.error(marker, "A {} fájl nem törölhető. A rendszer hibaüzenete: {}", filePath, e.getMessage()); + } + return result; + } + + private boolean removeFiles(Path filePath, List killDateFiles) { + if (!removeFile(filePath)) + return false; + + removeFile(Paths.get(filePath.toString() + EWC2EXT)); + removeFile(Paths.get(filePath.toString() + XMPEXT)); + removeFile(Paths.get(filePath.getParent().toString(), STATUSFOLDER, filePath.getFileName().toString() + CATCHEDEXT)); + removeFile(Paths.get(filePath.getParent().toString(), STATUSFOLDER, filePath.getFileName().toString() + JSONEXT)); + + boolean result = true; + for (Path killDateFile : killDateFiles) { + if (!removeFile(killDateFile)) + result = false; + } + + return result; + } + + @Override + public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) throws IOException { + processPathItem(file); + return FileVisitResult.CONTINUE; + } + + @Override + public FileVisitResult visitFileFailed(Path file, IOException e) throws IOException { + logger.error(marker, "A {} fájl nem érhető el. A rendszer hibaüzenete: {}", file.toString(), e.getMessage()); + return FileVisitResult.CONTINUE; + } +} diff --git a/server/-product/production/OMAR/jobs/steps/CreateArchiveItemStep.java b/server/-product/production/OMAR/jobs/steps/CreateArchiveItemStep.java new file mode 100644 index 00000000..ec165e87 --- /dev/null +++ b/server/-product/production/OMAR/jobs/steps/CreateArchiveItemStep.java @@ -0,0 +1,47 @@ +package user.jobengine.server.steps; + +import java.nio.file.Paths; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + +import com.ibm.nosql.json.api.BasicDBObject; +import com.ibm.nosql.json.api.DB; +import com.ibm.nosql.json.api.DBCollection; + +import user.commons.nosql.NoSQLUtils; +import user.jobengine.db.Media; +import user.jobengine.db.Store; + +public class CreateArchiveItemStep extends JobStep { + private static final Logger logger = LogManager.getLogger(); + + @StepEntry + public Object[] execute(Media mediaCubeMedia, String localHiresPath) throws Exception { + DB db = NoSQLUtils.getNoSQLDB(); + DBCollection collection = db.getCollection("missing_lowres"); + Store highResStore = getManager().getSystemStore(false); + + ArchiveItem archiveItem = null; + try { + if (mediaCubeMedia.getMediaFilesCount() != 1) { + throw new Exception("Expected media count is 1, found " + mediaCubeMedia.getMediaFilesCount()); + } + if (mediaCubeMedia.getMediaFiles().get(0).getStoreId() != highResStore.getId()) { + throw new Exception("Expected media store is a high-res store"); + } + + String name = mediaCubeMedia.getMediaFileRealName(); + archiveItem = new ArchiveItem(); + archiveItem.setMediaFile(Paths.get(localHiresPath, name).toString()); + collection.save(new BasicDBObject("name", name)); + } catch (Exception e) { + logger.catching(e); + logger.info(getMarker(), e.getMessage()); + throw e; + } finally { + setProgress(100); + } + return new Object[] { archiveItem }; + } +} diff --git a/server/-product/production/OMAR/jobs/steps/CreateMissingLowresStep.java b/server/-product/production/OMAR/jobs/steps/CreateMissingLowresStep.java new file mode 100644 index 00000000..72b4cfb1 --- /dev/null +++ b/server/-product/production/OMAR/jobs/steps/CreateMissingLowresStep.java @@ -0,0 +1,95 @@ +package user.jobengine.server.steps; + +import java.nio.file.Paths; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + +import com.ibm.nosql.json.api.BasicDBObject; +import com.ibm.nosql.json.api.DB; +import com.ibm.nosql.json.api.DBCollection; + +import user.commons.log4j2.marker.MediaCubeUndoMarker; +import user.commons.nosql.NoSQLUtils; +import user.jobengine.db.IItemManager; +import user.jobengine.db.Media; +import user.jobengine.db.MediaFile; +import user.jobengine.server.IJobEngine; +import user.jobengine.server.IJobRuntime; + +public class CreateMissingLowresStep extends JobStep { + private static final Logger logger = LogManager.getLogger(); + + @StepEntry + public Object[] execute(String localHiresPath, IJobEngine jobEngine, IJobRuntime jobRuntime) throws Exception { + Object[] result = new Object[] { null, null, "%s", null, 0, true }; + + DB db = NoSQLUtils.getNoSQLDB(); + DBCollection collection = db.getCollection("missing_lowres"); + IItemManager manager = jobEngine.getItemManager(); + Media media = getFirstUntranscodedMedia(manager, collection); + + try { + if (media == null) { + logger.info(new MediaCubeUndoMarker(getSessionMarker().getSessionID()), "Nincs feldolgozandó hiány."); + // throw new Exception("Nincs feldolgozandó hiány."); + cancel(); + return null; + } + + String name = media.getMediaFileRealName(); + result[0] = media; + ArchiveItem archiveItem = new ArchiveItem(); + archiveItem.setMediaFile(Paths.get(localHiresPath, name).toString()); + result[1] = archiveItem; + collection.save(new BasicDBObject("name", name)); + logger.info(getSessionMarker(), "Processing mediaID: {}", media.getId()); + + } catch (Exception e) { + logger.catching(e); + logger.error(getSessionMarker(), e.getMessage()); + throw e; + } finally { + setProgress(100); + } + return result; + } + + private Media getFirstUntranscodedMedia(IItemManager manager, DBCollection collection) { + Media[] result = new Media[] { null }; + // MV + String query = "SELECT mediaid FROM VW_MISSING_PROXY_IDS WHERE HOUSEID like 'M%' or HOUSEID like 'P%' or HOUSEID like 'R%' ORDER BY modified DESC"; + + // HTV + // String query = "SELECT mediaid FROM VW_MISSING_PROXY_IDS"; + manager.executeQuery(query, rs -> { + try { + long mediaId = rs.getLong(1); + Media media = manager.getMedia(mediaId); + // a nevgeneralas miatt az eredeti MediaFilesName nem jo, a pontos nev kell + // nekunk + String name = media.getMediaFileRealName(); + //logger.info(getSessionMarker(), "Checking {}", name); + long existing = collection.find(new BasicDBObject("name", name)).count(); + if (existing > 0) { + // logger.info(getSessionMarker(), "{} is on missing_lowres list", name); + return true; + } + + // 210617 proxy keszites tiltasa + MediaFile mf = manager.getSystemMediaFile(media); + if (mf.isDisableProxy()) { + logger.info(getSessionMarker(), "Proxy disabled {}", name); + + return true; + } + + result[0] = media; + } catch (Exception e) { + logger.error(e); + } + return false; + }, null); + return result[0]; + } +} diff --git a/server/-product/production/OMAR/jobs/steps/DeleteFile.java b/server/-product/production/OMAR/jobs/steps/DeleteFile.java new file mode 100644 index 00000000..539faf74 --- /dev/null +++ b/server/-product/production/OMAR/jobs/steps/DeleteFile.java @@ -0,0 +1,22 @@ +package user.jobengine.server.steps; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + +import user.commons.RemoteFile; +import user.commons.StoreUri; + +public class DeleteFile extends JobStep { + private static final Logger logger = LogManager.getLogger(); + + @StepEntry + public Object[] execute(String fileName, StoreUri tempStoreUri) throws Exception { + try { + RemoteFile remoteFile = tempStoreUri.getRemoteFile(fileName); + tempStoreUri.delete(remoteFile); + } catch (Exception e) { + logger.warn(getJobRuntime().getSessionMarker(), e.getMessage()); + } + return null; + } +} diff --git a/server/-product/production/OMAR/jobs/steps/DummyTestStep1.java b/server/-product/production/OMAR/jobs/steps/DummyTestStep1.java new file mode 100644 index 00000000..22b3ab97 --- /dev/null +++ b/server/-product/production/OMAR/jobs/steps/DummyTestStep1.java @@ -0,0 +1,23 @@ +package user.jobengine.server.steps; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + +public class DummyTestStep1 extends JobStep { + private static final Logger logger = LogManager.getLogger(DummyTestStep1.class); + + @StepEntry + public Object[] execute(String param1) { + logger.info("Executing DummyTestStep1"); + int var1 = 0; + if (param1.equals("Jozsi")) { + var1 = 1; + Thread.sleep(1000); + setProgress(50); + Thread.sleep(1000); + setProgress(50); + } + + return new Object[] {var1}; + } +} diff --git a/server/-product/production/OMAR/jobs/steps/DummyTestStep2.java b/server/-product/production/OMAR/jobs/steps/DummyTestStep2.java new file mode 100644 index 00000000..e7579f12 --- /dev/null +++ b/server/-product/production/OMAR/jobs/steps/DummyTestStep2.java @@ -0,0 +1,20 @@ +package user.jobengine.server.steps; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + +public class DummyTestStep2 extends JobStep { + private static final Logger logger = LogManager.getLogger(DummyTestStep2.class); + + @StepEntry + public Object[] execute(int var2) { + logger.info("Executing DummyTestStep2"); + long var3 = 0L; + Thread.sleep(1000); + setProgress(50); + Thread.sleep(1000); + setProgress(50); + + return new Object[] { var3 }; + } +} diff --git a/server/-product/production/OMAR/jobs/steps/DummyTestStep3.java b/server/-product/production/OMAR/jobs/steps/DummyTestStep3.java new file mode 100644 index 00000000..93d76a75 --- /dev/null +++ b/server/-product/production/OMAR/jobs/steps/DummyTestStep3.java @@ -0,0 +1,17 @@ +package user.jobengine.server.steps; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + +public class DummyTestStep3 extends JobStep { + private static final Logger logger = LogManager.getLogger(DummyTestStep3.class); + + @StepEntry + public void execute(long var3) { + Thread.sleep(1000); + setProgress(50); + Thread.sleep(1000); + setProgress(50); + logger.info("Executing DummyTestStep3"); + } +} diff --git a/server/-product/production/OMAR/jobs/steps/FileCopyStep.java b/server/-product/production/OMAR/jobs/steps/FileCopyStep.java new file mode 100644 index 00000000..c4172550 --- /dev/null +++ b/server/-product/production/OMAR/jobs/steps/FileCopyStep.java @@ -0,0 +1,95 @@ +package user.jobengine.server.steps; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.apache.logging.log4j.Marker; + +import user.commons.StoreUri; +import user.commons.remotestore.IProgressEventListener; +import user.commons.remotestore.ProgressEvent; +import user.commons.remotestore.RemoteStoreProtocol; +import user.jobengine.db.IItemManager; +import user.jobengine.server.IJobEngine; +import user.jobengine.server.IJobRuntime; + +public class FileCopyStep extends JobStep { + private static final Logger logger = LogManager.getLogger(); + private IItemManager manager; + private Marker marker; + + private void check(String sourceProtocol, String sourcePath, String sourceFileName, String targetProtocol, String targetPath, String targetFileName, + IJobEngine jobEngine, IJobRuntime jobRuntime) { + if (jobEngine == null) { + logger.error(marker, "A folyamatkezelő réteg nem elérhető."); + throw new NullPointerException("Internal error, missing JobEngine reference."); + } + manager = jobEngine.getItemManager(); + if (manager == null) { + logger.error(marker, "Az adatbáziskezelő réteg nem elérhető."); + throw new NullPointerException("Internal error, missing ItemManager reference."); + } + if (sourceProtocol == null) { + logger.error(marker, "A forrás protokol bemeneti paramétere üres."); + throw new NullPointerException("System is not configured properly, 'sourceProtocol' input parameter missing."); + } + if (sourcePath == null) { + logger.error(marker, "A forrás fájl elérés bemeneti paramétere üres."); + throw new NullPointerException("System is not configured properly, 'sourcePath' input parameter missing."); + } + if (sourceFileName == null) { + logger.error(marker, "A forrás fájlnév bemeneti paramétere üres."); + throw new NullPointerException("System is not configured properly, 'sourceFileName' input parameter missing."); + } + if (targetProtocol == null) { + logger.error(marker, "A cél protokol bemeneti paramétere üres."); + throw new NullPointerException("System is not configured properly, 'targetProtocol' input parameter missing."); + } + if (targetPath == null) { + logger.error(marker, "A cél fájl elérés bemeneti paramétere üres."); + throw new NullPointerException("System is not configured properly, 'targetPath' input parameter missing."); + } + if (targetFileName == null) { + logger.error(marker, "A cél fájlnév bemeneti paramétere üres."); + throw new NullPointerException("System is not configured properly, 'targetFileName' input parameter missing."); + } + } + + @StepEntry + public Object[] execute(String sourceProtocol, String sourcePath, String sourceFileName, String targetProtocol, String targetPath, String targetFileName, + int killDateDays, IJobEngine jobEngine, IJobRuntime jobRuntime) throws Exception { + marker = jobRuntime.getSessionMarker(); + + check(sourceProtocol, sourcePath, sourceFileName, targetProtocol, targetPath, targetFileName, jobEngine, jobRuntime); + + StoreUri source = null; + StoreUri target = null; + + try { + source = manager.createStoreUri(Enum.valueOf(RemoteStoreProtocol.class, sourceProtocol), sourcePath); + target = manager.createStoreUri(Enum.valueOf(RemoteStoreProtocol.class, targetProtocol), targetPath); + source.addProgressListener(new IProgressEventListener() { + @Override + public void progressChanged(ProgressEvent evt) { + jobRuntime.incrementProgress(evt.getProgress()); + } + }); + + source.transferFrom(target, sourceFileName, targetFileName); + if (killDateDays > -1) + EscortFiles.createUNCKillDate(targetPath, targetFileName, killDateDays, marker); + + } catch (Exception e) { + logger.catching(e); + throw e; + } finally { + if (source != null) { + source.cleanUp(); + } + if (target != null) { + target.cleanUp(); + } + } + + return null; + } +} diff --git a/server/-product/production/OMAR/jobs/steps/GenerateJSONMetadataStep.java b/server/-product/production/OMAR/jobs/steps/GenerateJSONMetadataStep.java new file mode 100644 index 00000000..210d7354 --- /dev/null +++ b/server/-product/production/OMAR/jobs/steps/GenerateJSONMetadataStep.java @@ -0,0 +1,65 @@ +package user.jobengine.server.steps; + +import java.io.IOException; +import java.nio.file.Paths; +import java.sql.Timestamp; +import java.util.Map; + +import org.apache.commons.io.FilenameUtils; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + +import user.commons.DownloadableMedia; +import user.commons.StoreUri; +import user.commons.mediaarea.MediaArea; +import user.commons.remotestore.RemoteStoreProtocol; +import user.jobengine.server.steps.shared.EscortFiles; + +public class GenerateJSONMetadataStep extends JobStep { + private static final Logger logger = LogManager.getLogger(); + + @StepEntry + public Object[] execute(Map files, String sourceStoreName, String targetStoreName, + String escortStoreFolder) { + if (!files.isEmpty()) { + for (String fullPath : files.keySet()) { + MediaArea mediaArea = files.get(fullPath); + String title = FilenameUtils.getBaseName(fullPath).replace(FilenameUtils.getExtension(fullPath), ""); + String pathOnly = fullPath.substring(0, fullPath.lastIndexOf("\\") + 1); + String filenameOnly = FilenameUtils.getBaseName(fullPath); + Timestamp created = null; + Timestamp modified = null; + long frameCount = mediaArea.getFrameCount(); + long mediaId = 0; + StoreUri sourceStoreUri = getManager().getStoreUri(sourceStoreName, RemoteStoreProtocol.LOCAL); + StoreUri targetStoreUri = getManager().getStoreUri(targetStoreName, RemoteStoreProtocol.LOCAL); + StoreUri escortStoreUri = getManager().getStoreUri(escortStoreFolder, RemoteStoreProtocol.LOCAL); + + try { + String outputPath = Paths.get(escortStoreUri.toString(true)).toString(); + } catch (Exception e1) { + e1.printStackTrace(); + } + + DownloadableMedia downloadable = DownloadableMedia.create(title, filenameOnly, modified, created, + frameCount, 0, sourceStoreUri.getId(), targetStoreUri.getId(), mediaId); + String escortFileName = targetStoreName + "." + downloadable.getString("fileName"); // needed without + // extension + try { + if (EscortFiles.createMetadataIfNotExists(pathOnly, escortFileName, + downloadable.toPrettyString(""))) { + logger.info(getMarker(), "Archive status file created for {}", fullPath); + } else { + logger.info(getMarker(), "Archive status file already exists for {}", fullPath); + } + } catch (IOException e) { + logger.error("{}", e.getCause()); + e.printStackTrace(); + } + } + } else { + logger.info("files is empty!"); + } + return null; + } +} diff --git a/server/-product/production/OMAR/jobs/steps/HSMMigrateStep.java b/server/-product/production/OMAR/jobs/steps/HSMMigrateStep.java new file mode 100644 index 00000000..895b1411 --- /dev/null +++ b/server/-product/production/OMAR/jobs/steps/HSMMigrateStep.java @@ -0,0 +1,401 @@ +package user.jobengine.server.steps; + +import java.io.BufferedInputStream; +import java.io.BufferedOutputStream; +import java.io.File; +import java.io.FileInputStream; +import java.io.FileOutputStream; +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.nio.file.attribute.BasicFileAttributes; +import java.text.SimpleDateFormat; +import java.util.Arrays; +import java.util.LinkedHashMap; +import java.util.List; +import java.util.Map; + +import org.apache.commons.io.FilenameUtils; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.apache.logging.log4j.Marker; + +import com.ibm.nosql.json.api.BasicDBObject; +import com.ibm.nosql.json.api.DB; +import com.ibm.nosql.json.api.DBCollection; +import com.ibm.nosql.json.api.QueryBuilder; + +import user.commons.IEntityBase; +import user.commons.log4j2.marker.MediaCubeMarker; +import user.commons.nosql.NoSQLUtils; +import user.jobengine.db.IItemManager; +import user.jobengine.db.MediaFile; +import user.jobengine.db.MediaFileDAO; +import user.mediacube.metadata.interfaces.IMetadata; +import user.mediacube.metadata.interfaces.IMetadataProvider; +import user.mediacube.metadata.interfaces.IMetadataProviderFactory; +import user.mediacube.metadata.interfaces.MetadataProviderType; +import user.mediacube.metadata.interfaces.MetadataType; +import user.mediacube.metadata.interfaces.PlanAirMetadataListOptions; + +public class HSMMigrateStep extends JobStep { + private static final Logger logger = LogManager.getLogger(); + private static final String MXFEXT = ".mxf"; + private static final String MOVEXT = ".mov"; + private Marker marker = null; + private IMetadataProvider hsmProvider; + private Map tapeContents = new LinkedHashMap<>(); + private DBCollection excludes; + private DBCollection fileHistory; + private DBCollection volumeHistory; + private DB db; + private IMetadataProvider planairProvider; + private SimpleDateFormat df = new SimpleDateFormat("yyyy.MM.dd"); + + private void cleanupHistory() { + if (db == null) + db = NoSQLUtils.getNoSQLDB(); + if (fileHistory == null) + fileHistory = db.getCollection("hsm_migrate_file_history"); + if (volumeHistory == null) + volumeHistory = db.getCollection("hsm_migrate_volume_history"); + fileHistory.drop(); + volumeHistory.drop(); + } + + private BasicDBObject createMetadata(String volumeName, String fileName) throws Exception { + + Path filePath = Paths.get(fileName); + String mediaHouseId = FilenameUtils.removeExtension(filePath.getFileName().toString()); + BasicDBObject result = null; + try { + result = getPlanAirMetadata(mediaHouseId); + } catch (Exception e) { + + logger.error("PlanAir metadata error", e); + //nem latja a drivert pl. + //throw e; + } + + if (result == null) { + result = new BasicDBObject(); + BasicFileAttributes attr = Files.readAttributes(filePath, BasicFileAttributes.class); + result.put("itemHouseId", df.format(attr.lastModifiedTime().toMillis())); + result.put("itemTitle", filePath.getParent().toString()); + result.put("mediaHouseId", mediaHouseId); + result.put("mediaTitle", fileName); + result.put("mediaDescription", volumeName); + result.put("mediaType", "Generic"); + } + result.put("userName", "mediacube"); + return result; + } + + @SuppressWarnings("serial") + @StepEntry + public Object[] execute(String sourceLocation, String targetLocation) throws Exception { + marker = getJobRuntime().getSessionMarker(); + //remove from prod + //cleanupHistory(); + hsmProvider = getMetadataProvider(MetadataProviderType.HSM); + if (hsmProvider == null) + throw new NullPointerException("No HSM metadata provider available"); + planairProvider = getMetadataProvider(MetadataProviderType.PLANAIR); + if (planairProvider == null) + throw new NullPointerException("No PLANAIR metadata provider available"); + Path targetPath = Paths.get(targetLocation); + try { + List volumes = hsmProvider.list(new BasicDBObject()); + for (IMetadata volume : volumes) { + String volumeName = volume.getTitle(); + + BasicDBObject historyResult = queryVolumeHistory(volumeName); + if (historyResult != null) { + logger.info(marker, "A kazetta már feldolgozásra került: {}", volumeName); + continue; + } + logger.info(marker, "A kazetta feldolgozása elindul: {}", volumeName); + List contents = getContents(volumeName); + int p = 0; + boolean oneSuccess = false; + boolean hasError = false; + for (IMetadata content : contents) { + BasicDBObject c = content.asJSON(); + String hsmFileName = c.getString("fileName"); + long contentFileSize = NoSQLUtils.asLong(c, "fileSize"); + if (!tapeContents.containsKey(hsmFileName)) { + tapeContents.put(hsmFileName, c); + Path sourceFilePath = Paths.get(sourceLocation, hsmFileName); + if (contentFileSize < Files.getFileStore(targetPath).getUsableSpace()) { + Path targetFilePath = Paths.get(targetLocation, sourceFilePath.getFileName().toString()); + + try { + if (processHSMFile(volumeName, hsmFileName, sourceFilePath, targetFilePath)) { + oneSuccess = true; + } + } catch (Exception e) { + hasError = true; + } + } + } + setProgress(p++ * 100 / contents.size()); + + if (getJobRuntime().isWaitingCancel()) { + logger.info("Job canceled by user"); + //ne mentsuk a szalagot meg + return null; + } + + } + + if (oneSuccess && !hasError) { + saveVolumeHistory(volumeName); + String subject = "A kazetta eltávolítható a HSM rendszerből: " + volumeName; + logger.info(new MediaCubeMarker() { + { + setSubject(subject); + } + }, subject); + } + + //ha mar sikerult valamit archivalni kilepunk + if (oneSuccess) + break; + } + } catch (Exception e) { + logger.error(marker, "Hiba a migráció során. A rendszer hibaüzenete: {}", e.getMessage()); + throw e; + } + + return null; + } + + private List getContents(String volumeName) throws Exception { + List contents = null; + contents = hsmProvider.list(new BasicDBObject("volumeName", volumeName)); + return contents; + } + + protected IMetadataProvider getMetadataProvider(MetadataProviderType type) { + IMetadataProviderFactory factory = getService(IMetadataProviderFactory.class); + if (factory == null) + return null; + return factory.getProvider(type); + } + + private BasicDBObject getPlanAirMetadata(String mediaHouseId) throws Exception { + PlanAirMetadataListOptions opt = new PlanAirMetadataListOptions(); + opt.setSearch(mediaHouseId); + + List result = null; + opt.setType(MetadataType.Material); + result = planairProvider.list(opt); + if (result.size() != 0) + return result.get(0).asJSON(); + + opt.setType(MetadataType.Promo); + result = planairProvider.list(opt); + if (result.size() != 0) + return result.get(0).asJSON(); + + opt.setType(MetadataType.AD); + result = planairProvider.list(opt); + if (result.size() != 0) + return result.get(0).asJSON(); + + return null; + } + + //true if need copy + public boolean prepareCopy(String hsmFileName, Path source, Path target) throws IOException { + boolean result = true; + BasicDBObject excludeResult = queryExclude(hsmFileName); + if (excludeResult != null) { + logger.warn(marker, "Kivételként megjelölt: {}", hsmFileName); + return false; + } + + File sourceFile = source.toFile(); + File targetFile = target.toFile(); + + if (!sourceFile.exists()) { + logger.warn(marker, "A forrás nem elérhető: {}", source); + return false; + } + + // BasicDBObject historyResult = queryFileHistory(contentFileName); + // if (historyResult != null) + // return false; + + // if (!mediaFile.getName().toLowerCase().endsWith(MOVEXT.toLowerCase()) && !mediaFile.getName().toLowerCase().endsWith(MXFEXT.toLowerCase())) + // return; + // logger.info("Start copy from {} to {}", sourceFilePath, targetFilePath); + + boolean targetExists = targetFile.exists(); + + long targetLength = targetFile.length(); + long sourceLength = sourceFile.length(); + + if (targetLength == sourceLength) { + logger.warn(marker, "A fájl már fel van dolgozva: {}, {} -> {}", source, sourceLength, targetLength); + return false; + } + + if (targetLength > sourceLength) { + logger.warn(marker, "A célfájl nagyobb, törlöm: {}", target); + Files.delete(target); + targetLength = 0; + targetExists = false; + } + + if (targetExists) { + logger.warn(marker, "A fájl már létezik, a másolás folytatódik: {}, {} -> {}", target, sourceLength, targetLength); + } else + logger.warn(marker, "Migrálás: {}, {} -> {}", source, sourceLength, targetLength); + + return result; + } + + private boolean processHSMFile(String volumeName, String hsmFileName, Path sourceFilePath, Path targetFilePath) throws Exception { + int repeat = 4; + boolean successCopy = false; + + IItemManager manager = getManager(); + MediaFileDAO mfDAO = (MediaFileDAO) manager.getBaseDAO(MediaFile.class); + List mediaFiles = mfDAO.getByHouseId(sourceFilePath.getFileName().toString()); + if (mediaFiles != null && mediaFiles.size() > 0) { + logger.warn(marker, "Már archivált: {}", hsmFileName); + return false; + } + + if (prepareCopy(hsmFileName, sourceFilePath, targetFilePath)) { + while (repeat > 0) { + try { + resumeableCopy(sourceFilePath, targetFilePath); + repeat = 0; + successCopy = true; + } catch (Exception e) { + if (Files.exists(targetFilePath) && targetFilePath.toFile().length() == 0) + Files.delete(targetFilePath); + //logger.warn(marker, "Hiba a másolás során: {} ({})", sourceFilePath, e.getMessage()); + repeat--; + } + } + } + String metadataFileName = sourceFilePath.getFileName() + EscortFiles.DOT_JSON; + Path metadataPath = Paths.get(targetFilePath.getParent().toString(), EscortFiles.STATUSFOLDER, metadataFileName); + boolean createMetadata = Files.exists(targetFilePath) && !Files.exists(metadataPath); + + if (successCopy || createMetadata) { + String metadata = null; + try { + metadata = createMetadata(volumeName, hsmFileName).toPrettyString(""); + EscortFiles.createMetadata(targetFilePath.getParent().toString(), targetFilePath.getFileName().toString(), metadata); + //saveFileHistory(contentFileName); + + } catch (Exception e) { + logger.error(marker, "Metadata error", e); + return false; + } + return true; + } + + logger.error(marker, "A fájl másolása nem lehetséges: {}", sourceFilePath); + return false; + } + + public BasicDBObject queryExclude(String fileName) { + if (db == null) + db = NoSQLUtils.getNoSQLDB(); + if (excludes == null) + excludes = db.getCollection("hsm_migrate_exclude"); + Path filePath = Paths.get(fileName); + String pureFileName = FilenameUtils.removeExtension(filePath.getFileName().toString()); + QueryBuilder qb = QueryBuilder.start("name").in(Arrays.asList(fileName, pureFileName)); + BasicDBObject exceptionResult = NoSQLUtils.asSingle(excludes.find(qb.get())); + return exceptionResult; + } + + public BasicDBObject queryFileHistory(String fileName) { + if (db == null) + db = NoSQLUtils.getNoSQLDB(); + if (fileHistory == null) + fileHistory = db.getCollection("hsm_migrate_file_history"); + QueryBuilder qb = QueryBuilder.start("name").in(Arrays.asList(fileName)); + BasicDBObject historyResult = NoSQLUtils.asSingle(fileHistory.find(qb.get())); + return historyResult; + } + + public BasicDBObject queryVolumeHistory(String volumeName) { + if (db == null) + db = NoSQLUtils.getNoSQLDB(); + if (volumeHistory == null) + volumeHistory = db.getCollection("hsm_migrate_volume_history"); + QueryBuilder qb = QueryBuilder.start("name").in(Arrays.asList(volumeName)); + BasicDBObject historyResult = NoSQLUtils.asSingle(volumeHistory.find(qb.get())); + return historyResult; + } + + public void resumeableCopy(Path source, Path target) throws Exception { + File sourceFile = source.toFile(); + File targetFile = target.toFile(); + boolean targetExists = targetFile.exists(); + + long targetLength = targetFile.length(); + long sourceLength = sourceFile.length(); + + try (InputStream in = new BufferedInputStream(new FileInputStream(sourceFile)); + OutputStream out = new BufferedOutputStream(new FileOutputStream(targetFile, targetExists))) { + + byte[] buffer = new byte[128 * 1024]; + int lengthRead; + + if (targetExists) + in.skip(targetLength); + + while ((lengthRead = in.read(buffer)) > 0) { + out.write(buffer, 0, lengthRead); + out.flush(); + targetLength = targetFile.length(); + if (targetLength > sourceLength) { + throw new Exception("Hiba! A fájl túl nagy lett."); + } + + if (getJobRuntime().isWaitingCancel()) { + break; + } + } + + targetLength = targetFile.length(); + sourceLength = sourceFile.length(); + if (targetLength != sourceLength) { + throw new Exception("Hiba! A fájl mérete nem egyezik."); + } + } + } + + private void saveFileHistory(String fileName) { + if (db == null) + db = NoSQLUtils.getNoSQLDB(); + if (fileHistory == null) + fileHistory = db.getCollection("hsm_migrate_file_history"); + BasicDBObject item = new BasicDBObject(); + item.put("name", fileName); + fileHistory.save(item); + + } + + private void saveVolumeHistory(String volumeName) { + if (db == null) + db = NoSQLUtils.getNoSQLDB(); + if (volumeHistory == null) + volumeHistory = db.getCollection("hsm_migrate_volume_history"); + BasicDBObject item = new BasicDBObject(); + item.put("name", volumeName); + volumeHistory.save(item); + + } +} diff --git a/server/-product/production/OMAR/jobs/steps/IntegrationTestStep.java b/server/-product/production/OMAR/jobs/steps/IntegrationTestStep.java new file mode 100644 index 00000000..081d02d8 --- /dev/null +++ b/server/-product/production/OMAR/jobs/steps/IntegrationTestStep.java @@ -0,0 +1,128 @@ +package user.jobengine.server.steps; + +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.concurrent.CountDownLatch; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + +import user.commons.ListUtils; +import user.jobengine.db.ArchivedMedia; +import user.jobengine.db.Item; +import user.jobengine.db.Media; +import user.jobengine.server.steps.shared.ItemManagerExtensions; + +public class IntegrationTestStep extends JobStep { + private static final Logger logger = LogManager.getLogger(); + private ArchivedMedia archivedMedia; + + private void _00_test_cancelable() throws Exception { + CountDownLatch finishLatch = new CountDownLatch(1); + getEngine().submit(null, e -> { + if (e.isRuntimeTerminated()) + finishLatch.countDown(); + }, "cancelable.xml", "Test cancelable", ListUtils.asMap("param", 1)); + finishLatch.await(); + logger.info("_00_test_cancelable SUCCESS"); + } + + private void _01_test_retrieve_ondemand() throws Exception { + archivedMedia = new ArchivedMedia(); + Media media = getManager().getMedia(15285); + Item item = getManager().getItem(media.getItemId()); + archivedMedia.setItem(item); + archivedMedia.setMedia(media); + + CountDownLatch finishLatch = new CountDownLatch(1); + getEngine().submit(null, e -> { + if (e.isRuntimeTerminated()) + finishLatch.countDown(); + + }, "retrieve-ondemand.xml", "Test retrieve-ondemand", + ListUtils.asMap("globalRetrievePath", "file://10.11.1.100", "localRetrievePath", "/mediacube/data", "materialOutputFolder", "/", + "promoOutputFolder", "/", "advertisementOutputFolder", "/", "octopusOutputFolder", "/", "genericOutputFolder", "/", + "onlineOutputFolder", "/", "killDateDays", -1, "nexioAgency", "ARCHIVE_RESTORE", "nexioPort", 2098, "nexioUserName", "administrator", + "nexioPassword", "system", "archivedMedia", archivedMedia, "successRecipient", "vasary@elgekko.net", "houseId", + archivedMedia.getMedia().getHouseId(), "targetPathType", "0")); + + finishLatch.await(); + Path output = Paths.get("/mediacube/data", archivedMedia.getMedia().getHouseId(), + archivedMedia.getMedia().getHouseId() + "-ARCH-" + archivedMedia.getMedia().getMediaFileRealName()); + if (!Files.exists(output)) + throw new Exception("File does not exist: " + output); + + logger.info("_01_test_retrieve_ondemand SUCCESS"); + } + + private void _02_test_archive_material() throws Exception { + Path input = Paths.get("/mediacube/data", archivedMedia.getMedia().getHouseId(), + archivedMedia.getMedia().getHouseId() + "-ARCH-" + archivedMedia.getMedia().getMediaFileRealName()); + + String outputName = "IntegrationTest_" + System.currentTimeMillis() + ".mxf"; + Path output = Paths.get(input.getParent().toString(), outputName); + Files.copy(input, output); + if (!Files.exists(output)) + throw new Exception("File does not exist: " + output); + + ArchiveItem archiveItem = new ArchiveItem(); + archiveItem.setMediaFile(output.toString()); + archiveItem.setItemHouseId(outputName); + archiveItem.setItemTitle(outputName); + archiveItem.setMediaHouseId(outputName); + archiveItem.setMediaTitle(outputName); + archiveItem.setMediaType("Generic"); + + CountDownLatch finishLatch = new CountDownLatch(1); + getEngine().submit(null, e -> { + if (e.isRuntimeTerminated()) + finishLatch.countDown(); + }, "archive-material.xml", "Test archive-material", ListUtils.asMap("archiveItem", archiveItem, "killDateDays", 0)); + finishLatch.await(); + + if (!ItemManagerExtensions.isArchived(getManager(), output)) + throw new Exception("File not archived: " + output); + + Files.delete(output); + + logger.info("_02_test_archive_material SUCCESS"); + } + + private void _03_test_delete_materials() throws Exception { + Path output = Paths.get("/mediacube/data", archivedMedia.getMedia().getHouseId(), + archivedMedia.getMedia().getHouseId() + "-ARCH-" + archivedMedia.getMedia().getMediaFileRealName()); + String source = output.getParent().toString(); + + CountDownLatch finishLatch = new CountDownLatch(1); + getEngine().submit(null, e -> { + if (e.isRuntimeTerminated()) + finishLatch.countDown(); + }, "delete-materials.xml", "Test delete-materials", ListUtils.asMap("sourcePath", source, "skipArchiveCheck", true)); + finishLatch.await(); + + if (Files.exists(output)) + throw new Exception("File exists: " + output); + } + + @StepEntry + public Object[] execute() throws Exception { + getJobRuntime().setDescription("_00_test_cancelable"); + _00_test_cancelable(); + setProgress(25); + + getJobRuntime().setDescription("_01_test_retrieve_ondemand"); + _01_test_retrieve_ondemand(); + setProgress(50); + + getJobRuntime().setDescription("_02_test_archive_material"); + _02_test_archive_material(); + setProgress(75); + + getJobRuntime().setDescription("_03_test_delete_materials"); + _03_test_delete_materials(); + setProgress(100); + return null; + } + +} diff --git a/server/-product/production/OMAR/jobs/steps/MXFCutterStep.java b/server/-product/production/OMAR/jobs/steps/MXFCutterStep.java new file mode 100644 index 00000000..58a5944d --- /dev/null +++ b/server/-product/production/OMAR/jobs/steps/MXFCutterStep.java @@ -0,0 +1,129 @@ +package user.jobengine.server.steps; + +import org.apache.commons.lang.StringUtils; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.apache.logging.log4j.Marker; + +import user.commons.RemoteFile; +import user.commons.StoreUri; +import user.commons.remotestore.IProgressEventListener; +import user.commons.remotestore.ProgressEvent; +import user.commons.remotestore.RemoteStoreProtocol; +//import user.jobengine.db.Media; +import user.jobengine.db.ArchivedMedia; +import user.jobengine.db.IItemManager; +import user.jobengine.db.Store; +import user.jobengine.server.IJobEngine; +import user.jobengine.server.IJobRuntime; + +public class MXFCutterStep extends JobStep { + private static final String TARGETNAMEPATTERN = "-ARCH-%s"; + private static final Logger logger = LogManager.getLogger(); + private IItemManager manager; + private StoreUri tempTargetUri; + private StoreUri tempSourceUri; + private String sourceFileName; + private Marker marker; + private int nexioPort; + private String nexioUserName, nexioPassword; + private String nexioHost; + + protected void checkTargetPath(String targetPath) { + if (StringUtils.isBlank(targetPath)) { + logger.error(marker, "A folyamat 'targetPath' bemeneti paramétere üres."); + throw new NullPointerException("System is not configured properly, 'targetPath' input parameter missing."); + } + } + + protected StoreUri createTargetUri(IItemManager manager, String targetPath) throws NullPointerException { + StoreUri result = null; + result = manager.createStoreUri(RemoteStoreProtocol.FTP, nexioHost); + result.setPortNumber(nexioPort); + result.setUserName(nexioUserName); + result.setPassword(nexioPassword); + + return result; + } + + @StepEntry + public Object[] execute(ArchivedMedia archivedMedia, String targetPath, String houseId, String successRecipient, int killDateDays, boolean useNexioTarget, + String nexioAgency, int nexioPort, String nexioUserName, String nexioPassword, IJobEngine jobEngine, IJobRuntime jobRuntime) throws Exception { + this.nexioPort = nexioPort; + this.nexioUserName = nexioUserName; + this.nexioPassword = nexioPassword; + nexioHost = System.getProperty("nexio.host"); + marker = jobRuntime.getSessionMarker(); + + if (useNexioTarget && archivedMedia.getTcIn() != null && archivedMedia.getTcOut() != null) { + setAndCheck(archivedMedia, houseId, targetPath, useNexioTarget, jobEngine); + + final IJobRuntime runtime = jobRuntime; + sourceFileName = houseId + TARGETNAMEPATTERN; + tempSourceUri.addProgressListener(new IProgressEventListener() { + @Override + public void progressChanged(ProgressEvent evt) { + runtime.incrementProgress(evt.getProgress()); + } + }); + + RemoteFile result = tempSourceUri.transferFrom(tempTargetUri, sourceFileName, sourceFileName); + + EscortFiles.setNEXIOKillDate(killDateDays, houseId, nexioAgency, tempTargetUri); + + logger.info("A {} videó kivágva {}s - {}s", sourceFileName, archivedMedia.getTcIn(), archivedMedia.getTcOut()); + } + + return null; + } + + // private String getSourceFileName(ArchivedMedia archivedMedia, Store + // store) { + // List mediaFiles = archivedMedia.getMedia().getMediaFiles(); + // if (mediaFiles == null) + // return null; + // for (MediaFile mediaFile : mediaFiles) { + // if (mediaFile.getStore().getId() == store.getId()) + // return mediaFile.getRelativePath(); + // } + // return null; + // } + + private void setAndCheck(ArchivedMedia archivedMedia, String houseId, String targetPath, boolean useNexioTarget, IJobEngine jobEngine) { + if (jobEngine == null) { + logger.error(marker, "A folyamatkezelő réteg nem elérhető."); + throw new NullPointerException("Internal error, missing JobEngine reference."); + } + manager = jobEngine.getItemManager(); + if (manager == null) { + logger.error(marker, "Az adatbáziskezelő réteg nem elérhető."); + throw new NullPointerException("Internal error, missing ItemManager reference."); + } + if (archivedMedia == null) { + logger.error(marker, "A folyamat 'mediaCubeMedia' bemeneti paramétere üres."); + throw new NullPointerException("System is not configured properly, 'mediaCubeMedia' input parameter missing."); + } + checkTargetPath(targetPath); + if (StringUtils.isBlank(houseId)) { + logger.error(marker, "A folyamat 'houseId' bemeneti paramétere üres."); + throw new NullPointerException("System is not configured properly, 'houseId' input parameter missing."); + } + Store tsmStore = manager.getSystemStore(false); + if (tsmStore == null) { + logger.error(marker, "A TSM rendszer beállítás nem elérhető."); + throw new NullPointerException("System is not configured properly, missing TSM Store."); + } + + tempSourceUri = manager.createStoreUri(RemoteStoreProtocol.LOCAL, targetPath); + if (tempSourceUri == null) { + logger.error(marker, "A TSM rendszer beállítás paraméterei nem elérhetőek."); + throw new NullPointerException("System is not configured properly, missing TSM StoreUri."); + } + tempTargetUri = createTargetUri(manager, targetPath); + // sourceFileName = getSourceFileName(archivedMedia, tsmStore); + if (sourceFileName == null) { + logger.error(marker, "Adatbázis bejegyzés hiba, a visszatöltendő fájl neve nem található."); + throw new NullPointerException("Database error, missing MediaFile 'relativePath'."); + } + } +} diff --git a/server/-product/production/OMAR/jobs/steps/MediaToolStep.java b/server/-product/production/OMAR/jobs/steps/MediaToolStep.java new file mode 100644 index 00000000..ecc0ba2d --- /dev/null +++ b/server/-product/production/OMAR/jobs/steps/MediaToolStep.java @@ -0,0 +1,29 @@ +package user.jobengine.server.steps; + +import java.nio.file.Path; +import java.nio.file.Paths; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + +import user.commons.mediaarea.MediaArea; +import user.jobengine.db.Media; + +public class MediaToolStep extends JobStep { + private static final Logger logger = LogManager.getLogger(); + + @StepEntry + public Object[] execute(ArchiveItem archiveItem, Media mediaCubeMedia) throws Exception { + Path filePath = Paths.get(archiveItem.getMediaFile()); + MediaArea ma = new MediaArea(filePath); + ma.process(); + long frames = ma.getFrameCount(); + if (frames > 0) { + logger.info("Media {} length is {}", filePath, frames); + mediaCubeMedia.setLength(frames); + getManager().modify(mediaCubeMedia); + } + return null; + } + +} diff --git a/server/-product/production/OMAR/jobs/steps/MetadataTransformStep.java b/server/-product/production/OMAR/jobs/steps/MetadataTransformStep.java new file mode 100644 index 00000000..c7bfb3ca --- /dev/null +++ b/server/-product/production/OMAR/jobs/steps/MetadataTransformStep.java @@ -0,0 +1,168 @@ +package user.jobengine.server.steps; + +import java.io.File; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.nio.file.attribute.FileAttribute; +import java.nio.file.attribute.PosixFilePermission; +import java.nio.file.attribute.PosixFilePermissions; +import java.util.Date; +import java.util.Set; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.apache.logging.log4j.Marker; + +import com.ibm.nosql.json.api.BasicDBList; + +import user.jobengine.db.Item; +import user.jobengine.db.ItemManager; +import user.jobengine.db.ItemType; +import user.jobengine.db.Media; +import user.jobengine.server.IJobEngine; +import user.jobengine.server.IJobRuntime; + +/** + * Itemek es mediak krealasa az ArchiveItem objektum alapjan. + * + * @author robi + */ +public class MetadataTransformStep extends JobStep { + private static final String CONFLICT = ".CONFLICT"; + private static final Logger logger = LogManager.getLogger(); + private static final String ITEM_MANAGER_IS_NULL = "ItemManager is null"; + public static final String DEFAULT_MEDIATYPE = "Generic"; + private ItemManager itemManager; + + private Marker marker;; + + private void addTags(ArchiveItem archiveItem, Media mediaCubeMedia) { + BasicDBList tags = archiveItem.getTags(); + if (tags != null) { + for (Object tag : tags) { + + try { + String tagText = String.valueOf(tag); + itemManager.addMediaTag(tagText, mediaCubeMedia.getId()); + System.out.println(); + + } catch (Exception e) { + logger.catching(e); + } + } + } + } + + private void checkDuplicates(ArchiveItem archiveItem, String sourceFileName) throws Exception { + if (itemManager.isMediaFileExists(sourceFileName)) { + try { + Path sourcePath = Paths.get(archiveItem.getMediaFile()); + Path parent = sourcePath.getParent(); + Path conflictPath = Paths.get(parent.toString(), CONFLICT); + File folder = conflictPath.toFile(); + if (!folder.exists() || !folder.isDirectory()) { + Set perms = PosixFilePermissions.fromString("rwxrwxrwx"); + FileAttribute> attr = PosixFilePermissions.asFileAttribute(perms); + try { + Files.createDirectories(conflictPath, attr); + } catch (Exception e) { + try { + Files.createDirectory(conflictPath); + } catch (Exception e1) { + logger.catching(e); + throw e; + } + } + } + + Files.move(sourcePath, Paths.get(conflictPath.toString(), sourceFileName + (new Date()).getTime())); + } catch (Exception e1) { + logger.catching(e1); + logger.error(marker, "Hiba az '{}' állomány mappába másolásakor. A rendszer üzenete: {}", CONFLICT, e1.getMessage()); + } + throw new Exception("Az '" + sourceFileName + "' állomány már megtalálható az archívumban, archiválása nem lehetséges."); + } + } + + private Item createItem(ArchiveItem archiveItem) { + Item mediaCubeItem = getExistingItem(archiveItem.getItemHouseId(), archiveItem.getItemTitle()); + if (mediaCubeItem == null) + mediaCubeItem = itemManager.createItem(DEFAULT_MEDIATYPE, archiveItem.getItemTitle(), archiveItem.getItemDescription(), + archiveItem.getItemHouseId()); + return mediaCubeItem; + } + + private Media createMedia(ArchiveItem archiveItem, Item mediaCubeItem, String mediaType) { + Media mediaCubeMedia; + mediaCubeMedia = itemManager.createMedia(mediaType, archiveItem.getMediaTitle(), archiveItem.getMediaDescription(), archiveItem.getMediaHouseId()); + mediaCubeMedia.setLength(archiveItem.getDuration()); + mediaCubeItem.appendMedia(mediaCubeMedia); + + return mediaCubeMedia; + } + + @StepEntry + public Object[] execute(ArchiveItem archiveItem, IJobEngine jobEngine, IJobRuntime jobRuntime) throws Exception { + marker = jobRuntime.getSessionMarker(); + Media mediaCubeMedia = null; + itemManager = (ItemManager) jobEngine.getItemManager(); + if (itemManager == null) + throw new NullPointerException(ITEM_MANAGER_IS_NULL); + try { + File sourceMediaFile = new File(archiveItem.getMediaFile()); + String sourceFileName = sourceMediaFile.getName(); + checkDuplicates(archiveItem, sourceFileName); + Item mediaCubeItem = createItem(archiveItem); + jobRuntime.incrementProgress(50); + String mediaType = getCreateType(archiveItem); + mediaCubeMedia = createMedia(archiveItem, mediaCubeItem, mediaType); + //ha itemid 0 akkor merge, egyebkent media insert + + if (mediaCubeItem.getId() == 0) + itemManager.mergeItemStructure(mediaCubeItem); + else { + mediaCubeMedia.setItemId(mediaCubeItem.getId()); + mediaCubeMedia.add(); + } + + addTags(archiveItem, mediaCubeMedia); + + } catch (Exception e) { + logger.catching(e); + String fileName = new File(archiveItem.getMediaFile()).getName(); + logger.error(marker, "Az '{}' állomány nem archiválható, mert a metaadat transzformáció sikertelen. A rendszer üzenete: {}", fileName, + e.getMessage()); + if (!archiveItem.removeCatchedFile()) + logger.error(marker, "Az '{}' állomány .catched jelző állománya nem törölhető.", fileName); + throw e; + } finally { + jobRuntime.incrementProgress(100); + } + return new Object[] { mediaCubeMedia }; + } + + private String getCreateType(ArchiveItem archiveItem) { + String mediaType = archiveItem.getMediaType(); + if (mediaType == null || mediaType.length() == 0) + mediaType = DEFAULT_MEDIATYPE; + else { + ItemType mediaItemType = itemManager.getItemType(mediaType); + if (mediaItemType == null) + itemManager.createItemType(mediaType, mediaType).add(); + } + return mediaType; + } + + private Item getExistingItem(String itemHouseId, String itemTitle) { + Item[] result = new Item[] { null }; + String sql = String.format("select id from item where houseid='%s' and title='%s'", itemHouseId, itemTitle); + itemManager.executeQuery(sql, rs -> { + long id = rs.getLong("id"); + result[0] = itemManager.getItem(id); + return true; + }, null); + return result[0]; + } + +} diff --git a/server/-product/production/OMAR/jobs/steps/MetadataUpdater.java b/server/-product/production/OMAR/jobs/steps/MetadataUpdater.java new file mode 100644 index 00000000..a28abfa2 --- /dev/null +++ b/server/-product/production/OMAR/jobs/steps/MetadataUpdater.java @@ -0,0 +1,297 @@ +package user.jobengine.server.steps; + +import java.io.IOException; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.sql.ResultSet; +import java.text.SimpleDateFormat; +import java.util.Date; +import java.util.LinkedHashSet; +import java.util.List; +import java.util.Set; + +import org.apache.commons.io.FileUtils; +import org.apache.ibatis.jdbc.SQL; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.apache.logging.log4j.Marker; +import org.apache.logging.log4j.MarkerManager; + +import com.ibm.nosql.json.api.BasicDBObject; + +import user.jobengine.db.IItemManager; +import user.jobengine.server.steps.shared.MetadataTypeDetector; +import user.mediacube.metadata.interfaces.IMetadata; +import user.mediacube.metadata.interfaces.IMetadataProvider; +import user.mediacube.metadata.interfaces.IMetadataProviderFactory; +import user.mediacube.metadata.interfaces.MetadataProviderType; +import user.mediacube.metadata.interfaces.MetadataType; +import user.mediacube.metadata.interfaces.PlanAirMetadataListOptions; + +public class MetadataUpdater extends JobStep { + private static final String EMPTY = ""; + private static final String DOT = "."; + private static final Logger logger = LogManager.getLogger(MetadataUpdater.class); + private Marker csvMarker = MarkerManager.getMarker("METADATA-UPDATER-CSV"); + + private static final String ITEM_TITLE = "itemTitle"; + private static final String ITEM_HOUSEID = "itemHouseId"; + private static final String MEDIA_HOUSEID = "mediaHouseId"; + private static final String MEDIA_TITLE = "mediaTitle"; + private static final String MEDIA_DESCRIPTION = "mediaDescription"; + private static final String MEDIA_TYPE = "mediaType"; + private static final String MEDIAFILE_HOUSEID = "mediaFileHouseId"; + + private SimpleDateFormat df = new SimpleDateFormat("yyyy.MM.dd HH:mm:ss"); + private Set includeList; + + public String createQuery() { + SQL isql = new SQL(); + isql.SELECT("id"); + isql.FROM("STORE"); + isql.WHERE("name = 'TSM'"); + + SQL sql = new SQL(); + sql.SELECT("i.id AS itemId"); + sql.SELECT("i.houseid AS itemHouseId"); + sql.SELECT("i.title AS itemTitle"); + //sql.SELECT("i.description AS itemdescription"); + sql.SELECT("m.id AS mediaId"); + sql.SELECT("m.houseid AS mediaHouseId"); + sql.SELECT("m.title AS mediaTitle"); + sql.SELECT("m.description AS mediaDescription"); + sql.SELECT("mf.houseid AS mediaFileHouseId"); + sql.FROM("MEDIAFILE mf"); + sql.LEFT_OUTER_JOIN("MEDIA m ON (m.id = mf.mediaid)"); + sql.LEFT_OUTER_JOIN("ITEM i ON (i.id = m.itemid)"); + sql.WHERE(String.format("mf.storeid = (%s)", isql.toString())); + return sql.toString(); + } + + public String createCountQuery() { + SQL sql = new SQL(); + sql.SELECT("COUNT(*) as count"); + sql.FROM("MEDIA"); + return sql.toString(); + } + + @StepEntry + public Object[] execute() throws Exception { + try { + + String location = "/opt/test-mediacube/file_list_original.txt"; + includeList = loadIncludeList(location); + + IItemManager manager = getManager(); + String sql = createCountQuery(); + long[] count = new long[1]; + count[0] = 0; + manager.executeQuery(sql, rs -> { + try { + count[0] = rs.getLong("count"); + logger.info(getSessionMarker(), "Processing rs"); + } catch (Exception e) { + logger.error(getMarker(), e.getMessage()); + } + return true; + }, null); + + logger.info(getSessionMarker(), "Count {}", count[0]); + + long[] current = new long[1]; + current[0] = 0; + + logger.info(csvMarker, + "Date;Name;isProgramById;includeContains;isMetadataEquals;itemHouseId;P itemHouseId;itemHouseIdEquals;itemTitle;P itemTitle;itemTitleEquals;mediaHouseId;" + + "P mediaHouseId;mediaHouseIdEquals;mediaTitle;P mediaTitle;mediaTitleEquals;" + + "mediaDescription;P mediaDescription;mediaDescriptionEquals;"); + + sql = createQuery(); + manager.executeQuery(sql, rs -> { + + if (getJobRuntime().isWaitingCancel()) + return false; + + current[0]++; + processRecord(rs); + int p = ((int) current[0] * 100) / ((int) count[0]); + setProgress(p); + return true; + }, null); + + } catch (Exception e) { + logger.error(getSessionMarker(), e.getMessage()); + } + return null; + } + + private String set(String value) { + return value == null ? EMPTY : value; + } + + private void processRecord(ResultSet rs) { + try { + String itemHouseId = set(rs.getString(ITEM_HOUSEID)); + String itemTitle = set(rs.getString(ITEM_TITLE)); + String mediaHouseId = set(rs.getString(MEDIA_HOUSEID)); + String mediaTitle = set(rs.getString(MEDIA_TITLE)); + String mediaDescription = set(rs.getString(MEDIA_DESCRIPTION)); + String pitemHouseId = EMPTY; + String pitemTitle = EMPTY; + String pmediaHouseId = EMPTY; + String pmediaTitle = EMPTY; + String pmediaDescription = EMPTY; + + String fileName = rs.getString(MEDIAFILE_HOUSEID); + String mediaFileHouseId = fileName; + if (mediaFileHouseId.contains(DOT)) + mediaFileHouseId = mediaFileHouseId.substring(0, mediaFileHouseId.lastIndexOf(DOT)); + + user.jobengine.server.steps.shared.MetadataType metadataType = MetadataTypeDetector + .GuessMetadataType(mediaFileHouseId); + + boolean isProgramById = false; + boolean includeContains = includeList.contains(fileName); + boolean itemHouseIdEquals = false; + boolean itemTitleEquals = false; + boolean mediaHouseIdEquals = false; + boolean mediaTitleEquals = false; + boolean mediaDescriptionEquals = false; + + logger.info(getMarker(), "{} {}", mediaFileHouseId, metadataType); + ArchiveItem archiveItem = getPlanAirMetadata(mediaFileHouseId); + if (archiveItem != null) { + isProgramById = true; + + if (isProgramById) { + + pitemHouseId = set(archiveItem.getItemHouseId()); + itemHouseIdEquals = pitemHouseId.equals(itemHouseId); + + pitemTitle = set(archiveItem.getItemTitle()); + itemTitleEquals = pitemTitle.equals(itemTitle); + + pmediaHouseId = set(archiveItem.getMediaHouseId()); + mediaHouseIdEquals = pmediaHouseId.equals(mediaHouseId); + + pmediaTitle = set(archiveItem.getMediaTitle()); + mediaTitleEquals = pmediaTitle.equals(mediaTitle); + + pmediaDescription = set(archiveItem.getMediaDescription()); + mediaDescriptionEquals = pmediaDescription.equals(mediaDescription); + + boolean isMetadataEquals = itemHouseIdEquals && itemTitleEquals && mediaHouseIdEquals + && mediaTitleEquals && mediaDescriptionEquals; + + Date now = new Date(System.currentTimeMillis()); + logger.info(csvMarker, "{};{};{};{};{};{};{};{};{};{};{};{};{};{};{};{};{};{};{};{};", D(now), + fileName, YN(isProgramById), YN(includeContains), YN(isMetadataEquals), itemHouseId, + pitemHouseId, itemHouseIdEquals, itemTitle, pitemTitle, itemTitleEquals, mediaHouseId, + pmediaHouseId, mediaHouseIdEquals, mediaTitle, pmediaTitle, mediaTitleEquals, + mediaDescription, pmediaDescription, mediaDescriptionEquals); + + logger.info(getSessionMarker(), "Processed {} {}{}{}", fileName, YN(isProgramById), + YN(includeContains), YN(isMetadataEquals)); + } + } + + } catch ( + + Exception e) { + logger.error(getSessionMarker(), e.getMessage()); + } + + } + + private ArchiveItem getPlanAirMetadata(String mediaHouseId) throws Exception { + PlanAirMetadataListOptions opt = new PlanAirMetadataListOptions(); + opt.setSearch(mediaHouseId); + opt.setType(MetadataType.Material); + + BasicDBObject json = null; + List data = null; + + IMetadataProviderFactory factory = getService(IMetadataProviderFactory.class); + if (factory == null) + logger.info(getSessionMarker(), "IMetadataProviderFactory is null"); + + IMetadataProvider planairProvider = factory.getProvider(MetadataProviderType.PLANAIR); + if (planairProvider == null) + logger.info(getSessionMarker(), "IMetadataProvider is null"); + + ArchiveItem result = null; + data = planairProvider.list(opt); + if (data.size() != 0) + json = data.get(0).asJSON(); + else { + opt.setType(MetadataType.Promo); + data = planairProvider.list(opt); + if (data.size() != 0) + json = data.get(0).asJSON(); + else { + opt.setType(MetadataType.AD); + data = planairProvider.list(opt); + if (data.size() != 0) + json = data.get(0).asJSON(); + } + } + if (json != null) { + result = new ArchiveItem(); + result.setItemHouseId(json.getString(ITEM_HOUSEID)); + result.setItemTitle(json.getString(ITEM_TITLE)); + result.setMediaHouseId(json.getString(MEDIA_HOUSEID)); + result.setMediaTitle(json.getString(MEDIA_TITLE)); + result.setMediaDescription(json.getString(MEDIA_DESCRIPTION)); + result.setMediaType(json.getString(MEDIA_TYPE)); + } + return result; + } + + private char YN(boolean value) { + return value ? 'Y' : 'N'; + } + + private String D(Date value) { + return value == null ? EMPTY : df.format(value); + } + + private Set loadIncludeList(String location) throws IOException { + logger.info(getSessionMarker(), "Loading include list {}", location); + Set result = new LinkedHashSet<>(); + + Path path = Paths.get(location); + List lines = FileUtils.readLines(path.toFile()); + + String lastDir = null; + for (String line : lines) { + line = line.trim(); + if (line.startsWith("Directory of")) { + lastDir = line; + lastDir = lastDir.replace("Directory of", EMPTY); + lastDir = lastDir.replace("X:", EMPTY); + lastDir = lastDir.replace("\\", "/"); + lastDir = lastDir.trim(); + } + + if (lastDir != null && lastDir.endsWith(".STATUS")) + continue; + + if (line.startsWith("2") && line.length() > 39) { + String file = line.substring(39).trim(); + + if (file.equals(DOT) || file.equals("..")) + continue; + + String len = line.substring(21, 39).trim(); + if (len.length() != 0) { + String fullpath = "/mnt/POLC/FINISHED_SHOWS" + lastDir + "/" + file; + result.add(Paths.get(fullpath).getFileName().toString()); + } + } + + } + + return result; + } + +} diff --git a/server/-product/production/OMAR/jobs/steps/MoveJpegToIsilonStep.java b/server/-product/production/OMAR/jobs/steps/MoveJpegToIsilonStep.java new file mode 100644 index 00000000..3dc75b55 --- /dev/null +++ b/server/-product/production/OMAR/jobs/steps/MoveJpegToIsilonStep.java @@ -0,0 +1,165 @@ +package user.jobengine.server.steps; + +import java.io.File; +import java.io.IOException; +import java.nio.file.FileVisitResult; +import java.nio.file.FileVisitor; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.nio.file.SimpleFileVisitor; +import java.nio.file.StandardCopyOption; +import java.nio.file.attribute.BasicFileAttributes; +import java.text.ParseException; +import java.text.SimpleDateFormat; +import java.util.ArrayList; +import java.util.Date; +import java.util.List; + +import org.apache.commons.io.FilenameUtils; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + +import user.commons.StoreUri; +import user.commons.remotestore.RemoteStoreProtocol; + +public class MoveJpegToIsilonStep extends JobStep { + private static final Logger logger = LogManager.getLogger(); + + @StepEntry + public Object[] execute(String sourceUri, String sourceProtocol, String sourceFolder, String targetUri, + String targetProtocol, String targetFolder, String userName, String password, int port) throws Exception { + StoreUri sourceStoreUri = new StoreUri(sourceUri); + if (sourceFolder.endsWith("/")) { + sourceFolder = sourceFolder.substring(0, sourceFolder.length() - 1); + } + + List foundItems = new ArrayList<>(); + sourceStoreUri.setRootPath(sourceFolder); + sourceStoreUri.setPortNumber(port); + sourceStoreUri.setPassword(password); + sourceStoreUri.setUserName(userName); + sourceStoreUri.setProtocol(RemoteStoreProtocol.valueOf(sourceProtocol)); + + FileVisitor visitor = new SimpleFileVisitor() { + String fileNamePattern = "yyyymmdd"; + SimpleDateFormat dateFormatter = new SimpleDateFormat(fileNamePattern); + Date currentDate = new Date(); // initializes with the current date + Date dateFromFileName; + boolean isRootFolder = true; + + @Override + public FileVisitResult preVisitDirectory(Path folder, BasicFileAttributes attrs) throws IOException { + FileVisitResult result = null; + boolean isDateParseable = true; + + if (folder != null) { + String splitter = File.separator.replace("\\", "\\\\"); + String[] parentFolderSegments = folder.toString().split(splitter); + + try { + dateFromFileName = dateFormatter.parse(parentFolderSegments[parentFolderSegments.length - 1]); + + if (!(dateFromFileName.compareTo(currentDate) > 0)) { + if (isRootFolder) { + result = FileVisitResult.CONTINUE; + isRootFolder = false; + } else { + result = FileVisitResult.SKIP_SUBTREE; + } + } else { + result = FileVisitResult.CONTINUE; + } + } catch (ParseException e) { + isDateParseable = false; + logger.info("Illegal argument to parse as date: {}", + parentFolderSegments[parentFolderSegments.length - 1]); + } finally { + isRootFolder = false; + if (!isDateParseable) { + result = FileVisitResult.CONTINUE; + } + } + } + return result; + } + + @Override + public FileVisitResult visitFile(Path path, BasicFileAttributes attrs) throws IOException { + logger.info("path: {}", path.toString()); + try { + String fileExtension = FilenameUtils.getExtension(path.toString()); + + if (fileExtension.equals("jpg") || fileExtension.equals("jpeg")) { + foundItems.add(path.toString()); + StoreUri targetStoreUri = getManager() + .createStoreUri(RemoteStoreProtocol.valueOf(targetProtocol), targetUri); + targetStoreUri.setProtocol(RemoteStoreProtocol.valueOf(targetProtocol)); + targetStoreUri.setRootPath(targetFolder); + + copyFile(path, Paths.get(targetStoreUri.getRootPath())); + } + } catch (Exception e) { + logger.info("Exception: {}", e.getStackTrace()); + } + return FileVisitResult.CONTINUE; + } + + @Override + public FileVisitResult visitFileFailed(Path path, IOException exc) throws IOException { + return FileVisitResult.CONTINUE; + } + }; + + try { + Files.walkFileTree(Paths.get(sourceStoreUri.getRootPath()), visitor); + } catch (IOException e) { + logger.info("Error processing Paths.get(sourceStoreUri.getRootPath()) '{}'. System message: {}", + Paths.get(sourceStoreUri.getRootPath()), e.getStackTrace()); + logger.catching(e); + throw e; + } catch (SecurityException se) { + logger.info("SecurityException: {}", se.getStackTrace()); + } finally { + } + + return new Object[] { foundItems }; + } + + private void copyFile(Path sourceFullPath, Path targetRootPath) { + Path targetPath = Paths.get(targetRootPath.toString(), sourceFullPath.getFileName().toString()); + logger.info(getMarker(), "Root {} exists {}", targetRootPath, Files.exists(targetRootPath)); + + if (Files.exists(targetPath)) { + logger.info(getMarker(), "Skipping {}, target exists", targetPath); + return; + } + + logger.info(getMarker(), "Target {} synchronization required", targetPath); + + try { + Files.copy(sourceFullPath, targetPath, StandardCopyOption.REPLACE_EXISTING); + } catch (Exception e) { + logger.error(getMarker(), "Error synchronize {} to {}. System message: {}", sourceFullPath, targetPath, + e.getMessage()); + } + } + + protected String getPathUntilCurrentFile(String rootFolder, Path fullPath) { + String tempRootfolder = ""; + if (rootFolder.contains("/") && fullPath.toString().contains("\\")) { + tempRootfolder = rootFolder.replace('/', '\\'); + if (!tempRootfolder.endsWith("\\")) { + tempRootfolder = tempRootfolder.concat("\\"); + } + } + if (rootFolder.contains("\\") && fullPath.toString().contains("/")) { + tempRootfolder = rootFolder.replace('\\', '/'); + if (!tempRootfolder.endsWith("/")) { + tempRootfolder = tempRootfolder.concat("/"); + } + } + + return fullPath.toString().replace(rootFolder, "").substring(1); + } +} diff --git a/server/-product/production/OMAR/jobs/steps/OutputPathAndNameSelectorStep.java b/server/-product/production/OMAR/jobs/steps/OutputPathAndNameSelectorStep.java new file mode 100644 index 00000000..f0203ca5 --- /dev/null +++ b/server/-product/production/OMAR/jobs/steps/OutputPathAndNameSelectorStep.java @@ -0,0 +1,154 @@ +package user.jobengine.server.steps; + +import java.io.IOException; +import java.nio.file.FileVisitResult; +import java.nio.file.FileVisitor; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.nio.file.SimpleFileVisitor; +import java.nio.file.attribute.BasicFileAttributes; + +import org.apache.commons.lang.StringUtils; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.apache.logging.log4j.Marker; + +import user.jobengine.db.ArchivedMedia; +import user.jobengine.server.IJobEngine; +import user.jobengine.server.IJobRuntime; +import user.jobengine.server.steps.shared.EscortFiles; +import user.jobengine.server.steps.shared.MetadataType; +import user.jobengine.server.steps.shared.MetadataTypeDetector; + +public class OutputPathAndNameSelectorStep extends JobStep { + + private static final String TARGETNAMEPATTERN = "-ARCH-%s"; + + private static final Logger logger = LogManager.getLogger(); + + private Marker marker; + + private void check(String localRetrievePath, String materialOutputFolder, String promoOutputFolder, String advertisementOutputFolder, + String octopusOutputFolder, String genericOutputFolder, String houseId, String targetPathType) { + if (StringUtils.isBlank(localRetrievePath)) { + logger.error(marker, "A folyamat 'localRetrievePath' bemeneti paramétere üres."); + throw new NullPointerException("System is not configured properly, 'localRetrievePath' input parameter missing."); + } + if (StringUtils.isBlank(materialOutputFolder)) { + logger.error(marker, "A folyamat 'materialOutputFolder' bemeneti paramétere üres."); + throw new NullPointerException("System is not configured properly, 'materialOutputFolder' input parameter missing."); + } + if (StringUtils.isBlank(promoOutputFolder)) { + logger.error(marker, "A folyamat 'promoOutputFolder' bemeneti paramétere üres."); + throw new NullPointerException("System is not configured properly, 'promoOutputFolder' input parameter missing."); + } + if (StringUtils.isBlank(advertisementOutputFolder)) { + logger.error(marker, "A folyamat 'advertisementOutputFolder' bemeneti paramétere üres."); + throw new NullPointerException("System is not configured properly, 'advertisementOutputFolder' input parameter missing."); + } + if (StringUtils.isBlank(octopusOutputFolder)) { + logger.error(marker, "A folyamat 'octopusOutputFolder' bemeneti paramétere üres."); + throw new NullPointerException("System is not configured properly, 'octopusOutputFolder' input parameter missing."); + } + if (StringUtils.isBlank(genericOutputFolder)) { + logger.error(marker, "A folyamat 'genericOutputFolder' bemeneti paramétere üres."); + throw new NullPointerException("System is not configured properly, 'genericOutputFolder' input parameter missing."); + } + if (StringUtils.isBlank(houseId)) { + logger.error(marker, "A folyamat 'houseId' bemeneti paramétere üres."); + throw new NullPointerException("System is not configured properly, 'houseId' input parameter missing."); + } + if (StringUtils.isBlank(targetPathType)) { + logger.error(marker, "A folyamat 'targetPathType' bemeneti paramétere üres."); + throw new NullPointerException("System is not configured properly, 'targetPathType' input parameter missing."); + } + } + + @StepEntry + public Object[] execute(String localRetrievePath, String materialOutputFolder, String promoOutputFolder, String advertisementOutputFolder, + String octopusOutputFolder, String genericOutputFolder, String onlineOutputFolder, String houseId, String targetPathType, + ArchivedMedia archivedMedia, IJobEngine jobEngine, IJobRuntime jobRuntime) throws Exception { + marker = jobRuntime.getSessionMarker(); + check(localRetrievePath, materialOutputFolder, promoOutputFolder, advertisementOutputFolder, octopusOutputFolder, genericOutputFolder, houseId, + targetPathType); + Object[] result = null; + switch (Integer.parseInt(targetPathType)) { + case 0: + String outputFolder = getFolderById(materialOutputFolder, promoOutputFolder, advertisementOutputFolder, octopusOutputFolder, genericOutputFolder, + houseId, archivedMedia); + result = localTargetInit(localRetrievePath, outputFolder, houseId, jobRuntime); + break; + case 1: + result = localTargetInit(localRetrievePath, onlineOutputFolder, houseId, jobRuntime); + break; + case 2: + + if (archivedMedia.getTcIn() != null && archivedMedia.getTcOut() != null) + result = new Object[] { genericOutputFolder, houseId, true }; + else + result = new Object[] { null, houseId, true }; + break; + } + return result; + } + + private String getFolderById(String materialOutputFolder, String promoOutputFolder, String advertisementOutputFolder, String octopusOutputFolder, + String genericOutputFolder, String houseId, ArchivedMedia archivedMedia) throws Exception { + String id = houseId.toUpperCase(); + MetadataType mdType = MetadataTypeDetector.GuessMetadataType(id); + String result = null; + + //a groovy nem latja enumnak, hanem az objektum tulajdonsaganak + switch (mdType.toString()) { + case "OctopusPlaceholder": + case "OctopusStory": + result = octopusOutputFolder; + break; + case "TrafficMaterial": + result = materialOutputFolder; + break; + case "TrafficPromo": + result = promoOutputFolder; + break; + case "TrafficAD": + result = advertisementOutputFolder; + break; + case "Generic": + result = genericOutputFolder; + break; + } + return result; + } + + private String getPossiblePath(String id, Path targetPath) throws IOException { + String[] result = new String[] { targetPath.toString() }; + FileVisitor matcherVisitor = new SimpleFileVisitor() { + @Override + public FileVisitResult preVisitDirectory(Path dir, BasicFileAttributes attrs) throws IOException { + String dirName = dir.getFileName().toString(); + if (dirName.startsWith(id + "-") || dirName.equals(id)) { + result[0] = dir.toString(); + return FileVisitResult.TERMINATE; + } + return FileVisitResult.CONTINUE; + } + }; + Files.walkFileTree(targetPath.getParent(), matcherVisitor); + return result[0]; + } + + private Object[] localTargetInit(String localRetrievePath, String outputFolder, String houseId, IJobRuntime jobRuntime) throws IOException { + String id = houseId.toUpperCase(); + String targetPath = getPossiblePath(id, Paths.get(localRetrievePath, outputFolder, id)).toString(); + String targetNamePattern = houseId + TARGETNAMEPATTERN; + try { + EscortFiles.ensureUNCFolder(Paths.get(targetPath)); + } catch (Exception e) { + logger.error(jobRuntime.getSessionMarker(), "A cél mappa '{}' nem létezik és nem hozható létre. A rendszer hibaüzenete: {}", targetPath, + e.getMessage()); + throw e; + } + return new Object[] { targetPath, targetNamePattern, false }; + } +} diff --git a/server/-product/production/OMAR/jobs/steps/PrepareMediaRestoreStep.java b/server/-product/production/OMAR/jobs/steps/PrepareMediaRestoreStep.java new file mode 100644 index 00000000..3708aa3e --- /dev/null +++ b/server/-product/production/OMAR/jobs/steps/PrepareMediaRestoreStep.java @@ -0,0 +1,37 @@ +package user.jobengine.server.steps; + +import java.util.List; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + +import user.commons.StoreUri; +import user.commons.remotestore.RemoteStoreProtocol; +import user.jobengine.db.Media; +import user.jobengine.db.MediaFile; +import user.jobengine.db.Store; + +public class PrepareMediaRestoreStep extends JobStep { + private static final Logger logger = LogManager.getLogger(); + + @StepEntry + public Object[] execute(Media media, String tempStoreName, String tempStoreProtocol) throws Exception { + StoreUri mediaStoreUri = null; + List mediaFiles = media.getMediaFiles(); + if (mediaFiles.size() > 1) + throw new Exception("Media " + media.getId() + " already has proxy"); + for (MediaFile mediaFile : mediaFiles) { + Store store = mediaFile.getStore(); + if (store.isSystem() && !store.isLowres()) { + mediaStoreUri = store.getStoreUri(RemoteStoreProtocol.TSM); + if (mediaStoreUri != null) + break; + } + } + + StoreUri tempStoreUri = getManager().getStoreUri(tempStoreName, Enum.valueOf(RemoteStoreProtocol.class, tempStoreProtocol)); + + return new Object[] { mediaStoreUri, tempStoreUri, media.getMediaFileRealName() }; + } + +} diff --git a/server/-product/production/OMAR/jobs/steps/PrepareRemoteTranscodeStep.java b/server/-product/production/OMAR/jobs/steps/PrepareRemoteTranscodeStep.java new file mode 100644 index 00000000..5116b1ef --- /dev/null +++ b/server/-product/production/OMAR/jobs/steps/PrepareRemoteTranscodeStep.java @@ -0,0 +1,48 @@ +package user.jobengine.server.steps; + +import java.nio.file.Path; +import java.nio.file.Paths; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + +import com.ibm.nosql.json.api.BasicDBObject; + +import user.commons.configuration.SystemConfiguration; + +public class PrepareRemoteTranscodeStep extends JobStep { + static private final Logger logger = LogManager.getLogger(); + private static boolean RANDOMIZE_ARCHIVES = SystemConfiguration.getInstance().value("tsm.randomize-archives", + false); + + @StepEntry + public Object[] execute(String profileName, String fileName) throws Exception { + String hiResRoot = "m:/"; + String lowResRoot = "m:/lowres"; + + Path inputPath = Paths.get(hiResRoot, fileName); + + String realFileName = fileName; + + if (RANDOMIZE_ARCHIVES) + realFileName = realFileName.substring(9); + + String outFileName = realFileName.substring(0, realFileName.lastIndexOf(".")) + ".mp4"; + + Path relativeOutputPath = null; + if (realFileName.length() > 2) + relativeOutputPath = Paths.get(realFileName.substring(0, 1), realFileName.substring(1, 2), + realFileName.substring(2, 3), outFileName); + else + relativeOutputPath = Paths.get("0", outFileName); + + BasicDBObject parameters = new BasicDBObject(); + parameters.put("profile", profileName); + parameters.put("input", inputPath.toString()); + parameters.put("output", Paths.get(lowResRoot, relativeOutputPath.toString()).toString()); + + logger.info("Prepared for remote transode {}", parameters); + return new Object[] { parameters, relativeOutputPath.toString() }; + } + +} diff --git a/server/-product/production/OMAR/jobs/steps/QueryMissingProxyMediaStep.java b/server/-product/production/OMAR/jobs/steps/QueryMissingProxyMediaStep.java new file mode 100644 index 00000000..7df2bed0 --- /dev/null +++ b/server/-product/production/OMAR/jobs/steps/QueryMissingProxyMediaStep.java @@ -0,0 +1,74 @@ +package user.jobengine.server.steps; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + +import com.ibm.nosql.json.api.BasicDBObject; +import com.ibm.nosql.json.api.DB; +import com.ibm.nosql.json.api.DBCollection; + +import user.commons.log4j2.marker.MediaCubeUndoMarker; +import user.commons.nosql.NoSQLUtils; +import user.jobengine.db.IItemManager; +import user.jobengine.db.Media; +import user.jobengine.db.MediaFile; + +public class QueryMissingProxyMediaStep extends JobStep { + private static final Logger logger = LogManager.getLogger(); + + @StepEntry + public Object[] execute() throws Exception { + setDescription("Looking for missing proxy"); + DB db = NoSQLUtils.getNoSQLDB(); + DBCollection collection = db.getCollection("missing_lowres"); + Media media = getFirstUntranscodedMedia(collection); + + if (media == null) { + logger.info(new MediaCubeUndoMarker(getSessionMarker().getSessionID()), "Nincs feldolgozandó hiány."); + cancel(); + return null; + } + + String name = media.getMediaFileRealName(); + collection.save(new BasicDBObject("name", name)); + logger.info(getSessionMarker(), "Processing mediaID: {}", media.getId()); + return new Object[] { media }; + } + + private Media getFirstUntranscodedMedia(DBCollection collection) { + Media[] result = new Media[] { null }; + //MV + String query = "SELECT mediaid FROM VW_MISSING_PROXY_IDS WHERE HOUSEID like 'M%' or HOUSEID like 'P%' or HOUSEID like 'R%' ORDER BY modified DESC"; + + //HTV + //String query = "SELECT mediaid FROM VW_MISSING_PROXY_IDS"; + IItemManager manager = getManager(); + manager.executeQuery(query, rs -> { + try { + long mediaId = rs.getLong(1); + Media media = manager.getMedia(mediaId); + //a nevgeneralas miatt az eredeti MediaFilesName nem jo, a pontos nev kell nekunk + String name = media.getMediaFileRealName(); + //logger.info(getSessionMarker(), "Checking {}", name); + long existing = collection.find(new BasicDBObject("name", name)).count(); + if (existing > 0) { + //logger.info(getSessionMarker(), "{} is on missing_lowres list", name); + return true; + } + // 210617 proxy keszites tiltasa + MediaFile mf = manager.getSystemMediaFile(media); + if (mf.isDisableProxy()) { + //logger.info(getSessionMarker(), "Proxy disabled {}", name); + + return true; + } + + result[0] = media; + } catch (Exception e) { + logger.error(e); + } + return false; + }, null); + return result[0]; + } +} diff --git a/server/-product/production/OMAR/jobs/steps/RemoteJobStep.java b/server/-product/production/OMAR/jobs/steps/RemoteJobStep.java new file mode 100644 index 00000000..d4900d76 --- /dev/null +++ b/server/-product/production/OMAR/jobs/steps/RemoteJobStep.java @@ -0,0 +1,36 @@ +package user.jobengine.server.steps; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + +import com.ibm.nosql.json.api.BasicDBObject; + +import user.jobengine.server.steps.shared.MediaCubeClient; + +public class RemoteJobStep extends JobStep { + private static final Logger logger = LogManager.getLogger(); + + @StepEntry + public Object[] execute(String remoteServer, String template, String name, BasicDBObject remoteJobParameters) throws Exception { + setDescription("Executing on {}, template {}, profile {}", remoteServer, template, remoteJobParameters.getString("profile")); + MediaCubeClient mc = new MediaCubeClient(remoteServer); + long jobId = mc.startjob(template, name, remoteJobParameters); + logger.info(getMarker(), "Started {} on server {}", jobId, remoteServer); + while (true) { + BasicDBObject status = mc.getStatus(jobId); + if (status != null) + setProgress(status.getInt("progress")); + + Thread.sleep(2000); + + String jobStatus = status.getString("status"); + if ("SUSPENDED".equals(jobStatus)) + throw new Exception(status.getString("description")); + + if ("FINISHED".equals(jobStatus)) + break; + } + + return null; + } +} diff --git a/server/-product/production/OMAR/jobs/steps/SafeDeleteRecursiveStep.java b/server/-product/production/OMAR/jobs/steps/SafeDeleteRecursiveStep.java new file mode 100644 index 00000000..238aae23 --- /dev/null +++ b/server/-product/production/OMAR/jobs/steps/SafeDeleteRecursiveStep.java @@ -0,0 +1,416 @@ +package user.jobengine.server.steps; + +import java.io.File; +import java.io.IOException; +import java.nio.file.DirectoryStream; +import java.nio.file.FileVisitResult; +import java.nio.file.FileVisitor; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.nio.file.attribute.BasicFileAttributes; +import java.text.SimpleDateFormat; +import java.util.ArrayList; +import java.util.Arrays; +import java.util.Collections; +import java.util.Date; +import java.util.LinkedHashSet; +import java.util.List; +import java.util.Set; + +import org.apache.commons.io.FileUtils; +import org.apache.commons.io.FilenameUtils; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + +import com.ibm.nosql.json.api.BasicDBObject; + +import user.commons.IEntityBase; +import user.commons.RemoteFile; +import user.commons.StoreUri; +import user.commons.mediatool.MediaInfo; +import user.commons.remotestore.RemoteStoreProtocol; +import user.jobengine.db.Item; +import user.jobengine.db.Media; +import user.jobengine.db.MediaFile; +import user.jobengine.db.MediaFileDAO; +import user.jobengine.db.Store; +import user.jobengine.server.steps.shared.EscortFiles; +import user.mediacube.metadata.interfaces.IMetadata; +import user.mediacube.metadata.interfaces.IMetadataListOptions; +import user.mediacube.metadata.interfaces.IMetadataProvider; +import user.mediacube.metadata.interfaces.IMetadataProviderFactory; +import user.mediacube.metadata.interfaces.MetadataProviderType; +import user.mediacube.metadata.interfaces.MetadataType; +import user.mediacube.metadata.interfaces.PlanAirMetadataListOptions; + +public class SafeDeleteRecursiveStep extends JobStep implements FileVisitor { + private static final Logger logger = LogManager.getLogger(); + private static final String ITEM_TITLE = "itemTitle"; + private static final String ITEM_HOUSEID = "itemHouseId"; + private static final String MEDIA_HOUSEID = "mediaHouseId"; + private static final String MEDIA_TITLE = "mediaTitle"; + private static final String MEDIA_DESCRIPTION = "mediaDescription"; + private static final String MEDIA_TYPE = "mediaType"; + private static final String KILLDATEEXT = ".killdate"; + private SimpleDateFormat df = new SimpleDateFormat("yyyy.MM.dd HH:mm:ss"); + + private List skipPathNames = Arrays.asList("!ARCHIVALAS_ALATT", EscortFiles.STATUSFOLDER, + EscortFiles.CONFLICTFOLDER); + private Set includeList; + + private boolean canReadMediaInfo(Path mediaFilePath) { + boolean result = false; + try { + MediaInfo mi = new MediaInfo(mediaFilePath); + mi.process(); + result = true; + } catch (Exception e) { + logger.warn(getSessionMarker(), e.getMessage()); + } + return result; + } + + private boolean checkArchiveItem(ArchiveItem archiveItem, Item item, Media media) { + if (archiveItem == null) + return false; + + String itemHouseId = archiveItem.getItemHouseId() == null ? "" : archiveItem.getItemHouseId(); + if (!itemHouseId.equals(item.getHouseId())) { + logger.error(getSessionMarker(), "ItemHouseId"); + return false; + } + + String itemTitle = archiveItem.getItemTitle() == null ? "" : archiveItem.getItemTitle(); + if (!itemTitle.equals(item.getTitle())) { + logger.error(getSessionMarker(), "ItemTitle"); + return false; + } + String mediaHouseId = archiveItem.getMediaHouseId() == null ? "" : archiveItem.getMediaHouseId(); + if (!mediaHouseId.equals(media.getHouseId())) { + logger.error(getSessionMarker(), "MediaHouseId"); + return false; + } + String mediaTitle = archiveItem.getMediaTitle() == null ? "" : archiveItem.getMediaTitle(); + if (!mediaTitle.equals(media.getTitle())) { + logger.error(getSessionMarker(), "MediaTitle"); + return false; + } + + String mediaDescription = archiveItem.getMediaDescription() == null ? "" : archiveItem.getMediaDescription(); + String storedMediaDescription = media.getDescription() == null ? "" : media.getDescription(); + if (!mediaDescription.equals(storedMediaDescription)) { + logger.error(getSessionMarker(), "MediaDescription"); + return false; + } + + return true; + } + + private ArchiveItem createArchiveItem(Path filePath) throws Exception { + ArchiveItem result = null; + + String fileName = filePath.getFileName().toString(); + String mediaHouseId = FilenameUtils.removeExtension(fileName); + try { + result = getPlanAirMetadata(mediaHouseId); + + if (result != null) + result.setMediaFile(filePath.toString()); + + } catch (Exception e) { + logger.error(getSessionMarker(), "PlanAir metadata error", e); + } + + return result; + } + + @StepEntry + public Object[] execute(String sourcePath) throws Exception { + logger.info(getSessionMarker(), "Starting in {}", sourcePath); + logger.info(getSessionMarker(), "{};{};{};{};{};{};{};{};{};{};{};{};{};{};{};{};{};{};", "Napló időbélyeg", + "Neve", "Elérése", "Mérete", "Létrehozva", "Módosítva", "Archiválva (MC)", "Archiválva (TSM)", + "Méret (TSM)", ".catched", ".killdate", ".nomd", "Archivált (MC)", "Archivált (TSM)", + "Metaadat egyezés", "Méret egyezés", "Törölhető", "Kezelendő"); + + try { + String location = "/opt/test-mediacube/file_list_original.txt"; + includeList = loadIncludeList(location); + Files.walkFileTree(Paths.get(sourcePath), this); + } catch (Exception e) { + logger.error(getSessionMarker(), "Az '{}' mappa elérése sikertelen. A rendszer hibaüzenete: {}", sourcePath, + e.getMessage()); + } + return null; + } + + private ArchiveItem getPlanAirMetadata(String mediaHouseId) throws Exception { + PlanAirMetadataListOptions opt = new PlanAirMetadataListOptions(); + opt.setSearch(mediaHouseId); + opt.setType(MetadataType.Material); + + BasicDBObject json = null; + List data = null; + + IMetadataProviderFactory factory = getService(IMetadataProviderFactory.class); + if (factory == null) + logger.info(getSessionMarker(), "IMetadataProviderFactory is null"); + + IMetadataProvider planairProvider = factory.getProvider(MetadataProviderType.PLANAIR); + if (planairProvider == null) + logger.info(getSessionMarker(), "IMetadataProvider is null"); + + ArchiveItem result = null; + data = planairProvider.list(opt); + if (data.size() != 0) + json = data.get(0).asJSON(); + else { + opt.setType(MetadataType.Promo); + data = planairProvider.list(opt); + if (data.size() != 0) + json = data.get(0).asJSON(); + else { + opt.setType(MetadataType.AD); + data = planairProvider.list(opt); + if (data.size() != 0) + json = data.get(0).asJSON(); + } + } + if (json != null) { + result = new ArchiveItem(); + result.setItemHouseId(json.getString(ITEM_HOUSEID)); + result.setItemTitle(json.getString(ITEM_TITLE)); + result.setMediaHouseId(json.getString(MEDIA_HOUSEID)); + result.setMediaTitle(json.getString(MEDIA_TITLE)); + result.setMediaDescription(json.getString(MEDIA_DESCRIPTION)); + result.setMediaType(json.getString(MEDIA_TYPE)); + } + return result; + } + + @Override + public FileVisitResult postVisitDirectory(Path paramT, IOException paramIOException) throws IOException { + return FileVisitResult.CONTINUE; + } + + @Override + public FileVisitResult preVisitDirectory(Path dir, BasicFileAttributes paramBasicFileAttributes) + throws IOException { + Path dirName = dir.getFileName(); + + if (skipPathNames.contains(dirName.toString())) { + //logger.info(getSessionMarker(), "PreVisit skip {}", dir); + return FileVisitResult.SKIP_SUBTREE; + } //else + //logger.info(getSessionMarker(), "PreVisit {}", dir); + + return FileVisitResult.CONTINUE; + } + + private boolean processPathItem(Path mediaPath) throws Exception { + if (getJobRuntime().isWaitingCancel()) { + cancel(); + return false; + } + + BasicFileAttributes attr = Files.readAttributes(mediaPath, BasicFileAttributes.class); + File mediaFSFile = mediaPath.toFile(); + String fileName = mediaPath.getFileName().toString(); + + if (mediaFSFile.isDirectory()) + return true; + + if (fileName.startsWith(".") || fileName.endsWith(".nomd")) + return true; + + Path nomdFile = Paths.get(mediaPath.toString() + ".nomd"); + ArchiveItem archiveItem = createArchiveItem(mediaPath); + List killDateFiles = getKillDateFiles(mediaPath); + MediaFile mediaFile = getMediaFile(fileName); + + long size = mediaFSFile.length(); + Date lastModifiedDate = new Date(attr.lastModifiedTime().toMillis()); + //Date lastAccesDate = new Date(attr.lastAccessTime().toMillis()); + Date createDate = new Date(attr.creationTime().toMillis()); + boolean catchedExists = EscortFiles.isMediaCatched(mediaPath); + boolean noMDExists = Files.exists(nomdFile); + boolean killdateExists = killDateFiles.size() > 0; + //boolean mediaInfoAvailable = canReadMediaInfo(mediaPath); + boolean mcArchived = mediaFile != null; + boolean metadataEquals = false; + boolean tsmArchived = false; + boolean sizeEquals = false; + Date mcArchivedDate = null; + long tsmSize = 0; + Date tsmBackupDate = null; + boolean includeContains = includeList.contains(mediaPath.toString()); + + if (mcArchived) { + //metadata + Media media = getManager().getMedia(mediaFile.getMediaId()); + mcArchivedDate = media.getCreated(); + Item item = getManager().getItem(media.getItemId()); + metadataEquals = checkArchiveItem(archiveItem, item, media); + + //tsm + String tsmFileName = mediaFile.getRelativePath(); + RemoteFile tsmFile = getTSMFile(tsmFileName); + tsmArchived = tsmFile != null; + if (tsmArchived) { + tsmSize = tsmFile.getSize(); + tsmBackupDate = getTSMBackupDate(tsmFileName); + sizeEquals = tsmSize == size; + } + } + + boolean canDelete = !noMDExists && catchedExists && killdateExists && mcArchived && tsmArchived + && metadataEquals; + Date now = new Date(System.currentTimeMillis()); + logger.info(getSessionMarker(), "{};{};{};{};{};{};{};{};{};{};{};{};{};{};{};{};{};{};", D(now), fileName, + mediaPath.getParent(), size, D(createDate), D(lastModifiedDate), D(mcArchivedDate), D(tsmBackupDate), + tsmSize, YN(catchedExists), YN(killdateExists), YN(noMDExists), YN(mcArchived), YN(tsmArchived), + YN(metadataEquals), YN(sizeEquals), YN(canDelete), YN(includeContains)); + + return true; + } + + private Date getTSMBackupDate(String tsmFileName) throws Exception { + Date result = null; + IMetadataProviderFactory factory = getService(IMetadataProviderFactory.class); + if (factory == null) + return null; + IMetadataProvider provider = factory.getProvider(MetadataProviderType.TSM); + if (provider == null) + return null; + + IMetadataListOptions opt = provider.createOptions(new BasicDBObject("fileName", tsmFileName)); + + List tsmContents = provider.list(opt); + if (tsmContents != null && tsmContents.size() > 0) { + + for (IMetadata md : tsmContents) { + Date backupDate = md.asJSON().getDate("backupDate"); + if (result == null) + result = backupDate; + else { + if (backupDate.after(result)) + result = backupDate; + } + } + } + return result; + } + + private char YN(boolean value) { + return value ? 'Y' : 'N'; + } + + private String D(Date value) { + return value == null ? "" : df.format(value); + } + + @Override + public FileVisitResult visitFile(Path filePath, BasicFileAttributes paramBasicFileAttributes) throws IOException { + //logger.info(getSessionMarker(), "Will checked {}", filePath); + try { + if (!processPathItem(filePath)) + return FileVisitResult.TERMINATE; + } catch (Exception e) { + logger.catching(e); + } + return FileVisitResult.CONTINUE; + } + + @Override + public FileVisitResult visitFileFailed(Path filePath, IOException paramIOException) throws IOException { + logger.info("Error archive {}", filePath); + return FileVisitResult.CONTINUE; + } + + private List getKillDateFiles(Path filePath) { + String killDateFilePattern = String.format("%s.*%s", filePath.getFileName().toString(), KILLDATEEXT); + List result = new ArrayList<>(); + Path statusPath = null; + try { + statusPath = Paths.get(filePath.getParent().toString(), EscortFiles.STATUSFOLDER); + } catch (Exception e) { + logger.catching(e); + return null; + } + File statusPathFile = statusPath.toFile(); + if (statusPathFile.exists() && statusPathFile.isDirectory()) { + try (DirectoryStream stream = Files.newDirectoryStream(statusPath, killDateFilePattern)) { + stream.forEach(p -> result.add(p)); + } catch (Exception e) { + logger.catching(e); + } + } + Collections.sort(result); + return result; + } + + MediaFile getMediaFile(String fileName) { + MediaFile result = null; + MediaFileDAO mfDAO = (MediaFileDAO) getManager().getBaseDAO(MediaFile.class); + List mfList = mfDAO.getByHouseId(fileName); + if (mfList != null && mfList.size() == 1) + result = (MediaFile) mfList.get(0); + return result; + } + + RemoteFile getTSMFile(String mcFileName) { + RemoteFile result = null; + Store tsmStore = getManager().getSystemStore(false); + StoreUri tsmStoreUri = tsmStore.getSourceStoreUri(RemoteStoreProtocol.TSM); + try { + result = tsmStoreUri.getRemoteFile(mcFileName); + } catch (Exception e) { + logger.error(e); + } finally { + tsmStoreUri.cleanUp(); + } + return result; + } + + private Set loadIncludeList(String location) throws IOException { + Set result = new LinkedHashSet<>(); + + Path path = Paths.get(location); + List lines = FileUtils.readLines(path.toFile()); + + String lastDir = null; + for (String line : lines) { + line = line.trim(); + if (line.startsWith("Directory of")) { + lastDir = line; + lastDir = lastDir.replace("Directory of", ""); + lastDir = lastDir.replace("X:", ""); + lastDir = lastDir.replace("\\", "/"); + lastDir = lastDir.trim(); + // if (!lastDir.endsWith(".STATUS")) + // System.out.println(">> " + lastDir); + } + + if (lastDir != null && lastDir.endsWith(".STATUS")) + continue; + + if (line.startsWith("2") && line.length() > 39) { + String file = line.substring(39).trim(); + + if (file.equals(".") || file.equals("..")) + continue; + + String len = line.substring(21, 39).trim(); + if (len.length() != 0) { + String fullpath = "/mnt/POLC/FINISHED_SHOWS" + lastDir + "/" + file; + //System.out.println("/mnt/POLC/FINISHED_SHOWS" + lastDir + "/" + file + " : " + len); + result.add(fullpath); + } + } + + } + + return result; + } + +} diff --git a/server/-product/production/OMAR/jobs/steps/SaveMediaProxy.java b/server/-product/production/OMAR/jobs/steps/SaveMediaProxy.java new file mode 100644 index 00000000..3f0f667d --- /dev/null +++ b/server/-product/production/OMAR/jobs/steps/SaveMediaProxy.java @@ -0,0 +1,22 @@ +package user.jobengine.server.steps; + +import user.jobengine.db.IItemManager; +import user.jobengine.db.Media; +import user.jobengine.db.MediaFile; +import user.jobengine.db.Store; + +public class SaveMediaProxy extends JobStep { + + @StepEntry + public Object[] execute(Media media, String proxyRelativePath) throws Exception { + IItemManager manager = getManager(); + Store store = manager.getCurrentLowresStore(); + MediaFile mediaFile = new MediaFile(); + mediaFile.setMedia(media); + mediaFile.setStore(store); + mediaFile.setFileType(manager.getFileType("Low-res")); + mediaFile.setRelativePath(proxyRelativePath); + manager.add(mediaFile); + return null; + } +} diff --git a/server/-product/production/OMAR/jobs/steps/TSMBackupStep.java b/server/-product/production/OMAR/jobs/steps/TSMBackupStep.java new file mode 100644 index 00000000..b526f16f --- /dev/null +++ b/server/-product/production/OMAR/jobs/steps/TSMBackupStep.java @@ -0,0 +1,239 @@ +package user.jobengine.server.steps; + +import java.io.File; +import java.io.IOException; +import java.sql.Timestamp; +import java.util.Date; +import java.util.List; + +import org.apache.commons.lang.RandomStringUtils; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.apache.logging.log4j.Marker; +import org.apache.logging.log4j.message.Message; +import org.apache.logging.log4j.message.ParameterizedMessage; + +import user.commons.RemoteFile; +import user.commons.StoreUri; +import user.commons.configuration.SystemConfiguration; +import user.commons.mediatool.Timecode; +import user.commons.mediatool.Timecode.Type; +import user.commons.remotestore.IProgressEventListener; +import user.commons.remotestore.IStatusEventListener; +import user.commons.remotestore.ProgressEvent; +import user.commons.remotestore.RemoteStoreProtocol; +import user.commons.remotestore.StatusEvent; +import user.jobengine.db.FileType; +import user.jobengine.db.IItemManager; +import user.jobengine.db.Media; +import user.jobengine.db.MediaFile; +import user.jobengine.db.Store; +import user.jobengine.server.IJobEngine; +import user.jobengine.server.IJobRuntime; +import user.jobengine.server.JobEngineException; +import user.jobengine.server.steps.shared.EscortFiles; +import user.jobengine.server.steps.shared.ItemManagerExtensions; + +public class TSMBackupStep extends JobStep { + private static final String MXFEXT = ".MXF"; + private static final Logger logger = LogManager.getLogger(); + private static boolean RANDOMIZE_ARCHIVES = SystemConfiguration.getInstance().value("tsm.randomize-archives"); + private IItemManager manager; + private File sourceMediaFile; + private Store tsmStore; + private StoreUri targetUri; + private FileType fileType; + private Marker marker; + + @StepEntry + public Object[] execute(ArchiveItem archiveItem, Media mediaCubeMedia, int killDateDays) throws Exception { + marker = getSessionMarker(); + + File sourceMediaFile = new File(archiveItem.getMediaFile()); + String sourceFileName = sourceMediaFile.getName(); + long fileSize = sourceMediaFile.length(); + + try { + Timecode timecode = new Timecode(mediaCubeMedia.getLength(), Type.PAL); + String details = String.format("%s (%s, %d bytes)", sourceFileName, timecode.toString(), fileSize); + logger.info(marker, details); + getJobRuntime().setDescription(details); + } catch (Exception e) { + String details = String.format("%s (%d bytes)", sourceFileName, fileSize); + getJobRuntime().setDescription(details); + } + Timecode timecode = new Timecode(mediaCubeMedia.getLength(), Type.PAL); + String details = String.format("%s (%s, %d bytes)", sourceFileName, timecode.toString(), fileSize); + getJobRuntime().setDescription(details); + try { + + setAndCheck(archiveItem, mediaCubeMedia, getEngine()); + + // TODO mxf helyett az osszes kiterjesztest!!!!! + // A dupla ellenorzes a napon beluli ismetlesek miatt kell + long existingMediaId = archiveItem.getExistingMediaId(); + + if (fileSize == 0 && existingMediaId == 0) { + existingMediaId = ItemManagerExtensions.getExistingRundownMedia(manager, + sourceFileName.replace(MXFEXT, "")); + if (existingMediaId == 0) + existingMediaId = -1; + } + + if (existingMediaId == 0) + existingMediaId = ItemManagerExtensions.getExistingRundownMedia(manager, + sourceFileName.replace(MXFEXT, "")); + String targetFileName; + if (RANDOMIZE_ARCHIVES) { + // a-z, A-Z, 0-9. For example: WRMcpIk7, s57JwCVA + // veletlenszeru neveket adunk! + targetFileName = String.format("%s-%s", RandomStringUtils.randomAlphanumeric(8), sourceFileName); + } else + targetFileName = sourceFileName; + + if (existingMediaId == 0) { + StoreUri sourceUri = manager.createStoreUri(RemoteStoreProtocol.LOCAL, + sourceMediaFile.getParent().toString()); + + final IJobRuntime runtime = getJobRuntime(); + sourceUri.addProgressListener(new IProgressEventListener() { + @Override + public void progressChanged(ProgressEvent evt) { + runtime.incrementProgress(evt.getProgress()); + } + }); + sourceUri.addStatusListener(new IStatusEventListener() { + @Override + public void statusChanged(StatusEvent evt) { + evt.setCancel(!canContinue()); + } + }); + + RemoteFile remoteFile = sourceUri.transferFrom(targetUri, sourceFileName, targetFileName); + } + + if (existingMediaId > 0) + logger.info(marker, "Az '{}' TSM mentése nem szükséges, mert már megtalálható az archívumban.", + sourceFileName); + + // Fel kell szabadítani, hogy a kovetkezo archivalaskor is nekifusson + if (existingMediaId == -1) { + logger.info(marker, + "Az '{}' mentése jelenleg nem lehetséges, mert a szükséges metaadat még nem található meg az archívumban.", + sourceFileName); + if (!archiveItem.removeCatchedFile()) + logger.error(marker, + "Az '{}' állomány .catched jelző állománya nem törölhető. Az újabb archiválási kísérlethez annak kézi eltávolítása szükséges!", + sourceMediaFile.getName()); + mediaCubeMedia.remove(); + } else { + saveMetadata(mediaCubeMedia, sourceMediaFile, targetFileName, existingMediaId, fileSize, + archiveItem.isDisableProxy()); + logger.info(marker, "Az '{}' archiválása sikeres.", sourceFileName); + if (killDateDays != 0) + EscortFiles.createUNCKillDate(sourceMediaFile.getParent(), sourceFileName, killDateDays, marker); + } + + } catch (Exception e) { + logger.catching(e); + Message m = new ParameterizedMessage("Az '{}' állomány archiválása sikertelen. A rendszer hibaüzenete: {}", + details, e.getMessage()); + logger.error(marker, m); + if (!archiveItem.removeCatchedFile()) + logger.error(marker, + "Az '{}' állomány .catched jelző állománya nem törölhető. Az újabb archiválási kísérlethez annak kézi eltávolítása szükséges!", + sourceMediaFile.getName()); + throw new Exception(m.getFormattedMessage()); + } + return null; + } + + private void saveMetadata(Media mediaCubeMedia, File sourceFile, String targetFileName, long existingMediaId, + long fileSize, boolean disableProxy) { + + if (existingMediaId == 0) { + MediaFile mf = manager.createMediaFile(targetFileName, fileType, tsmStore, mediaCubeMedia); + mf.setHouseId(sourceFile.getName()); + mf.setFileSize(fileSize); + // 210617 proxy keszites tiltasa + mf.setDisableProxy(disableProxy); + mf.add(); + } else { + Media existingMedia = manager.getMedia(existingMediaId); + List mediaFiles = existingMedia.getMediaFiles(); + if (mediaFiles != null) { + for (MediaFile mf : mediaFiles) { + mf.setPersister(manager); + mf.setId(0); + mf.setMedia(mediaCubeMedia); + // mivel itt masolat keszul, nem allitunk at semmit +// mf.setFileSize(fileSize); +// mf.setDisableProxy(disableProxy); + mf.add(); + } + } + } + mediaCubeMedia.setPersister(manager); + + // 210614 megis maradjon az aktualis idopont + mediaCubeMedia.setArchived(new Timestamp(new Date().getTime())); + + /* + try { + BasicFileAttributes attr = Files.readAttributes(sourceFile.toPath(), BasicFileAttributes.class); + mediaCubeMedia.setArchived(new Timestamp(attr.creationTime().toMillis())); + } catch (IOException e) { + logger.catching(e); + } + */ + mediaCubeMedia.modify(); + } + + private void setAndCheck(ArchiveItem archiveItem, Media mediaCubeMedia, IJobEngine jobEngine) + throws JobEngineException, IOException { + if (jobEngine == null) { + logger.error(marker, "Az folyamatkezelő réteg nem elérhető."); + throw new NullPointerException("Internal error, missing JobEngine reference."); + } + manager = jobEngine.getItemManager(); + if (manager == null) { + logger.error(marker, "Az adatbáziskezelő réteg nem elérhető."); + throw new NullPointerException("Internal error, missing ItemManager reference."); + } + if (archiveItem == null) { + logger.error(marker, "A folyamat 'archiveItem' bemeneti paramétere üres."); + throw new NullPointerException("System is not configured properly, missing 'archiveItem' input parameter."); + } + sourceMediaFile = new File(archiveItem.getMediaFile()); + if (sourceMediaFile == null) { + logger.error(marker, "A folyamat 'archiveItem' bemeneti paraméter 'mediaFile' értéke üres."); + throw new NullPointerException( + "System is not configured properly, missing 'mediaFile' value in 'archiveItem' input parameter."); + } + if (!sourceMediaFile.exists()) { + logger.error(marker, "A(z) {} állomány nem létezik vagy nem érhető el.", sourceMediaFile.getName()); + throw new IOException(String.format("Input file {} does not exist or unreachable.", sourceMediaFile.getName())); + } + tsmStore = manager.getSystemStore(false); + if (tsmStore == null) { + logger.error(marker, "A TSM rendszer beállítás nem elérhető."); + throw new NullPointerException("System is not configured properly, missing TSM Store."); + } + targetUri = tsmStore.getSourceStoreUri(RemoteStoreProtocol.TSM); + if (targetUri == null) { + logger.error(marker, "A TSM rendszer beállítás paraméterei nem elérhetőek."); + throw new NullPointerException("System is not configured properly, missing TSM StoreUri."); + } + fileType = manager.getFileType("High-res"); + if (fileType == null) { + logger.error(marker, "Adatbázis bejegyzés hiba, a 'High-res' FileType nem található."); + throw new NullPointerException("System is not configured properly, missing 'High-res' FileType."); + } + if (mediaCubeMedia == null) { + logger.error(marker, "A folyamat 'mediaCubeMedia' bemeneti paramétere üres."); + throw new NullPointerException( + "System is not configured properly, 'mediaCubeMedia' input parameter missing."); + } + + } +} diff --git a/server/-product/production/OMAR/jobs/steps/TSMExtendedRetrieveStep.java b/server/-product/production/OMAR/jobs/steps/TSMExtendedRetrieveStep.java new file mode 100644 index 00000000..fd222344 --- /dev/null +++ b/server/-product/production/OMAR/jobs/steps/TSMExtendedRetrieveStep.java @@ -0,0 +1,95 @@ +package user.jobengine.server.steps; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + +import user.commons.StoreUri; +import user.commons.remotestore.RemoteStoreProtocol; +import user.jobengine.db.ArchivedMedia; +import user.jobengine.db.IItemManager; +import user.jobengine.server.IJobEngine; +import user.jobengine.server.IJobRuntime; + +/* +import user.jobengine.server.steps.EscortFiles; +*/ + +public class TSMExtendedRetrieveStep extends TSMRestoreStep { + private static String NEXIO_HOST = System.getProperty("nexio.host"); + // private static String NEXIO_HOST = SystemConfiguration.getInstance().value("services.nexio.host"); + private static final Logger logger = LogManager.getLogger(); + + private boolean useNexioTarget; + private int nexioPort; + private String nexioUserName, nexioPassword; + private String nexioAgency; + + @Override + protected void afterRestore(StoreUri targetUri, String targetPath, int killDateDays, String targetFileName) throws Exception { + if (useNexioTarget) { + EscortFiles.setNEXIOKillDate(killDateDays, targetFileName, nexioAgency, targetUri); + } else { + super.afterRestore(targetUri, targetPath, killDateDays, targetFileName); + } + } + + @Override + protected void beforeRestore(StoreUri targetURI, String targetName) throws Exception { + String newTargetName = targetName; + if (targetName.contains(".")) + newTargetName = targetName.substring(0, targetName.lastIndexOf('.')); + if (useNexioTarget) + if (targetURI.fileExists(newTargetName + ".mxf")) + throw new Exception(String.format("%s-The newly created file name is existed.", getClass().getSimpleName())); + } + + @Override + protected void checkTargetPath(String targetPath) { + if (!useNexioTarget) + super.checkTargetPath(targetPath); + } + + @Override + protected StoreUri createTargetUri(IItemManager manager, String targetPath) throws NullPointerException { + StoreUri result = null; + logger.info(getSessionMarker(), "Create target uri {}", targetPath); + if (useNexioTarget) { + if (NEXIO_HOST == null) { + throw new NullPointerException("Missing system property on 'nexio.host' name"); + } + result = manager.createStoreUri(RemoteStoreProtocol.FTP, NEXIO_HOST); + result.setPortNumber(nexioPort); + result.setUserName(nexioUserName); + result.setPassword(nexioPassword); + } else + result = super.createTargetUri(manager, targetPath); + return result; + } + + @StepEntry + public Object[] execute(ArchivedMedia archivedMedia, String targetPath, String targetNamePattern, String successRecipient, int killDateDays, + String localRetrievePath, String globalRetrievePath, boolean useNexioTarget, String nexioAgency, int nexioPort, String nexioUserName, + String nexioPassword, IJobEngine jobEngine, IJobRuntime jobRuntime) throws Exception { + this.useNexioTarget = useNexioTarget; + this.nexioAgency = nexioAgency; + this.nexioPort = nexioPort; + this.nexioUserName = nexioUserName; + this.nexioPassword = nexioPassword; + if (nexioPort == 0) { + throw new NullPointerException("System is not configured properly, 'nexioPort' input parameter missing."); + } + if (nexioUserName == null) { + throw new NullPointerException("System is not configured properly, 'nexioUserName' input parameter missing."); + } + if (nexioPassword == null) { + throw new NullPointerException("System is not configured properly, 'nexioPassword' input parameter missing."); + } + if (nexioAgency == null) { + throw new NullPointerException("System is not configured properly, 'nexioAgency' input parameter missing."); + } + + return super.execute(archivedMedia.getMedia(), targetPath, targetNamePattern, successRecipient, killDateDays, localRetrievePath, globalRetrievePath, + jobEngine, jobRuntime); + } + +} diff --git a/server/-product/production/OMAR/jobs/steps/TSMRestoreStep.java b/server/-product/production/OMAR/jobs/steps/TSMRestoreStep.java new file mode 100644 index 00000000..26d8607b --- /dev/null +++ b/server/-product/production/OMAR/jobs/steps/TSMRestoreStep.java @@ -0,0 +1,187 @@ +package user.jobengine.server.steps; + +import java.io.IOException; +import java.nio.file.Paths; +import java.text.Normalizer; +import java.util.List; +import java.util.regex.Pattern; + +import org.apache.commons.lang.StringUtils; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.apache.logging.log4j.Marker; +import org.apache.logging.log4j.message.Message; + +import user.commons.LogUtils; +import user.commons.RemoteFile; +import user.commons.StoreUri; +import user.commons.mediatool.Timecode; +import user.commons.mediatool.Timecode.Type; +import user.commons.remotestore.IProgressEventListener; +import user.commons.remotestore.IStatusEventListener; +import user.commons.remotestore.ProgressEvent; +import user.commons.remotestore.RemoteStoreProtocol; +import user.commons.remotestore.StatusEvent; +import user.jobengine.db.IItemManager; +import user.jobengine.db.Media; +import user.jobengine.db.MediaFile; +import user.jobengine.db.Store; +import user.jobengine.server.IJobEngine; +import user.jobengine.server.IJobRuntime; +import user.jobengine.server.steps.shared.EscortFiles; + +public class TSMRestoreStep extends JobStep { + private static final String DOT = "."; + public static final Pattern DIACRITICS_AND_FRIENDS = Pattern.compile("[\\p{InCombiningDiacriticalMarks}\\p{IsLm}\\p{IsSk}]+"); + private static final Logger logger = LogManager.getLogger(); + private IItemManager manager; + private StoreUri targetUri; + private StoreUri sourceUri; + private String sourceFileName; + private Marker marker; + + protected void afterRestore(StoreUri targetUri, String targetPath, int killDateDays, String targetFileName) throws IOException, Exception { + if (killDateDays != 0) + EscortFiles.createUNCKillDate(targetPath, targetFileName, killDateDays, marker); + } + + protected void beforeRestore(StoreUri targetURI, String targetFileName) throws Exception { + } + + protected void checkTargetPath(String targetPath) { + if (StringUtils.isBlank(targetPath)) { + logger.error(marker, "A folyamat 'targetPath' bemeneti paramétere üres."); + throw new NullPointerException("System is not configured properly, 'targetPath' input parameter missing."); + } + } + + protected StoreUri createTargetUri(IItemManager manager, String targetPath) { + return manager.createStoreUri(RemoteStoreProtocol.LOCAL, targetPath); + } + + @StepEntry + public Object[] execute(Media mediaCubeMedia, String targetPath, String targetNamePattern, String successRecipient, int killDateDays, + String localRetrievePath, String globalRetrievePath, IJobEngine jobEngine, IJobRuntime jobRuntime) throws Exception { + marker = jobRuntime.getSessionMarker(); + setAndCheck(mediaCubeMedia, targetPath, targetNamePattern, localRetrievePath, globalRetrievePath, jobEngine); + String targetFileName = String.format(targetNamePattern, sourceFileName); + //20210129 + //targetFileName = getMaximizedFileName(mediaCubeMedia, targetFileName, 120); + + Timecode timecode = new Timecode(mediaCubeMedia.getLength(), Type.PAL); + try { + String details = String.format("%s (%s)", sourceFileName, timecode.toString()); + jobRuntime.setDescription(details); + beforeRestore(targetUri, targetNamePattern); + final IJobRuntime runtime = jobRuntime; + sourceUri.addProgressListener(new IProgressEventListener() { + @Override + public void progressChanged(ProgressEvent evt) { + runtime.incrementProgress(evt.getProgress()); + } + }); + sourceUri.addStatusListener(new IStatusEventListener() { + @Override + public void statusChanged(StatusEvent evt) { + evt.setCancel(!canContinue()); + } + }); + RemoteFile result = sourceUri.transferFrom(targetUri, sourceFileName, targetFileName); + + String globalTargetPath = Paths.get(targetPath, targetFileName) + .getParent() + .toString() + .replace(Paths.get(localRetrievePath).toString(), globalRetrievePath); + + logger.info(marker, + "Az '{}' állomány visszatöltése sikeres volt '{}' néven. A célmappa a ide kattintva nyitható meg.", + sourceFileName, targetFileName, globalTargetPath); + afterRestore(targetUri, targetPath, killDateDays, targetFileName); + + } catch (Exception e) { + Message msg = LogUtils.format("Az '{}' állomány visszatöltése sikertelen. A rendszer hibaüzenete: {}", sourceFileName, e.getMessage()); + logger.error(marker, msg); + // logger.error(jobRuntime.marker, msg); + logger.catching(e); + throw e; + } + + return null; + } + + private String getMaximizedFileName(Media mediaCubeMedia, String targetFileName, int limit) { + String name = targetFileName; + String extension = ""; + if (name.contains(DOT)) { + extension = DOT + name.substring(name.lastIndexOf(DOT) + 1); + name = name.substring(0, name.lastIndexOf(DOT)); + } + String typeName = Normalizer.normalize(mediaCubeMedia.getItemType().getName(), Normalizer.Form.NFD); + typeName = DIACRITICS_AND_FRIENDS.matcher(typeName).replaceAll(""); + typeName = typeName.replace(" ", "_"); + + int allowedSize = limit - typeName.length() - 1 - extension.length(); + if (name.length() > allowedSize) + name = name.substring(0, allowedSize); + + return String.format("%s_%s%s", name, typeName, extension); + } + + private String getSourceFileName(Media mediaCubeMedia, Store store) { + List mediaFiles = mediaCubeMedia.getMediaFiles(); + if (mediaFiles == null) + return null; + for (MediaFile mediaFile : mediaFiles) { + if (mediaFile.getStore().getId() == store.getId()) + return mediaFile.getRelativePath(); + } + return null; + } + + private void setAndCheck(Media mediaCubeMedia, String targetPath, String targetNamePattern, String localRetrievePath, String globalRetrievePath, + IJobEngine jobEngine) { + if (jobEngine == null) { + logger.error(marker, "Az folyamatkezelő réteg nem elérhető."); + throw new NullPointerException("Internal error, missing JobEngine reference."); + } + manager = jobEngine.getItemManager(); + if (manager == null) { + logger.error(marker, "Az adatbáziskezelő réteg nem elérhető."); + throw new NullPointerException("Internal error, missing ItemManager reference."); + } + if (mediaCubeMedia == null) { + logger.error(marker, "A folyamat 'mediaCubeMedia' bemeneti paramétere üres."); + throw new NullPointerException("System is not configured properly, 'mediaCubeMedia' input parameter missing."); + } + checkTargetPath(targetPath); + if (StringUtils.isBlank(targetNamePattern)) { + logger.error(marker, "A folyamat 'targetNamePattern' bemeneti paramétere üres."); + throw new NullPointerException("System is not configured properly, 'targetNamePattern' input parameter missing."); + } + Store tsmStore = manager.getSystemStore(false); + if (tsmStore == null) { + logger.error(marker, "A TSM rendszer beállítás nem elérhető."); + throw new NullPointerException("System is not configured properly, missing TSM Store."); + } + sourceUri = tsmStore.getSourceStoreUri(RemoteStoreProtocol.TSM); + if (sourceUri == null) { + logger.error(marker, "A TSM rendszer beállítás paraméterei nem elérhetőek."); + throw new NullPointerException("System is not configured properly, missing TSM StoreUri."); + } + targetUri = createTargetUri(manager, targetPath); + sourceFileName = getSourceFileName(mediaCubeMedia, tsmStore); + if (sourceFileName == null) { + logger.error(marker, "Adatbázis bejegyzés hiba, a visszatöltendő fájl neve nem található."); + throw new NullPointerException("Database error, missing MediaFile 'relativePath'."); + } + + if (StringUtils.isBlank(localRetrievePath)) { + logger.error(marker, "A folyamat 'localRetrievePath' bemeneti paramétere üres."); + throw new NullPointerException("System is not configured properly, 'localRetrievePath' input parameter missing."); + } + if (StringUtils.isBlank(globalRetrievePath)) { + logger.error(marker, "A folyamat 'globalRetrievePath' bemeneti paramétere üres."); + throw new NullPointerException("System is not configured properly, 'globalRetrievePath' input parameter missing."); + } + } +} diff --git a/server/-product/production/OMAR/jobs/steps/TSMSimpleRestoreStep.java b/server/-product/production/OMAR/jobs/steps/TSMSimpleRestoreStep.java new file mode 100644 index 00000000..ba63212c --- /dev/null +++ b/server/-product/production/OMAR/jobs/steps/TSMSimpleRestoreStep.java @@ -0,0 +1,57 @@ +package user.jobengine.server.steps; + +import java.nio.file.Files; +import java.nio.file.Paths; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + +import user.commons.StoreUri; +import user.commons.mediatool.Timecode; +import user.commons.mediatool.Timecode.Type; +import user.commons.remotestore.IProgressEventListener; +import user.commons.remotestore.IStatusEventListener; +import user.commons.remotestore.ProgressEvent; +import user.commons.remotestore.RemoteStoreProtocol; +import user.commons.remotestore.StatusEvent; +import user.jobengine.db.Media; +import user.jobengine.db.MediaFile; +import user.jobengine.db.Store; + +public class TSMSimpleRestoreStep extends JobStep { + private static final Logger logger = LogManager.getLogger(); + + @StepEntry + public Object[] execute(long mediaId, String targetPath) throws Exception { + String fileName = null; + try { + Media media = getManager().getMedia(mediaId); + MediaFile mediaFile = getManager().getSystemMediaFile(media); + fileName = mediaFile.getRelativePath(); + Timecode timecode = new Timecode(media.getLength(), Type.PAL); + getJobRuntime().setDescription(String.format("%s (%s)", fileName, timecode.toString())); + Store tsmStore = getManager().getSystemStore(false); + StoreUri sourceStoreUri = getManager().getStoreUri(tsmStore.getName(), RemoteStoreProtocol.TSM); + sourceStoreUri.addProgressListener(new IProgressEventListener() { + @Override + public void progressChanged(ProgressEvent evt) { + setProgress(evt.getProgress()); + } + }); + sourceStoreUri.addStatusListener(new IStatusEventListener() { + @Override + public void statusChanged(StatusEvent evt) { + evt.setCancel(!canContinue()); + } + }); + StoreUri targetStoreUri = getManager().createStoreUri(RemoteStoreProtocol.LOCAL, targetPath); + sourceStoreUri.transferFrom(targetStoreUri, fileName, fileName + ".part"); + Files.move(Paths.get(targetPath, fileName + ".part"), Paths.get(targetPath, fileName)); + } catch (Exception e) { + logger.error("Az '{}' állomány visszatöltése sikertelen. A rendszer üzenete: {}", fileName, e.getMessage()); + throw e; + } + + return null; + } +} diff --git a/server/-product/production/OMAR/jobs/steps/TestForkCancelableStep.java b/server/-product/production/OMAR/jobs/steps/TestForkCancelableStep.java new file mode 100644 index 00000000..c1b49f76 --- /dev/null +++ b/server/-product/production/OMAR/jobs/steps/TestForkCancelableStep.java @@ -0,0 +1,35 @@ +package user.jobengine.server.steps; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + +import user.commons.IJob; +import user.commons.JobStatus; +import user.commons.ListUtils; +import user.jobengine.server.IJobEngine; +import user.jobengine.server.IJobRuntime; + +public class TestForkCancelableStep extends JobStep { + private static final String CHILD_TITLE = "Párhuzamosított alfolyamat"; + // private static final String CHILD_TEMPLATE = "fake-concurrent.xml"; + private static final String CHILD_TEMPLATE = "cancelable.xml"; + private static final Logger logger = LogManager.getLogger(); + int count = 3; + + @StepEntry + public Object[] execute(IJobEngine jobEngine, IJobRuntime jobRuntime) throws Exception { + for (int i = 0; i < count; i++) { + IJobRuntime rt = getEngine().submit(null, e -> { + if (e.getStatus().equals(JobStatus.SUSPENDED)) { + IJobRuntime rt = (IJobRuntime) e.getSource(); + logger.info("Cleanup on SUSPEND {}, {}", rt.getId(), rt.isDisableRetry()); + } + }, CHILD_TEMPLATE, "JOB " + i, 0, IJobEngine.DEFAULT_OWNER, ListUtils.asMap("param", i)); + + rt.setRelated("TEST" + rt.getId()); + } + + logger.info("Done"); + return null; + } +} diff --git a/server/-product/production/OMAR/jobs/steps/TranscodeFFAStranStep.java b/server/-product/production/OMAR/jobs/steps/TranscodeFFAStranStep.java new file mode 100644 index 00000000..d09685bc --- /dev/null +++ b/server/-product/production/OMAR/jobs/steps/TranscodeFFAStranStep.java @@ -0,0 +1,151 @@ +package user.jobengine.server.steps; + +import java.io.File; +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; + +import org.apache.commons.io.FilenameUtils; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.apache.logging.log4j.Marker; +import org.apache.logging.log4j.message.Message; +import org.apache.logging.log4j.message.ParameterizedMessage; + +import user.commons.FFAStransAPI; +import user.commons.IFFAStransAPI; +import user.commons.StoreUri; +import user.commons.mediatool.Timecode; +import user.commons.mediatool.Timecode.Type; +import user.commons.remotestore.RemoteStoreProtocol; +import user.jobengine.db.FileType; +import user.jobengine.db.IItemManager; +import user.jobengine.db.Media; +import user.jobengine.db.Store; +import user.jobengine.server.IJobEngine; +import user.jobengine.server.IJobRuntime; + +public class TranscodeFFAStranStep extends JobStep { + private static final int POLL_INTERVALL = 3000; + private static final String MP4EXT = ".MP4"; + private static final String MXFEXT = ".MXF"; + private static final String LOWRES_FILETYPE = "Low-res"; + private static final Logger logger = LogManager.getLogger("TranscodeFFAStranStep"); + private IItemManager manager; + private Store store; + private FileType fileType; + private Media mediaCubeMedia; + private Marker marker; + + @StepEntry + public Object[] execute(ArchiveItem archiveItem, Media mediaCubeMedia, String transcoderAddress, String transcoderTemplateName, + String globalHiresSourcePath, String localLowresTargetPath, boolean deleteSource, IJobEngine jobEngine, IJobRuntime jobRuntime) throws Exception { + + this.marker = jobRuntime.getSessionMarker(); + this.manager = jobEngine.getItemManager(); + this.store = check(manager.getCurrentLowresStore(), "lowres Store"); + this.fileType = check(manager.getFileType(LOWRES_FILETYPE), "lowres FileType"); + this.mediaCubeMedia = check(mediaCubeMedia, "mediaCubeMedia"); + check(archiveItem, "archiveItem"); + check(transcoderAddress, "transcoderAddress"); + check(transcoderTemplateName, "transcoderTemplateName"); + check(globalHiresSourcePath, "globalHiresSourcePath"); + check(localLowresTargetPath, "localLowresTargetPath"); + + File sourceMediaFile = new File(archiveItem.getMediaFile()); + logger.info("Transcoding {}", archiveItem.getMediaFile()); + String sourceFileName = sourceMediaFile.getName(); + Timecode timecode = new Timecode(mediaCubeMedia.getLength(), Type.PAL); + + String details = String.format("%s (%s, %d bytes)", sourceFileName, timecode.toString(), sourceMediaFile.length()); + + StoreUri storeUri = store.getTargetStoreUri(RemoteStoreProtocol.LOCAL); + if (storeUri == null) + throw new Exception("Can not detect proxy folder."); + + String webPath = storeUri.toString(true); + + Path targetPath = null; + try { + String targetFileName = FilenameUtils.removeExtension(sourceFileName) + MP4EXT; + targetPath = Paths.get(localLowresTargetPath, targetFileName); + if (!Files.exists(targetPath)) { + // jobRuntime.setDescription(String.format("%s: %s", jobRuntime.getDescription(), details)); + jobRuntime.setDescription(String.format("%s transzkódolása", details)); + String sourceFile = Paths.get(globalHiresSourcePath, sourceFileName).toString(); + IFFAStransAPI api = new FFAStransAPI(transcoderAddress, p -> { + if (p <= 100) + jobRuntime.incrementProgress(p); + }); + + api.submit(transcoderTemplateName, sourceFile); + api.monitor(POLL_INTERVALL); + } + + //a sikeres transzkod utan nem mindig van ott egybol a fajl + long started = System.currentTimeMillis(); + while (!Files.exists(targetPath)) { + long current = System.currentTimeMillis(); + //max 5 perc varakozas + if (current - started > 5 * 60 * 1000) + throw new Exception("Transcode job target file access timed out"); + Thread.sleep(POLL_INTERVALL); + } + + postprocess(targetPath, webPath); + + } catch (Exception e) { + logger.catching(e); + Message m = new ParameterizedMessage("{} átkódolás hiba: {}", sourceFileName, e.getMessage()); + logger.error(marker, m); + throw new Exception(m.getFormattedMessage()); + } finally { + try { + if (deleteSource && sourceMediaFile != null && sourceMediaFile.exists()) + sourceMediaFile.delete(); + } catch (Exception e) { + logger.catching(e); + } + try { + if (deleteSource && targetPath != null && Files.exists(targetPath)) + Files.delete(targetPath); + } catch (Exception e) { + logger.catching(e); + } + } + return null; + } + + private void postprocess(Path transcodedFilePath, String webPath) throws IOException { + Path lowresPath = null; + try { + String transcodedFileName = transcodedFilePath.getFileName().toString(); + String targetPath = null; + if (transcodedFileName.indexOf(".") > 2) { + Path subdir = Paths.get(transcodedFileName.substring(0, 1), transcodedFileName.substring(1, 2), transcodedFileName.substring(2, 3)); + EscortFiles.ensureUNCFolder(webPath, subdir.toString()); + targetPath = Paths.get(subdir.toString(), transcodedFileName).toString(); + } else { + targetPath = transcodedFileName; + } + lowresPath = Paths.get(webPath, targetPath); + int version = 1; + while (Files.exists(lowresPath)) { + String fileName = transcodedFileName + version + MP4EXT; + lowresPath = Paths.get(lowresPath.toString().replace(transcodedFileName, fileName)); + targetPath = targetPath.replace(transcodedFileName, fileName); + transcodedFileName = fileName; + version++; + } + + Files.move(transcodedFilePath, lowresPath); + manager.createMediaFile(targetPath, fileType, store, mediaCubeMedia).add(); + } catch (IOException e) { + logger.catching(e); + logger.error(marker, "A '{}' állomány mozgatása a '{}' helyre nem sikerült.", transcodedFilePath, lowresPath); + throw e; + } + } + +} diff --git a/server/-product/production/OMAR/jobs/steps/TransferStep.java b/server/-product/production/OMAR/jobs/steps/TransferStep.java new file mode 100644 index 00000000..c10084fa --- /dev/null +++ b/server/-product/production/OMAR/jobs/steps/TransferStep.java @@ -0,0 +1,110 @@ +package user.jobengine.server.steps; + +import java.nio.file.Path; +import java.nio.file.Paths; + +import org.apache.commons.net.ftp.FTPClient; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + +import user.commons.RemoteFile; +import user.commons.StoreUri; +import user.commons.remotestore.FtpDirectoryLister; +import user.commons.remotestore.RemoteStoreProtocol; +import user.jobengine.db.Store; + +public class TransferStep extends JobStep { + private static final String DOT_PART = ".part"; + private static final Logger logger = LogManager.getLogger(); + + private void copy(StoreUri sourceStoreUri, String sourceFileName, StoreUri targetStoreUri, String targetFileName) throws Exception { + String currentTargetFileName = targetFileName; + + boolean renameAfterCopy = false; + boolean renameAfterFTP = false; + if (getTmpExtension() != null) { + if (RemoteStoreProtocol.LOCAL.equals(targetStoreUri.getProtocol())) { + currentTargetFileName += getTmpExtension(); + renameAfterCopy = true; + } + Store targetStore = getManager().getStore(targetStoreUri.getStoreId()); + + if (RemoteStoreProtocol.FTP.equals(targetStoreUri.getProtocol()) && !"NEXIO1".equals(targetStore.getName()) + && !"NEXIO2".equals(targetStore.getName())) { + currentTargetFileName += getTmpExtension(); + renameAfterFTP = true; + } + } + + sourceStoreUri.transferFrom(targetStoreUri, sourceFileName, currentTargetFileName); + + logger.info(getMarker(), "Transfer of {} completed from {} to {}", sourceFileName, sourceStoreUri, currentTargetFileName); + + if (renameAfterCopy) { + Path tmpTargetFile = Paths.get(targetStoreUri.toString(true), currentTargetFileName); + Path targetFile = Paths.get(targetStoreUri.toString(true), targetFileName); + try { + logger.info(getMarker(), "Renaming LOCAL file from {} to {} on {}", currentTargetFileName, targetFileName, sourceStoreUri); + tmpTargetFile.toFile().renameTo(targetFile.toFile()); + } catch (Exception e) { + logger.error(getMarker(), e.getMessage()); + } + } + if (renameAfterFTP) { + try { + FtpDirectoryLister lister = (FtpDirectoryLister) targetStoreUri.getLister(); + FTPClient client = lister.connect(); + logger.info(getMarker(), "Renaming FTP file from {} to {} on {}", currentTargetFileName, targetFileName, sourceStoreUri); + client.rename(currentTargetFileName, targetFileName); + } catch (Exception e) { + logger.error(getMarker(), e.getMessage()); + } finally { + targetStoreUri.cleanUp(); + } + } + } + + @StepEntry + public Object[] execute(StoreUri sourceStoreUri, String sourceFileName, StoreUri targetStoreUri, String targetFileName) throws Exception { + try { + getJobRuntime().setCancelable(false); + + Store sourceStore = getManager().getStore(sourceStoreUri.getStoreId()); + Store targetStore = getManager().getStore(targetStoreUri.getStoreId()); + + setDescription("{} -> {} : {}", sourceStore.getName(), targetStore.getName(), sourceFileName); + + sourceStoreUri.addProgressListener(e -> setProgress(e.getProgress())); + + StoreUri currentTargetStoreUri = getTargetStoreUri(targetStoreUri); + + try { + RemoteFile remoteFile = currentTargetStoreUri.getRemoteFile(targetFileName); + if (remoteFile != null) { + logger.info("File {} already exists on target {}, skipping transfer", targetFileName, targetStore.getName()); + } + } catch (Exception e) { + } + + copy(sourceStoreUri, sourceFileName, currentTargetStoreUri, targetFileName); + } catch (Exception e) { + logger.error(getMarker(), "Error in transfer of {} when copying from {} to {}.", sourceFileName, sourceStoreUri, targetStoreUri); + throw e; + } finally { + getJobRuntime().setDescription(null); + if (sourceStoreUri != null) + sourceStoreUri.cleanUp(); + if (targetStoreUri != null) + targetStoreUri.cleanUp(); + } + return null; + } + + protected StoreUri getTargetStoreUri(StoreUri targetStoreUri) { + return targetStoreUri; + } + + protected String getTmpExtension() { + return DOT_PART; + } +} diff --git a/server/-product/production/OMAR/jobs/steps/UpdateGhostMediaDataStep.java b/server/-product/production/OMAR/jobs/steps/UpdateGhostMediaDataStep.java new file mode 100644 index 00000000..39cb7b30 --- /dev/null +++ b/server/-product/production/OMAR/jobs/steps/UpdateGhostMediaDataStep.java @@ -0,0 +1,89 @@ +package user.jobengine.server.steps; + +import java.util.List; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.apache.logging.log4j.Marker; + +import user.commons.remotestore.RemoteStoreProtocol; +import user.jobengine.db.IItemManager; +import user.jobengine.db.Media; +import user.jobengine.db.MediaFile; +import user.jobengine.db.Store; +import user.jobengine.server.IJobEngine; +import user.jobengine.server.IJobRuntime; +import user.jobengine.server.steps.shared.MetadataType; +import user.jobengine.server.steps.shared.MetadataTypeDetector; + +public class UpdateGhostMediaDataStep extends JobStep { + private static final Logger logger = LogManager.getLogger(); + private Marker marker; + + @StepEntry + public Object[] execute(Media mediaCubeMedia, IJobEngine jobEngine, IJobRuntime jobRuntime) throws Exception { + marker = jobRuntime.getSessionMarker(); + + IItemManager manager = jobEngine.getItemManager(); + //Refresh from db + List mediaFiles = manager.getMedia(mediaCubeMedia.getId()).getMediaFiles(); + if (mediaFiles != null && mediaFiles.size() == 2) { + MediaFile lowres = null; + MediaFile highres = null; + + for (MediaFile mf : mediaFiles) { + if (mf.getStore().getSourceStoreUri(RemoteStoreProtocol.HTTP) != null) + lowres = mf; + else + highres = mf; + } + + if (highres == null) { + logger.info(marker, "Nincs highres mediaId: {}", mediaCubeMedia.getId()); + return null; + } + if (lowres == null) { + logger.info(marker, "Nincs lowres mediaId: {}", mediaCubeMedia.getId()); + return null; + } + + String id = MetadataTypeDetector.truncateExtension(highres.getRelativePath()); + id = MetadataTypeDetector.truncateVersion(id); + boolean detect = MetadataTypeDetector.GuessMetadataType(id) == MetadataType.OctopusPlaceholder + || MetadataTypeDetector.GuessMetadataType(id) == MetadataType.OctopusStory; + if (!detect) { + logger.info(marker, "Nem bejátszó mediaId: {}, file: {}", mediaCubeMedia.getId(), highres.getRelativePath()); + return null; + } + + Store highresStore = manager.getSystemStore(false); + final long sourceMediaId = lowres.getId(); + final long highresMediaFileId = highres.getId(); + final String highresRealtivePath = highres.getRelativePath(); + + manager.executeQuery("SELECT mediaid FROM mediafile WHERE relativepath=? and storeid=? and id!=?", rs -> { + long mediaId = rs.getLong(1); + Media media = manager.getMedia(mediaId); + if (media.getMediaFilesCount() == 1) { + logger.info(marker, "Hiányzó szellem lowres hozzáadása {} alapján", media.getId()); + + MediaFile mf = (MediaFile) manager.get(MediaFile.class, sourceMediaId); + mf.setMedia(media); + mf.setId(0); + manager.add(mf); + media.setLength(mediaCubeMedia.getLength()); + manager.modify(media); + } + return true; + }, st -> { + st.setString(1, highresRealtivePath); + st.setLong(2, highresStore.getId()); + st.setLong(3, highresMediaFileId); + }); + + } + + return null; + } + +} diff --git a/server/-product/production/OMAR/jobs/steps/ValidateProResStep.java b/server/-product/production/OMAR/jobs/steps/ValidateProResStep.java new file mode 100644 index 00000000..e36f784d --- /dev/null +++ b/server/-product/production/OMAR/jobs/steps/ValidateProResStep.java @@ -0,0 +1,32 @@ +package user.jobengine.server.steps; + +import java.util.HashMap; +import java.util.Map; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + +import user.commons.mediaarea.MediaArea; + +public class ValidateProResStep extends JobStep { + private static final Logger logger = LogManager.getLogger(); + + @StepEntry + public Object[] execute(Map proResFiles) { + Map files = new HashMap(); + + if (!proResFiles.isEmpty()) { + for (String fileName : proResFiles.keySet()) { + MediaArea mediaArea = proResFiles.get(fileName); + String videoFormat = mediaArea.getFormat(); + + if (mediaArea.getFrameRate() == 23.976 && videoFormat.equals("PRORES")) { + files.put(fileName, mediaArea); + } + } + } else { + logger.info("proResFiles is empty!"); + } + return new Object[] { files }; + } +} diff --git a/server/-product/production/OMAR/jobs/steps/shared/EscortFiles.java b/server/-product/production/OMAR/jobs/steps/shared/EscortFiles.java new file mode 100644 index 00000000..c888f722 --- /dev/null +++ b/server/-product/production/OMAR/jobs/steps/shared/EscortFiles.java @@ -0,0 +1,348 @@ +package user.jobengine.server.steps.shared; + +import java.io.ByteArrayInputStream; +import java.io.File; +import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; +import java.io.StringWriter; +import java.io.UnsupportedEncodingException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.nio.file.attribute.FileAttribute; +import java.nio.file.attribute.PosixFilePermission; +import java.nio.file.attribute.PosixFilePermissions; +import java.text.SimpleDateFormat; +import java.util.Calendar; +import java.util.Date; +import java.util.Set; + +import javax.xml.parsers.DocumentBuilder; +import javax.xml.parsers.DocumentBuilderFactory; +import javax.xml.transform.OutputKeys; +import javax.xml.transform.Transformer; +import javax.xml.transform.TransformerConfigurationException; +import javax.xml.transform.TransformerException; +import javax.xml.transform.TransformerFactory; +import javax.xml.transform.TransformerFactoryConfigurationError; +import javax.xml.transform.dom.DOMSource; +import javax.xml.transform.stream.StreamResult; + +import org.apache.commons.lang.StringUtils; +import org.apache.commons.net.ftp.FTPClient; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.apache.logging.log4j.Marker; +import org.apache.logging.log4j.message.Message; +import org.w3c.dom.DOMImplementation; +import org.w3c.dom.Document; +import org.w3c.dom.Element; + +import com.ibm.nosql.json.api.BasicDBObject; +import com.ibm.nosql.json.util.JSON; + +import user.commons.CalendarUtils; +import user.commons.log4j2.marker.MediaCubeMarker; +import user.commons.StoreUri; +import user.commons.remotestore.FtpDirectoryLister; + +public class EscortFiles { + private static final String RECORDTIMESTAMP = "RecordTimeStamp"; + private static final String MODIFIEDTIMESTAMP = "ModifiedTimeStamp"; + public static final String DOT_CATCHED = ".catched"; + public static final String DOT_JSON = ".json"; + private static final Logger logger = LogManager.getLogger(); + private static final String EXTENDEDAGENCY = "ExtendedAgency"; + private static final String EXTENDEDDESCRIPTION = "ExtendedDescription"; + private static final String KILLDATE = "KillDate"; + private static final String FORMAT_KILLDATE = "MM-dd-yyyy"; + private static final String EXTENDEDID = "extendedId"; + private static final String ID = "ID"; + private static final String KILLDATE_FILENAME = "%s.%s.killdate"; + private static final String FORMAT_KILLDATENAME = "yyyyMMdd"; + public static final String STATUSFOLDER = ".STATUS"; + public static final String CONFLICTFOLDER = ".CONFLICT"; + + public static String composeKillDate(int days) { + Calendar killDate = Calendar.getInstance(); + killDate.add(Calendar.DAY_OF_YEAR, days); + SimpleDateFormat dateFormat = new SimpleDateFormat(FORMAT_KILLDATENAME); + return dateFormat.format(killDate.getTime()); + } + /* + * + * 02-02-2018 TEST + * TEST AGENT AGENT + */ + + private static String composeKillDateFileName(String fileName, int days) { + return String.format(KILLDATE_FILENAME, fileName, composeKillDate(days)); + } + + public static void createCatchedFile(Path escortFile) throws IOException { + Path catchedFilePath = Paths.get(escortFile.toString() + DOT_CATCHED); + Files.createFile(catchedFilePath); + } + + public static void createFellow(String escortFile, String extension) throws IOException { + Files.copy(Paths.get(escortFile), Paths.get(escortFile + "." + extension)); + } + + /*** + * A media eleresi utjan alapjan a .STATUS almappaban letrehozza a .catch fajlt. + * + * @param mediaFile + * @throws IOException + */ + public static void createMediaCatch(Path mediaFile) throws IOException { + Path catchedFile = createMediaCatchFilePath(mediaFile); + ensureUNCFolder(catchedFile.getParent()); + Files.createFile(catchedFile); + } + + public static Path createMediaCatchFilePath(Path mediaFile) { + String fileName = mediaFile.getFileName().toString() + DOT_CATCHED; + return Paths.get(mediaFile.getParent().toString(), STATUSFOLDER, fileName); + } + + public static void createMetadata(String filePath, String fileName, String metadata) throws IOException { + ensureUNCFolder(filePath, STATUSFOLDER); + String metadataFileName = fileName + DOT_JSON; + Path metadataPath = Paths.get(filePath, STATUSFOLDER, metadataFileName); + Files.write(metadataPath, metadata.getBytes()); + } + + public static boolean createMetadataIfNotExists(String filePath, String fileName, String metadata) + throws IOException { + boolean result = false; + if (!EscortFiles.isMetadataExists(filePath, fileName)) { + EscortFiles.createMetadata(filePath, fileName, metadata); + result = true; + } + return result; + } + + public static void createMorpheusXML(String filePath, String fileName, String content) throws IOException { + ensureUNCFolder(filePath, STATUSFOLDER); + Path xmlPath = Paths.get(filePath, fileName); + if (Files.exists(xmlPath)) + throw new IOException(String.format("Az '%s' állomány már létezik.", xmlPath)); + Files.write(xmlPath, content.getBytes()); + } + + public static byte[] createNEXIODatesMeta(String fileName, Date recorded, Date modified) throws Exception { + DocumentBuilderFactory dbf = DocumentBuilderFactory.newInstance(); + DocumentBuilder db = dbf.newDocumentBuilder(); + DOMImplementation impl = db.getDOMImplementation(); + Document xmlDocument = impl.createDocument(null, null, null); + + Element root = xmlDocument.createElement(ID); + root.setAttribute(EXTENDEDID, fileName); + // 07-13-2020 (19:36:52) + // 05-18-2013 (18:52:24) + SimpleDateFormat df = new SimpleDateFormat("MM-dd-yyyy (HH:mm:ss)"); + root.appendChild(xmlDocument.createElement(MODIFIEDTIMESTAMP)) + .appendChild(xmlDocument.createTextNode(df.format(modified))); + root.appendChild(xmlDocument.createElement(RECORDTIMESTAMP)) + .appendChild(xmlDocument.createTextNode(df.format(recorded))); + xmlDocument.appendChild(root); + + return xmDocumentToString(xmlDocument); + } + + public static byte[] createNEXIOKillDateFile(String fileName, Date killDate, String description, String agency) + throws Exception { + DocumentBuilderFactory dbf = DocumentBuilderFactory.newInstance(); + DocumentBuilder db = dbf.newDocumentBuilder(); + DOMImplementation impl = db.getDOMImplementation(); + Document xmlDocument = impl.createDocument(null, null, null); + + Element root = xmlDocument.createElement(ID); + root.setAttribute(EXTENDEDID, fileName); + if (killDate != null) { + String sKillDate = CalendarUtils.toString(CalendarUtils.createCalendar(killDate), FORMAT_KILLDATE); + root.appendChild(xmlDocument.createElement(KILLDATE)).appendChild(xmlDocument.createTextNode(sKillDate)); + } + + if (StringUtils.isNotBlank(description)) + root.appendChild(xmlDocument.createElement(EXTENDEDDESCRIPTION)) + .appendChild(xmlDocument.createTextNode(description)); + if (StringUtils.isNotBlank(agency)) + root.appendChild(xmlDocument.createElement(EXTENDEDAGENCY)).appendChild(xmlDocument.createTextNode(agency)); + xmlDocument.appendChild(root); + + return xmDocumentToString(xmlDocument); + } + + public static Document createNEXIOMeta(byte[] content) throws Exception { + DocumentBuilderFactory dbf = DocumentBuilderFactory.newInstance(); + DocumentBuilder db = dbf.newDocumentBuilder(); + DOMImplementation impl = db.getDOMImplementation(); + Document xmlDocument = null; + + try (InputStream is = new ByteArrayInputStream(content)) { + xmlDocument = db.parse(is); + } catch (Exception e) { + logger.catching(e); + } + + return xmlDocument; + } + + public static void createUNCKillDate(String filePath, String fileName, int days, Marker marker) throws IOException { + ensureUNCFolder(filePath, STATUSFOLDER); + String killDateFileName = composeKillDateFileName(fileName, days); + Path killDatePath = Paths.get(filePath, STATUSFOLDER, killDateFileName); + if (Files.exists(killDatePath)) + logger.warn(marker, "Az '{}' állomány már létezik.", killDatePath); + else + Files.createFile(killDatePath); + } + + @SuppressWarnings("unchecked") + public static T decode(Path escortFile) { + T result = null; + try { + byte[] bytes = Files.readAllBytes(escortFile); + String content = new String(bytes); + result = (T) JSON.parse(content); + } catch (Exception e) { + logger.error("Decode error. System message is: ", e.getMessage()); + } + return result; + } + + public static void ensureUNCFolder(Path filePath) throws IOException { + File folder = filePath.toFile(); + if (!folder.exists() || !folder.isDirectory()) { + try { + Set perms = PosixFilePermissions.fromString("rwxrwxrwx"); + FileAttribute> attr = PosixFilePermissions.asFileAttribute(perms); + Files.createDirectories(filePath, attr); + } catch (Exception e) { + // logger.catching(e); + try { + Files.createDirectories(filePath); + } catch (Exception e1) { + logger.catching(e1); + throw e1; + } + } + } + } + + public static void ensureUNCFolder(String filePath, String folderName) throws IOException { + Path statusPath = Paths.get(filePath, folderName); + ensureUNCFolder(statusPath); + } + + public static boolean isCatchedFileExists(Path escortFile) { + Path catchedFilePath = Paths.get(escortFile.toString() + DOT_CATCHED); + return Files.exists(catchedFilePath); + } + + /*** + * A media elérési útján alapján a .STATUS almappában vizsgálja .catch fajl + * létezését. + * + * @param mediaFile + * @return + */ + public static boolean isMediaCatched(Path mediaFile) { + Path catchedFile = createMediaCatchFilePath(mediaFile); + return Files.exists(catchedFile); + } + + public static boolean isMetadataExists(String filePath, String fileName) throws IOException { + boolean result = false; + String metadataFileName = fileName + DOT_JSON; + Path metadataPath = Paths.get(filePath, STATUSFOLDER, metadataFileName); + result = Files.exists(metadataPath); + return result; + } + + public static void notifyRecipient(Path escortFile, Logger logger, Message msg) { + if (Files.exists(escortFile)) { + try { + BasicDBObject downloadable = EscortFiles.decode(escortFile); + String recipientKey = "recipient"; + if (downloadable.containsKey(recipientKey)) { + String recipient = downloadable.getString(recipientKey); + logger.info(new MediaCubeMarker(recipient, "MediaCube rendszerüzenet"), msg); + } + } catch (Exception e) { + logger.catching(e); + } + + } + } + + public static void remove(Path file) { + try { + file.toFile().delete(); + } catch (Exception e) { + logger.error("Unable to delete {}", file.toAbsolutePath().toString()); + } + } + + public static void removeCatchedFile(Path escortFile) { + remove(Paths.get(escortFile.toString() + DOT_CATCHED)); + } + + /*** + * A media eleresi utjan alapjan a .STATUS almappabol torli a .catch fajlt. + * + * @param mediaFile + * @throws IOException + */ + public static void removeMediaCatch(Path mediaFile) { + Path catchedFile = createMediaCatchFilePath(mediaFile); + remove(catchedFile); + } + + public static void setNEXIOKillDate(int killDateDays, String targetFileName, String nexioAgency, StoreUri targetUri) + throws Exception { + OutputStream outStream = null; + try { + FTPClient targetFTP = ((FtpDirectoryLister) targetUri.getLister()).connect(); + Calendar killDate = CalendarUtils.createCalendar(new Date()); + killDate.add(Calendar.DAY_OF_YEAR, killDateDays); + if (targetFileName.toLowerCase().contains(".mxf")) + targetFileName = targetFileName.substring(0, targetFileName.lastIndexOf('.')); + byte[] killDateFile = EscortFiles.createNEXIOKillDateFile(targetFileName, killDate.getTime(), null, + nexioAgency); + String xml = targetFileName + ".xml"; + outStream = targetFTP.storeFileStream(xml); + if (outStream == null) { + throw new NullPointerException( + "Can not open: " + targetFileName.substring(0, targetFileName.lastIndexOf('.')) + ".xml" + + " Reply:" + targetFTP.getReplyString()); + } + outStream.write(killDateFile); + outStream.flush(); + } catch (Exception e) { + throw e; + } finally { + if (outStream != null) + outStream.close(); + targetUri.cleanUp(); + } + } + + private static byte[] xmDocumentToString(Document xmlDocument) throws TransformerFactoryConfigurationError, + TransformerConfigurationException, TransformerException, IOException, UnsupportedEncodingException { + DOMSource domSource = new DOMSource(xmlDocument); + TransformerFactory tf = TransformerFactory.newInstance(); + Transformer transformer = tf.newTransformer(); + transformer.setOutputProperty(OutputKeys.METHOD, "xml"); + transformer.setOutputProperty(OutputKeys.ENCODING, "UTF-16"); + transformer.setOutputProperty(OutputKeys.INDENT, "yes"); + StringWriter sw = new StringWriter(); + StreamResult sr = new StreamResult(sw); + transformer.transform(domSource, sr); + String result = sw.toString(); + sw.close(); + return result.getBytes("UTF-16"); + } +} diff --git a/server/-product/production/OMAR/jobs/steps/shared/ExternalCommand.java b/server/-product/production/OMAR/jobs/steps/shared/ExternalCommand.java new file mode 100644 index 00000000..307f36d0 --- /dev/null +++ b/server/-product/production/OMAR/jobs/steps/shared/ExternalCommand.java @@ -0,0 +1,79 @@ +package user.jobengine.server.steps.shared; + +import java.io.BufferedReader; +import java.io.InputStreamReader; +import java.util.ArrayList; +import java.util.List; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + +public class ExternalCommand { + private static final Logger logger = LogManager.getLogger(); + private ExternalProfile profile; + + public ExternalCommand(ExternalProfile profile) { + this.profile = profile; + } + + public String execute(String input, String output, boolean firstResponse, IExternalCallback responseCallBack) throws Exception { + List arguments = getArguments(input, output); + List command = new ArrayList<>(); + command.add(profile.getExecutable()); + command.addAll(arguments); + + ProcessBuilder processBuilder = new ProcessBuilder(); + processBuilder.command(command); + + String result = null; + try { + logger.info("Executing : {}", processBuilder.command()); + + Process process = processBuilder.start(); + try (BufferedReader reader = new BufferedReader(new InputStreamReader(process.getInputStream()))) { + String line = null; + while ((line = reader.readLine()) != null) { + logger.debug("Process response: {}", line); + if (responseCallBack != null) + responseCallBack.onResponse(line); + //System.out.println(line); + if (line != null && line.length() > 0) { + result = line; + if (firstResponse) + break; + } + } + int exitCode = process.waitFor(); + if (exitCode != 0) { + StringBuilder msg = new StringBuilder(); + try (BufferedReader errReader = new BufferedReader(new InputStreamReader(process.getErrorStream()))) { + String errline = null; + while ((errline = errReader.readLine()) != null) { + msg.append(errline); + } + } catch (Exception ex) { + } + + throw new Exception("Exited with error code : " + exitCode + ". " + msg); + } + } catch (Exception e) { + throw e; + } + } catch (Exception e) { + logger.error(e); + throw e; + } + + return result; + } + + private List getArguments(String input, String output) { + List result = new ArrayList<>(); + + profile.getArguments().forEach(i -> { + result.add(i.replace("%i", input).replace("%o", output)); + }); + return result; + } + +} \ No newline at end of file diff --git a/server/-product/production/OMAR/jobs/steps/shared/ExternalCommandExecutor.java b/server/-product/production/OMAR/jobs/steps/shared/ExternalCommandExecutor.java new file mode 100644 index 00000000..a34ff7f0 --- /dev/null +++ b/server/-product/production/OMAR/jobs/steps/shared/ExternalCommandExecutor.java @@ -0,0 +1,32 @@ +package user.jobengine.server.steps.shared; + +import user.commons.configuration.SystemConfiguration; + +public class ExternalCommandExecutor { + + public void execute(String profileName, String input, String output, IExternalCallback responseCallBack) throws Exception { + ExternalCommand externalCommand = getExternalCommand(profileName); + externalCommand.execute(input, output, false, responseCallBack); + } + + private ExternalCommand getExternalCommand(String profileName) throws Exception { + ExternalProfilesConfig config = SystemConfiguration.getInstance().load("settings/external-commands.yaml", ExternalProfilesConfig.class); + + if (config == null) + throw new Exception("Missing external-commands.yaml configuration"); + + ExternalProfile selectedProfile = null; + for (ExternalProfile profile : config.getProfiles()) { + if (profileName.equals(profile.getName())) { + selectedProfile = profile; + break; + } + } + + if (selectedProfile == null) + throw new Exception("Missing profile " + profileName + " in external-commands.yaml configuration"); + + return new ExternalCommand(selectedProfile); + } + +} diff --git a/server/-product/production/OMAR/jobs/steps/shared/ExternalProfile.java b/server/-product/production/OMAR/jobs/steps/shared/ExternalProfile.java new file mode 100644 index 00000000..68e22f4c --- /dev/null +++ b/server/-product/production/OMAR/jobs/steps/shared/ExternalProfile.java @@ -0,0 +1,33 @@ +package user.jobengine.server.steps.shared; + +import java.util.List; + +public class ExternalProfile { + private String executable; + private String name; + private List arguments; + + public List getArguments() { + return arguments; + } + + public String getExecutable() { + return executable; + } + + public String getName() { + return name; + } + + public void setArguments(List arguments) { + this.arguments = arguments; + } + + public void setExecutable(String executable) { + this.executable = executable; + } + + public void setName(String name) { + this.name = name; + } +} \ No newline at end of file diff --git a/server/-product/production/OMAR/jobs/steps/shared/ExternalProfilesConfig.java b/server/-product/production/OMAR/jobs/steps/shared/ExternalProfilesConfig.java new file mode 100644 index 00000000..7ef77291 --- /dev/null +++ b/server/-product/production/OMAR/jobs/steps/shared/ExternalProfilesConfig.java @@ -0,0 +1,15 @@ +package user.jobengine.server.steps.shared; + +import java.util.List; + +public class ExternalProfilesConfig { + private List profiles; + + public List getProfiles() { + return profiles; + } + + public void setProfiles(List profiles) { + this.profiles = profiles; + } +} \ No newline at end of file diff --git a/server/-product/production/OMAR/jobs/steps/shared/FileSearchFilterOptions.java b/server/-product/production/OMAR/jobs/steps/shared/FileSearchFilterOptions.java new file mode 100644 index 00000000..b5a8d0e8 --- /dev/null +++ b/server/-product/production/OMAR/jobs/steps/shared/FileSearchFilterOptions.java @@ -0,0 +1,41 @@ +package user.jobengine.server.steps.shared; + +import java.nio.file.Path; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +import com.ibm.nosql.json.api.BasicDBObject; + +public class FileSearchFilterOptions { + + private BasicDBObject filter; + + public FileSearchFilterOptions(BasicDBObject filter) { + this.filter = filter; + } + + public boolean acceptFile(Path file) { + if (filter == null) + return true; + + if (filter.containsKey("fileName")) { + //.*\.(sh|ini|conf|vhost|xml|php)$ + String fileNamePattern = filter.getString("fileName"); + if (fileNamePattern == null || fileNamePattern.trim().length() == 0) + return true; + + Pattern pattern = Pattern.compile(fileNamePattern, Pattern.CASE_INSENSITIVE); + + Matcher matcher = pattern.matcher(file.getFileName().toString()); + if (matcher.find()) + return true; + + } + + return false; + } + + public boolean preAcceptDirectory(Path file) { + return true; + } +} diff --git a/server/-product/production/OMAR/jobs/steps/shared/IExternalCallback.java b/server/-product/production/OMAR/jobs/steps/shared/IExternalCallback.java new file mode 100644 index 00000000..1e071913 --- /dev/null +++ b/server/-product/production/OMAR/jobs/steps/shared/IExternalCallback.java @@ -0,0 +1,5 @@ +package user.jobengine.server.steps.shared; + +public interface IExternalCallback { + void onResponse(String data); +} \ No newline at end of file diff --git a/server/-product/production/OMAR/jobs/steps/shared/ItemManagerExtensions.java b/server/-product/production/OMAR/jobs/steps/shared/ItemManagerExtensions.java new file mode 100644 index 00000000..32f17cb5 --- /dev/null +++ b/server/-product/production/OMAR/jobs/steps/shared/ItemManagerExtensions.java @@ -0,0 +1,94 @@ +package user.jobengine.server.steps.shared; + +import java.nio.file.Path; + +import com.ibm.nosql.json.api.BasicDBObject; + +import user.commons.RemoteFile; +import user.commons.StoreUri; +import user.commons.remotestore.RemoteStoreProtocol; +import user.jobengine.db.IItemManager; +import user.jobengine.db.IResultSetConsumer; +import user.jobengine.db.Store; + +public class ItemManagerExtensions { + + public static BasicDBObject getArchiveInfo(IItemManager manager, long houseid) { + final BasicDBObject[] result = { null }; + StringBuilder query = new StringBuilder(); + query.append("select count(*) as count, sum(length) as duration FROM media"); + query.append(" "); + query.append(String.format("where houseid='%d' and itemtypeid = 82", houseid)); + query.append(" "); + query.append("group by houseid"); + IResultSetConsumer consumer = rs -> { + BasicDBObject o = new BasicDBObject(); + o.put("count", rs.getLong("count")); + o.put("duration", rs.getLong("duration")); + result[0] = o; + return false; + }; + manager.executeQuery(query.toString(), consumer, null); + return result[0]; + } + + public static long getExistingRundownMedia(IItemManager manager, String houseid) { + final long[] result = new long[] { 0 }; + final String[] idToCheck = new String[] { houseid }; + int pos = houseid.lastIndexOf("-"); + //a hivas a CopyForArchiveNEXIOMaterialsStep-bol is johet, ott meg nincs idobelyegezve a nev! + if (pos > 0 && houseid.length() - pos > 4) + idToCheck[0] = houseid.substring(0, pos); + MetadataType metadataType = MetadataTypeDetector.GuessMetadataType(idToCheck[0]); + if (metadataType == MetadataType.OctopusPlaceholder) { + StringBuilder query = new StringBuilder(); + query.append("select mediaid, mediafilehouseid, filename"); + query.append(" "); + query.append(String.format("from vw_rundown_items where mediafilehouseid like '%s%%'", idToCheck[0])); + query.append(" "); + query.append("order by filename, mediaid"); + IResultSetConsumer consumer = rs -> { + String fileName = rs.getString("filename"); + if (idToCheck[0].equals(fileName)) { + result[0] = rs.getLong("mediaid"); + return false; + } else + return true; + }; + manager.executeQuery(query.toString(), consumer, null); + } + + return result[0]; + } + + static public boolean isArchived(IItemManager manager, Path filePath) { + boolean result = false; + String name = filePath.getFileName().toString(); + String[] tsmName = new String[] { null }; + String query = String.format("SELECT relativepath FROM MEDIAFILE WHERE houseid = '%s'", name); + manager.executeQuery(query, rs -> { + tsmName[0] = rs.getString("relativepath"); + return false; + }, null); + + Store tsmStore = manager.getSystemStore(false); + if (tsmStore == null) + throw new NullPointerException("A TSM bejegyzés nem található!"); + + StoreUri tsmStoreUri = tsmStore.getSourceStoreUri(RemoteStoreProtocol.TSM); + if (tsmStoreUri == null) + throw new NullPointerException("A TSM forrás elérése nem található!"); + + if (tsmName[0] != null) { + try { + RemoteFile remoteFile = tsmStoreUri.getRemoteFile(tsmName[0]); + result = remoteFile != null; + } catch (Exception e) { + result = false; + } finally { + tsmStoreUri.cleanUp(); + } + } + return result; + } +} diff --git a/server/-product/production/OMAR/jobs/steps/shared/MediaCubeClient.java b/server/-product/production/OMAR/jobs/steps/shared/MediaCubeClient.java new file mode 100644 index 00000000..0915cc33 --- /dev/null +++ b/server/-product/production/OMAR/jobs/steps/shared/MediaCubeClient.java @@ -0,0 +1,77 @@ +package user.jobengine.server.steps.shared; + +import javax.ws.rs.client.Entity; +import javax.ws.rs.client.Invocation.Builder; +import javax.ws.rs.core.MediaType; +import javax.ws.rs.core.MultivaluedMap; +import javax.ws.rs.core.Response; +import javax.ws.rs.core.Response.Status; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.jboss.resteasy.client.jaxrs.ResteasyClient; +import org.jboss.resteasy.client.jaxrs.ResteasyClientBuilder; +import org.jboss.resteasy.client.jaxrs.ResteasyWebTarget; +import org.jboss.resteasy.specimpl.MultivaluedMapImpl; + +import com.ibm.nosql.json.JSONUtil; +import com.ibm.nosql.json.api.BasicDBObject; + +public class MediaCubeClient { + private static Logger logger = LogManager.getLogger(); + private ResteasyWebTarget webTarget; + + public MediaCubeClient(String address) { + ResteasyClient client = new ResteasyClientBuilder().build(); + webTarget = client.target(address); + } + + BasicDBObject getDbObject(String json) { + BasicDBObject result = (BasicDBObject) JSONUtil.jsonToDbObject(json); + + if (result == null) + throw new NullPointerException("API Result is null!"); + + if (result.containsKey("exception")) { + BasicDBObject e = (BasicDBObject) result.get("exception"); + throw new RuntimeException(e.getString("message")); + } + //{"exception":{"message":"Invalid credentials.","publicName":"AuthenticationFailedException"}} + return result; + } + + public BasicDBObject getStatus(long jobId) { + MultivaluedMap vars = new MultivaluedMapImpl<>(); + vars.add("jobId", jobId); + Response response = query("services/rest/jobengine/jobstatus", vars).get(); + if (response.getStatus() != Status.OK.getStatusCode()) { + logger.error(response.readEntity(String.class)); + System.out.println(response.readEntity(String.class)); + return null; + } + String result = response.readEntity(String.class); + return getDbObject(result); + } + + private Builder query(String path, MultivaluedMap vars) { + ResteasyWebTarget target = webTarget.path(path).queryParams(vars); + Builder result = target.request(); + return result; + } + + public long startjob(String template, String name, BasicDBObject jobParams) throws Exception { + MultivaluedMap vars = new MultivaluedMapImpl<>(); + vars.add("template", template); + vars.add("name", name); + Response response = query("services/rest/jobengine/startjob", vars).post(Entity.entity(jobParams.toString(), MediaType.APPLICATION_JSON)); + + if (response.getStatus() != Status.OK.getStatusCode()) { + logger.error(response.readEntity(String.class)); + return 0; + } + + String resultObject = response.readEntity(String.class); + return Long.parseLong(resultObject); + } + +} diff --git a/server/-product/production/OMAR/jobs/steps/shared/MediaFileSearchFilterOptions.java b/server/-product/production/OMAR/jobs/steps/shared/MediaFileSearchFilterOptions.java new file mode 100644 index 00000000..ee467e5b --- /dev/null +++ b/server/-product/production/OMAR/jobs/steps/shared/MediaFileSearchFilterOptions.java @@ -0,0 +1,42 @@ +package user.jobengine.server.steps.shared; + +import java.nio.file.Path; +import java.util.regex.Matcher; +import java.util.regex.Pattern; + +import com.ibm.nosql.json.api.BasicDBObject; + +public class MediaFileSearchFilterOptions extends FileSearchFilterOptions { + + private BasicDBObject filter; + + public MediaFileSearchFilterOptions(BasicDBObject filter) { + super(filter); + } + + @Override + public boolean acceptFile(Path file) { + if (filter == null) + return true; + + if (filter.containsKey("fileName")) { + // .*\.(sh|ini|conf|vhost|xml|php)$ + String fileNamePattern = filter.getString("fileName"); + if (fileNamePattern == null || fileNamePattern.trim().length() == 0) + return true; + + Pattern pattern = Pattern.compile(fileNamePattern, Pattern.CASE_INSENSITIVE); + + Matcher matcher = pattern.matcher(file.getFileName().toString()); + if (matcher.find()) + return true; + } + + return false; + } + + @Override + public boolean preAcceptDirectory(Path file) { + return true; + } +} diff --git a/server/-product/production/OMAR/jobs/steps/shared/MetadataType.java b/server/-product/production/OMAR/jobs/steps/shared/MetadataType.java new file mode 100644 index 00000000..0e27bf34 --- /dev/null +++ b/server/-product/production/OMAR/jobs/steps/shared/MetadataType.java @@ -0,0 +1,5 @@ +package user.jobengine.server.steps.shared; + +public enum MetadataType { + TrafficMaterial, TrafficPromo, TrafficAD, OctopusStory, OctopusPlaceholder, Generic +} diff --git a/server/-product/production/OMAR/jobs/steps/shared/MetadataTypeDetector.java b/server/-product/production/OMAR/jobs/steps/shared/MetadataTypeDetector.java new file mode 100644 index 00000000..9e9e6be9 --- /dev/null +++ b/server/-product/production/OMAR/jobs/steps/shared/MetadataTypeDetector.java @@ -0,0 +1,49 @@ +package user.jobengine.server.steps.shared; + +import org.apache.commons.lang.StringUtils; + +public class MetadataTypeDetector { + + private static final String HYPHEN = "-"; + private static final String DOT = "."; + + private static final String REGEXP_TRAFFICMATERIALID = "^M{1}[0-9]{6}[A-Z]{1}"; + private static final String REGEXP_TRAFFICADID = "^R{1}[0-9]{6}[A-Z]{1}"; + private static final String REGEXP_TRAFFICPROMOID = "^P{1}[0-9]{6}[A-Z]{1}"; + private static final String REGEXP_OCTOPUSSTORYID = "^[0-9]+"; + private static final String REGEXP_OCTOPUSPLACEHOLDERID = "^[0-9]+_[0-9]+"; + private static final String REGEXP_OCTOPUSPLACEHOLDERVERSIONEDID = "^[0-9]+_[0-9]+-[0-9]{3}"; + + public static MetadataType GuessMetadataType(String id) { + if (StringUtils.isBlank(id)) + return MetadataType.Generic; + if (id.matches(REGEXP_TRAFFICMATERIALID)) + return MetadataType.TrafficMaterial; + if (id.matches(REGEXP_TRAFFICPROMOID)) + return MetadataType.TrafficPromo; + if (id.matches(REGEXP_TRAFFICADID)) + return MetadataType.TrafficAD; + if (id.matches(REGEXP_OCTOPUSSTORYID)) + return MetadataType.OctopusStory; + if (id.matches(REGEXP_OCTOPUSPLACEHOLDERID)) + return MetadataType.OctopusPlaceholder; + if (id.matches(REGEXP_OCTOPUSPLACEHOLDERVERSIONEDID)) + return MetadataType.OctopusPlaceholder; + return MetadataType.Generic; + } + + public static String truncateExtension(String name) { + String result = name; + if (result != null && result.contains(DOT)) + result = result.substring(0, result.lastIndexOf(DOT)); + return result; + } + + public static String truncateVersion(String name) { + String result = name; + if (result != null && result.contains(HYPHEN)) + result = result.split(HYPHEN)[0]; + return result; + } + +} diff --git a/server/-product/production/OMAR/jobs/steps/shared/TestLib.java b/server/-product/production/OMAR/jobs/steps/shared/TestLib.java new file mode 100644 index 00000000..d12738ae --- /dev/null +++ b/server/-product/production/OMAR/jobs/steps/shared/TestLib.java @@ -0,0 +1,9 @@ +package user.jobengine.server.steps.shared; + +public class TestLib { + + public void hello() { + TestLib1 lib = new TestLib1(); + lib.helo(); + } +} diff --git a/server/-product/production/OMAR/jobs/steps/shared/TestLib1.java b/server/-product/production/OMAR/jobs/steps/shared/TestLib1.java new file mode 100644 index 00000000..99d47e5f --- /dev/null +++ b/server/-product/production/OMAR/jobs/steps/shared/TestLib1.java @@ -0,0 +1,8 @@ +package user.jobengine.server.steps.shared; + +public class TestLib1 { + + public void hello() { + System.out.println("Hello from lib1"); + } +} diff --git a/server/-product/production/OMAR/jobs/templates/archive-limited.xml b/server/-product/production/OMAR/jobs/templates/archive-limited.xml new file mode 100644 index 00000000..6ea81306 --- /dev/null +++ b/server/-product/production/OMAR/jobs/templates/archive-limited.xml @@ -0,0 +1,43 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/server/-product/production/OMAR/jobs/templates/archive-material.xml b/server/-product/production/OMAR/jobs/templates/archive-material.xml new file mode 100644 index 00000000..973ac9fa --- /dev/null +++ b/server/-product/production/OMAR/jobs/templates/archive-material.xml @@ -0,0 +1,49 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/server/-product/production/OMAR/jobs/templates/archive-ondemand.xml b/server/-product/production/OMAR/jobs/templates/archive-ondemand.xml new file mode 100644 index 00000000..54555f6c --- /dev/null +++ b/server/-product/production/OMAR/jobs/templates/archive-ondemand.xml @@ -0,0 +1,43 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/server/-product/production/OMAR/jobs/templates/archive-recursive.xml b/server/-product/production/OMAR/jobs/templates/archive-recursive.xml new file mode 100644 index 00000000..02b97884 --- /dev/null +++ b/server/-product/production/OMAR/jobs/templates/archive-recursive.xml @@ -0,0 +1,25 @@ + + + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/server/-product/production/OMAR/jobs/templates/batch-retrieve-ondemand.xml b/server/-product/production/OMAR/jobs/templates/batch-retrieve-ondemand.xml new file mode 100644 index 00000000..20332cf0 --- /dev/null +++ b/server/-product/production/OMAR/jobs/templates/batch-retrieve-ondemand.xml @@ -0,0 +1,29 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/server/-product/production/OMAR/jobs/templates/calculatemd5.xml b/server/-product/production/OMAR/jobs/templates/calculatemd5.xml new file mode 100644 index 00000000..637c7638 --- /dev/null +++ b/server/-product/production/OMAR/jobs/templates/calculatemd5.xml @@ -0,0 +1,19 @@ + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/server/-product/production/OMAR/jobs/templates/cancelable.xml b/server/-product/production/OMAR/jobs/templates/cancelable.xml new file mode 100644 index 00000000..2be289b3 --- /dev/null +++ b/server/-product/production/OMAR/jobs/templates/cancelable.xml @@ -0,0 +1,17 @@ + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/server/-product/production/OMAR/jobs/templates/common-copy.xml b/server/-product/production/OMAR/jobs/templates/common-copy.xml new file mode 100644 index 00000000..7abb6fcf --- /dev/null +++ b/server/-product/production/OMAR/jobs/templates/common-copy.xml @@ -0,0 +1,41 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/server/-product/production/OMAR/jobs/templates/create-lowres-ondemand.xml b/server/-product/production/OMAR/jobs/templates/create-lowres-ondemand.xml new file mode 100644 index 00000000..f7922734 --- /dev/null +++ b/server/-product/production/OMAR/jobs/templates/create-lowres-ondemand.xml @@ -0,0 +1,106 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/server/-product/production/OMAR/jobs/templates/create-proxy-ffmpeg.xml b/server/-product/production/OMAR/jobs/templates/create-proxy-ffmpeg.xml new file mode 100644 index 00000000..acb2cb62 --- /dev/null +++ b/server/-product/production/OMAR/jobs/templates/create-proxy-ffmpeg.xml @@ -0,0 +1,25 @@ + + + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/server/-product/production/OMAR/jobs/templates/delete-materials.xml b/server/-product/production/OMAR/jobs/templates/delete-materials.xml new file mode 100644 index 00000000..365fe19c --- /dev/null +++ b/server/-product/production/OMAR/jobs/templates/delete-materials.xml @@ -0,0 +1,17 @@ + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/server/-product/production/OMAR/jobs/templates/dummy-test-job-copy.xml b/server/-product/production/OMAR/jobs/templates/dummy-test-job-copy.xml new file mode 100644 index 00000000..2c5d5a5d --- /dev/null +++ b/server/-product/production/OMAR/jobs/templates/dummy-test-job-copy.xml @@ -0,0 +1,47 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/server/-product/production/OMAR/jobs/templates/dummy-test-job.xml b/server/-product/production/OMAR/jobs/templates/dummy-test-job.xml new file mode 100644 index 00000000..2c5d5a5d --- /dev/null +++ b/server/-product/production/OMAR/jobs/templates/dummy-test-job.xml @@ -0,0 +1,47 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/server/-product/production/OMAR/jobs/templates/integration-test.xml b/server/-product/production/OMAR/jobs/templates/integration-test.xml new file mode 100644 index 00000000..6593e5d7 --- /dev/null +++ b/server/-product/production/OMAR/jobs/templates/integration-test.xml @@ -0,0 +1,6 @@ + + + + + + \ No newline at end of file diff --git a/server/-product/production/OMAR/jobs/templates/metadata-updater.xml b/server/-product/production/OMAR/jobs/templates/metadata-updater.xml new file mode 100644 index 00000000..f516c3e2 --- /dev/null +++ b/server/-product/production/OMAR/jobs/templates/metadata-updater.xml @@ -0,0 +1,6 @@ + + + + + + \ No newline at end of file diff --git a/server/-product/production/OMAR/jobs/templates/migrate-hsm.xml b/server/-product/production/OMAR/jobs/templates/migrate-hsm.xml new file mode 100644 index 00000000..faec8724 --- /dev/null +++ b/server/-product/production/OMAR/jobs/templates/migrate-hsm.xml @@ -0,0 +1,21 @@ + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/server/-product/production/OMAR/jobs/templates/move-jpeg-to-isilon.xml b/server/-product/production/OMAR/jobs/templates/move-jpeg-to-isilon.xml new file mode 100644 index 00000000..ae74dab8 --- /dev/null +++ b/server/-product/production/OMAR/jobs/templates/move-jpeg-to-isilon.xml @@ -0,0 +1,51 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/server/-product/production/OMAR/jobs/templates/prores-archive.xml b/server/-product/production/OMAR/jobs/templates/prores-archive.xml new file mode 100644 index 00000000..2d49bcba --- /dev/null +++ b/server/-product/production/OMAR/jobs/templates/prores-archive.xml @@ -0,0 +1,76 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/server/-product/production/OMAR/jobs/templates/remote-transcode.xml b/server/-product/production/OMAR/jobs/templates/remote-transcode.xml new file mode 100644 index 00000000..ce4c77b7 --- /dev/null +++ b/server/-product/production/OMAR/jobs/templates/remote-transcode.xml @@ -0,0 +1,111 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/server/-product/production/OMAR/jobs/templates/retrieve-ondemand.xml b/server/-product/production/OMAR/jobs/templates/retrieve-ondemand.xml new file mode 100644 index 00000000..b58d8fdf --- /dev/null +++ b/server/-product/production/OMAR/jobs/templates/retrieve-ondemand.xml @@ -0,0 +1,150 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/server/-product/production/OMAR/jobs/templates/retrieve.xml b/server/-product/production/OMAR/jobs/templates/retrieve.xml new file mode 100644 index 00000000..e12fd950 --- /dev/null +++ b/server/-product/production/OMAR/jobs/templates/retrieve.xml @@ -0,0 +1,21 @@ + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/server/-product/production/OMAR/jobs/templates/safe-delete-recursive.xml b/server/-product/production/OMAR/jobs/templates/safe-delete-recursive.xml new file mode 100644 index 00000000..bfa8e371 --- /dev/null +++ b/server/-product/production/OMAR/jobs/templates/safe-delete-recursive.xml @@ -0,0 +1,17 @@ + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/server/-product/production/OMAR/jobs/templates/sync-subtitles.xml b/server/-product/production/OMAR/jobs/templates/sync-subtitles.xml new file mode 100644 index 00000000..f68c2781 --- /dev/null +++ b/server/-product/production/OMAR/jobs/templates/sync-subtitles.xml @@ -0,0 +1,23 @@ + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/server/-product/production/OMAR/jobs/templates/sys-recreate-lowres.xml b/server/-product/production/OMAR/jobs/templates/sys-recreate-lowres.xml new file mode 100644 index 00000000..9ef84b31 --- /dev/null +++ b/server/-product/production/OMAR/jobs/templates/sys-recreate-lowres.xml @@ -0,0 +1,118 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/server/-product/production/OMAR/jobs/templates/test-fork-cancelable.xml b/server/-product/production/OMAR/jobs/templates/test-fork-cancelable.xml new file mode 100644 index 00000000..8c88d84b --- /dev/null +++ b/server/-product/production/OMAR/jobs/templates/test-fork-cancelable.xml @@ -0,0 +1,6 @@ + + + + + + \ No newline at end of file diff --git a/server/-product/production/OMAR/log/.gitignore b/server/-product/production/OMAR/log/.gitignore new file mode 100644 index 00000000..c474102b --- /dev/null +++ b/server/-product/production/OMAR/log/.gitignore @@ -0,0 +1,5 @@ +/delete-materials.log +/markered-mediacube.log +/mediacube-err.log +/mediacube.log +/movtest.log diff --git a/server/-product/production/OMAR/log/2022-05/.gitignore b/server/-product/production/OMAR/log/2022-05/.gitignore new file mode 100644 index 00000000..5e8754b0 --- /dev/null +++ b/server/-product/production/OMAR/log/2022-05/.gitignore @@ -0,0 +1,6 @@ +/markered-mediacube-05-05-2022-1.log.gz +/markered-mediacube-05-06-2022-1.log.gz +/mediacube-05-05-2022-1.log.gz +/mediacube-05-06-2022-1.log.gz +/mediacube-err-05-05-2022-1.log.gz +/mediacube-err-05-06-2022-1.log.gz diff --git a/server/-product/production/OMAR/log/2022-05/markered-mediacube-05-09-2022-1.log.gz b/server/-product/production/OMAR/log/2022-05/markered-mediacube-05-09-2022-1.log.gz new file mode 100644 index 0000000000000000000000000000000000000000..53250356e7cc8a4dabe7b83ca4850a013c28a236 GIT binary patch literal 600 zcmV-e0;l~SiwFP!000000KJ(zZks_2g!=%w!(S#vh+RJR!OnIZz=e}42Tq z?f&-b-PO&{>sx%j+V1y<$HUFruYG&nPw)F_+n@3Ex978LpZ@=ar^EXp=lJ~b!&d9! zMG+lA;)Q2*DMcwf&4PNGwAom^$TFqJAn`2HRK{S{(|hU^_0HnWbdwsD#0$;Zr8n07 zMWieAz|lK+Ej6SCNAKW`tW$!fhnUM>Ms*4g=2B9Nc;o2L{`t#PPCYL?4=GOL;l04G zm7{mdS~9vsFDxD`yTh-I#q%sptg-X}9?s{J<7eQ(&zh)m`ny5wsa4ci_osE`vq^iO ziU>2rN%<4J$bjC1R?iMglU6wO^i)_{6*C&Iq2)q;jsRu<38%4cn`?$34QmrkoE<|CZXN&ObQk=~>p2G{1uvuxu&D z;;sH(F+XtjFF4#~9t%>QSj&)0-sSOPy_YbS{=hTLV}-_pUl+``w0e%UZ|#M}3tRqD zjMLv^?XS%xNc+}U`#SWaSvY!ctN$75**NoGTmFf8#YpoQczxYpuN=PyZ!-UXXlq}f mE`zjx0neA8>E!5PEw9)7uARnfuuxH@xapwa3XH#vO_sepBXi&&-o06vh4n z$Q@L$!dT-AcB}=YZv)4f16?`8nKh?`T<;#v^=$p|_ccuWgedIj-UHVlKuFiy@9RoW@GZr^v)RNf?sn>*>gkF`xPD0ZHGR- z_vLF3SMzFpycKUDb_xgVj>dmX!f)RgFE~u~tXfJz>{PVa*8i(BwTD>fjN$Hx?#Z|1 zuuxR=9}D3TQSf=au;6=Z9HumzDlW3!Q@X5juWKeWD22E^)ArlVE19n1Z+7BuCS$b|({^ysw)wqoEV6B>&CZM5XDbrl@1QUJDq&^j%KAv# z^6GXQ?Ez(T>^}yP?e&P|d&iN5`7%ak>$le@a9y~8v$`Rv>;tN6e=c_8r-D+NnfTKV z>M-WS*XqW~iE{dn+U$B2g*y|@s;_FDC1@}pL3lO#V1n2uVp|VRfA{o5+RXLpg|UmK zhNiDOnmLOtALsuFy8HeM?8}zp9VKYz0OV+5roXk$j(dIe<*-xv2v;T#9R5#vb6$Cu zh;95t-dcf7*XHc({`0D%ExRK(1NXhXN1c4g(pv^IwO{_NS)U$Ukx`qTWI?Qb*nj;@ z)OjK{yIHAl9n9QWSzCQiY9z2rH*cXadF#(HDQPIbv(Dj*uZ?T5;wok3FhI`i+D17f7^9N%^H*^eByPW^P#@8gC~g8be5{Y=3MzU%p^OCP4Prmveuep);4 zZSuuSuQO`R!h;9(XZi*X;hgwX75~^MbrW(1wq2_a~peHosj;2*v`G zP^Nrw?%RZ;+nt&M3)xf=32Qfu_kFqKcOpB^ZLe>|jTH1}QkS+I6)|Z+E_7FZOSu#7 zD;uAB>U|b%hB>QKW>` z-QM5vb4-%Yas1;K153AZ13l#Rt0RW@EdKpL3^Le0u9z_>SN|+uUsrvw*ReI-ygk#O z=dhc9D7EW5R+sJR>zrW{)ps+>!i=yGAH7v>{Cnc=KLIsO{I2VLQi zkI+8Kxt{SjU#T^_lij+tjAQln_DW~*TUL=JpZqHerOV}ok};3x!x&L9yK<)UvehYR zHwz;#9(xAd-nj>I8^|8ux-S4uo_e+7zS z^geN^bua$w{4w3U(U<2}mBm3l%F-Vc)xQOpT38l%Ti&yF5fN=xP`k$VZfj{%bEjf~ zHw7WIbeRjcIg-85wJOxh|0HQUUZ!KNt%!d$+~w7HMXig4;=N}H>+SUhA9P<5Bj<;= zGmL}$4j0|Nc&SMB2mAY0Dc740i0E^4%}WY@nA6i#$j&lr{q3)OvXNVtq--~syE@_r%v+~*K&l*yikBc z^cVWdg7K^D2ay*KHyquzrn0~28}k}h@&3Nma4IDymo|clvEWGlle|?>z;Sl!^=Aw2 zer~4yM!ZBmxO1Gi-4UOfEdO+)om$Re%iTZky&sn2Tk~%3D9a5kQ zG~|`aU^(x|*kSoi6suttM#f9}k^-*$R(|}QW#{P&GSV~oMehrlk4j9`en>ubMy1^O zr-CY5GoJtY`ci*e=+`gf#hHo{eOgJj=tUJ}|C8xoxjQv&ZF4W6M!vS5k*^Z+-TbRf z{|(9~7S#*$YOD+Mw9NZ_k{?#Wd`4`$>2b+K);!gDQ~P-KL-bgx;XK#QC$gf`tF6PG zj)82Rj_i7sqrd?FlC8j71M@X478B=7bacM7wB)lK7Ni)|d%TcP`h`e~`HkXQi{C5V zsy&^4_6M~iQh!>43px?{134S82;sXJViEM_i_}iY{y>j<{EI5Z(2RrJ+R9v9X+r7H z95uWu`7BF0$TTg10x`^9_Wt#{Z7;Fi6jR%?_y+1vpT2rizXh-$!dtt1ZfEwCC;RqszvYP+V)ya) zsr?hz4fsw=SAJiF!@r--^88!_Oj8wqGAfP~$CxVzZuQIJqNf{k*)!{X_vhQ52Cf~c zmA@R>m77)aW^y4Iewgkr{++(EE%ooQqISFw=;7_}*^xOXh4~L`N?)c()_)w{^>+(& zE+KUEB%DorJ^uA{=gs!wDJ;k;Ci1i4qkuDwe*Itfiw@#XKEHO*F(~zygRNd`ZBeQ1 zo2ymayw5Db&0^0N;M>omh53_P-=6mW0>3}gpGXW;Dy%aOFy71CE0(vc zt=~TKVte%YUAq)HZ)^hd>gL|+W;^I{eV+Np^=dQ2RNeYOsl#4pJZY_J(r+c)T-1aiR$67ha#|<6idzSEO#cjtGJbJVf9rWqP z+m}a}k;cHXH>EB2LU@H_D0aeCweR!P(4)E6OS+CE$VLlaU?~x=SQWZn9sM}ybmG?P ze3gJD_*(R{ei5v!df|107d#)3fSvVy8Qp)-twddZRIlKaeBxqn;cxa?;_rOTS>|J9lf<6rNfCcxf6$yq<6ME z{H;Ojw{B9OMlZ~=e?D~?8WHA$n4}o?T+Or zyT_Y>8DAqdS`d?R18FAS%1)Q9l-@W zX^!%oAK^O9Di}D!pL{UqFFXrLm8%Q-)YNHTQ}`gj?b$t*&TAp{AGFHZJ{!Jq{)~OZ zwZA`@axh? zI$NYGP0>&Kx{}|)NwmyfsH6r*jK+X*T3z6QzsLT3+jh@pXwSAt3lpXQjes)JA2JvH^GrKg>T|BfFYFSq3mJE#O6&3f*H zdn@t=7@UMZU3bYV-xaoDuU=+{iyT8RO z12dnbDmC}LIsx|GDz;(`LF=xLzqsXT>9GpJg zTQjRws`6f+Snn?kGCphYx7X0RNAx-?x=yr>@7r=9AHHLmUEOjk&kTNfQ$Ks7eP!pN zT_8pJ!g7M`j?>v$%GWXB&5nVX^xB&NC)+<{^N8ON{gTQm{?b~YaiPLO|)mE-X-2qdR`3 znCN{q=bYr=x*Bu&ngefPT1r}>%G+N&33KC+l7^~`V@oI>W0>-yk5RjBai8VKleZ%0 zzlVPqL|qD0SHnoT2=4l|O*%G;`?ygi+03u!1G7qjj^IhIb1H_Gm+M!h}yT_f(yJnG(6kO=c(NMJty_+CHv#s1#q+BzlP))%bf z`fQilIn*#45fCkfT*Rznqk^LNR@=B^V{!zkY_KW{=EMBM_poCqq{3rO%v}z%!Je8j z+%d*oOGV8+jL^qyu#-|qy=~mJ>cUI6*BsSH+-%%B-bBNEn6-^wwMj}cPUoyU$Bm=5ftWJ%NSn{ zadITM*}7LSt^?YgEaG53k+GWlQs*!mIRJx0GXlrGU^Q$JGT+t95*!TiG+v@SGcyl? z`7rCNwdj;Ft~8+?8>>kk7%850PN&9h%6HbR-GAU(lu6i?>7_+9WrTNJ`Bk;-7rWB2 zwIcb%e{CEa?+WLf9{C(>ZN=~;T%w#UdlV62J|FBNk1VQ3LMl8t_n^^3A#$QdPK}t1 ztP{w+YnTo82S+6zoLGgMoe*!5VLl9;2dfH-zL*Urkx!D(DZYHsRPnuN#hW6&Xxt26 zYm0B}(TrW%qtX)b6R9tkU2;Z3Wc-bTi&DKN-6@RgoYaDc$CSxHU9eN5LeY82yF3NS zuu5J-9*m15X^k=oHVq>Zu^kSq<7x<>0oO(KV#Wa{6g&OIaLAcyI`eDZ4|<;FPktv_ zuh;c)4PP_)a=7EAae1rGsZIRBJHer&`~BGcD{T6`drp>doE*(_>!Hi|1?Jmz@{tWM zF>;-o@7|A!J5z1UpbFYx4IxcplR>ZAv@I&KE?}~Ir*|o>7Yqv=5ttIe#@-E75OThz zd8FmByYJNT@bvVKqCkDB3Ek&kDvttB-XA~vhtYihBVB9e)^;bugmUKA3T}FFYXs`i zQQdDDcKT#~@A3PI)c6A@je1Q2#8$C$zT(Lpckwsf2M6Y$h4kJ7I*ov&?x~E|JBfI6 zkmMit{9IRpL!$ES{c|| zRgTv*1Bd8zF`(fLHp6NMI`;{=P8gJ(Q8tVisSa|;=TbY8ng&KbF!&leY(vd#=z+f$ z6k~VtF*=qAz^boAoHL*7fL5KK?q%5R9TSiCd6ReI_e+|RC zf7bh&$(EX;6C=Rxv^Y!2&z1!?psNnFfRXou-40RSuA)UYZqIQ7?2Phik8k& zIA~b64SX(^ni*`=>h#px)vj5%B2c=OFvl~Q`->Ba(?H!_2%9csCJ5P7liPpSP=Ysw{ z|D0>C0J%oC?6tm69-SE|?*fV|(xF5?TAhpk;AuR9khH?uV;N{5sfK`YF9OU3y((rO%?2$;| z&CX^IMYe@D*&p1SvXbZ^P=Y2fLi7C#x_VjQ{)4dnA_iVKXqc1#+-q>u!>^faVofX^ z;mZ>2t8~ANLBkCB*|PfS1VMHu7gk?WXIIe-gyXHH;IEkmBU#%T2%R)tk!zyd<7kY*GF<5=Hlcf|#LGALdtR?+U#7%S2oVq#W_#4oB)`?a zN`vV8K2aOLz@_B7k5HT<8|uYm$M?6b&KqFbnb@RbkE8+}>B_aG?J(`!p8`JpE?-A~ zb$_<&|ANmJHPk@8P9fx&L<|;g4x#TGT;lc797f*8pL^M@l$V|b-wjL1 zpi-&C61pq;H}ssK7>%kqoHc5QzWtGjH>N#8N4Mc&3E7cuj%fW9(@uQkJY7A4rg*q9 zbd3>9GS&B`QkP5}p%|&Ef3-@;j)3P?Y3}7@$D!Gu*ViZnHZz_8Yg_q4{1bs8Uo$5t zMyx6xoCUK9O=RyPme^b&bm}@nF{)Los8X`ydFfucnFdGbh~aIyXJ4J5d0&qyKm&gS zMvK=>yNK-Qaw#w{O~R8x&dsVGMpI(1D+}bo;O~fzC_S>!AexfGV!~SjBXgL~pb%gmLPR5}#4MC{8Tk(YRZPyEmLEYMX8*QA4WY%N+)|vNc?28 z!HSXdVK%HEvph0~&>oid6rh;;4e9_f|G!fc`^g&i-IlXa&k*Zhu>6M zLq4Y4Zo4WcAI-=Khm;@Xl2}2~-6Xo$7UFq?n{tQ=b0KvI3bPLV%D;4wyD@mZ1UYe>3pHPgtRW+4UiAKeQ$Ahok= z5~(+Oco9dy-rx4OP>WM{C*``wfuPG}`UvbzsZa=`e$2pM}tLApuAs=24CKtn(-@l4wb_0{Rc% zH89k#`yvhqH}XcXsEQ4y7=fU9m4psuKf*%<_U5e^2s|cX5lN@Z2*3Qd@-ngqGUrNi z*k%}P0R2Kxg5WC@vxE>b{W)LpD#Xp=*W(@*#K1jwQRpHKFY?Ra{C z^Npwpyp>Db)hWKVW7%!nM(Y$(qX+1S>Y?~=Q=2Rno0UTHm9@~`2SbUHkBknGI3QoU z(Hm1{zvF8)S$c-72+6l5B`{?0qAp8 zM-ThST;{?TL-jM_xLI&d)*#+XO0rT}xX=U=uBqs0*siz}-_PMR34=H@u2-+e@@8*!}j z1{Wy6;4A*M@>F7PciR!tbSqC)B=Mrq5c;~JTsd7RjQkq-^Qr7GI{59bwj~`#t68YR z80-WE7`BpQmneh?J>A#prR2N8A=MlqFlsMX*-@I16O=83_=6h$=trGuG5M}X$S_9; zjB%0E`;%;6)e!o+RMml285!_Q_92F)2;N*qzU$$ZOq^={7&V7`Z0ZPQdpGI=89`rv zSoIv>jbSMgiWBCEgOTY#pJ8PF<61{(cb@(f#YY%}6(|maksAm6OiQC-a6G=5m^DT| zhrtTu)xhBH+kH+@wol(e$x9Ra<<8^{nbtF_$lC`r^FWiL9c;;L6d25?Sa>#(8_m@H!{ z1g6h27%hfwWvGE8lv@PK=@$Ygez1(wm~kG2o+MWHos1ol4y?C#X_`{0Q3bnP(aKE~$1K^PQL#)!~Gq`@?C8 zYRu$lC0@2=w){O8rOT-{wPx)8I&-)1_3_{gE7a)GNivD;w(# z1>gqKDf@e5o%dQM-xHmZgX#eCx2o6#gNuU>D zI;NBQd55#IsO^!dRzrPeJSGZC7nBh_+xs0ycgCdcEP}eY&+tsVLac`Q_{RMzD(>$#2{J`@R9B;B6jKX!0U6z znFdie7+bx!ZZjUgP83G2NKO_;t%EiO)~(PD^^FET;&<*%7Dg@K+;|#heSZHR=K#Nk z`b+~KKS}6hVVCBG{guAU45!$JKf(0^;?3XDoM*B0r%5S_bY~GpYWoj3_GOB$-)D&k z4hmKH9UNpq7^6RBn=4{C#S(td1;I5QG+5w!l_oi2%Q)Q2=}(i2eg+3M=eflFtvAcM zf9pXX!)a2{54s>Tq+3IMW4|ofi(`KU-AjLpEfA**f}5K(SX?5|?oNp5_y#)sNgy#; zg>_w(;gsT}xyZ@H!@AG#B<#E|lJRHEc)E5pb#Ik!C!+i$B-*^8-qN9-B+ws}@1Lr2 zAeWkX#Ifmk^;>Yzld|RypLFPgFd}e86OB8Q6S`%OMaaP_0P5Ko*zn0YdmHN=ysfv{g&g7KwkB7D@N2 zfp>p8_Fc^7jh9hYh64EDAdcKMy0e?lQW;K_isWfe5XKk!h5+1n`gM+dUuMzq)>0Au zsq1|B4)dF!&t1{Ehhm?NTGW7T8ci}mB`Tce_iBfM9rpGdS5!`?QY9AV-c$_L7>QbI zEB;t*>nVS6AF@<5ys@61yjOVD!k}Zc_4vc#i!yUuogyuWs1ob^(zUaCd%R_N^7Qma z`p{;YX?~)mm?5ek^XmHvEsZ#;rKJ(EriKA}v@~MtMe*3ErKwL~Nxf0Ptrq<2ilB)< z-u4y+>`(HfLk|;j`h!nrIo2(LZoLAQDfF(B2wYHga% z+0J6n*}|XvHV4DBe)ZM*J+7FzTh!3Q40_|*;kw-snfM9A>^A??jaky^)~V#)ues$J z`O3W&p0nx?&Xeoc`Ieh1zGnx7C)jeVc-4O{WVQII0vhwTF-iTNwhB#KOG|t|Y@hId zIrkhI_2-QJPfr`|&Q$8xR&d?ki0UDq(gz@A<*TV?X(_)~!U(0)olo-$cMZnV3g?;C zrotOL>C%y(U=%B-6#GJ?ImWU0a4PZ@7Wx^MYAidzjj0X%E>I#nFl@ytA``h=*Yx%u z)WY!AKdkEB9;TRw_W^EB7oK-Le);`RWh@4xO}&wQOu=6K2W2K+b30vzsy{%)iP!!U z4GnzGTT&rAF^uMs*nR2TQ%?O^=UTM;Q6st#2F73g#q9=fDJRQD)US*GEidOZ?vMRFsGQ!Jup!AtrP>iMN>?+GJpT?K1irFKy*8@qEN45~0mbDoN1 zxVoZQ=0IHsXZq=SRkPa9)fZEoCgR)`$sdV`RaXYXisVmNpV2QxBb{#6p{_M_9H{SK zo>pL7IgiqK+ix|PjyeNGK6vm~jftloO$)AgF~}d(xw{(~DSmC5#Wd}2E|x2vHFOyn*Sp46nH=3rKyf3T zy*-NjZq70pSM+n`s>d!G@!0{$)nfo%XARC#XCf?C`gYu?v#N+liM6a7%ei|*9cP)0 zxa3@qiW__^%L}7p&O|o+OBdVhD%u`7{+hAi*tVBaz=~uiu1BFCSG@Wl+^#iCQ#CbX z!FpN{*9Oe$G zQhiaVvkY8ExFT8L_Sh;@kv7W<*NWtu64cdSS8Na!n8yV0n6n>Y%~_@cdGuCmd^HE5 z=~83=Aa2YVTV!34ERbI_@`xj@XO*ejyLmO|{K}fwlW!qYy@QP$hET@(@RAy*ypsn=jwiVVT zo^B;$)Ok%?`^0gDHznx`R!;IW# zN1d7YHAd78td+ztoA<)Wzk0K`7#QN|$^}TH{?o zI!&W7AB?YmIChrlNZ8oIk2WY|MF?bhqNh zyj{RJZ0yb~qTCif_dMBe2dm zf}rAgtDoT##t7_Ol)83E9W#Oj{P-uH-Y4o6U@Zkgy$@4U#JI}h2oY5Q?zswRag*zO z)+#4H%GXt4_1eq`tw zlCBl0TQIeVv~Qg~R&>7B9YK?Oi%2@Du(_^lZ+|20uQIP7?E#TXNc+WEFPn!57C`S> z<0UuJT}QX)G1IhFA{Q1y&Wyj|gTnyg+;n091FzGpy-MvYG8>@7`<8TBfUFR>^^62Xidb94xBCt*d!9H7J2QTbs zDy=vS0>3mdgzuHQ!{%v^zzRt8fY*g>Ag zWohCp;yw9+rVGi=ZA%t?_e~oP=xLM$AvYT6&R+sS1N6MyNq^u&o%k%|(r=`mwV*5- z2pt1M*+?P4m^EZi7+D^D*Ekgn#m4&MfN;BN2>i;gA$+f@n9;l}2wGkFvqM>)a2J6U zxD^e7;}hnQdgsUr=<7y|N;iA(S;+Qjq~#Pi7eb}LzAAbccH(=b%7d`{bNfDf0e@OX zcBaj0X0k28aS*blL%kM%e^AhI%T&cd^A5)2OlY;>Esl_57wN7bg3x;MA>fRE3CaJ& zMF9;szv)_=2qCMvlm?$yK-;;D=uQ7d76UF#Key&WV5yn`VH%K>&d|G(eXc1w@ve$p z@vfJ?f0*&O&DH@o)`+gXkhCR-4lb;lO2^awLmJ>;LTGX1uHgtwL~p@%SsIP?AT7qn zfuL*>50)NK=e$To*&MBSbUAG6OIDrwR@aX2;zD2?-12HnsOVveAkq1u9gzG?O_2}d_*Kpc#5an`F`5Q8PN&dqV=e zzBt!Nv>G~hVnX{hTggVhn@I2DhDg_;05dM@<71tm+_&G2d_IR^DQf%FMBDSw6z|F? zID!t2T^f3)UQ8Y|?D-Mz;RN;jDnJ1$`Fmqnq{IYq#*R?GN^1=ErdT5}tMcHQBUCj% z_WfKDd9kP05xSfrh{4Lk`iWW0-=CrAjKWpLn!lqnNrPyh+jD7IjY0#rh3V6fp5NS-_W zj!>lpHes z`1Ll3`=C)#yrnTKC1%F@J}*zaqjK7h1x?QIcq?H`RI$HjTr@t$8by~C`wsbyNS}-U z`$jodoKbX!Z$rc0^!F39bhJvz00|Ab2VZR}$N;}@;mU#-Ei~BDN6*fa2@ags1ckNP zil9Cwz4tNZ9JR|yyu~P%lZdr8&@vF;nLae{RiXkPL5V^m6a8^nz$c1tu6VdgJjTLI zCo!G5ny-f~6Es*cXn6=;2zrqV42CYLUG<}z*4t^q0h=%J-{csNfrP&mSwR;2EG;Y| zZ$FB@B$C76Mw)!z4~85sFyfF=UWkh&YmNIor8%@Dy7%r=0xqI0iePz-@E4$gi9wc)LnIG_GtiBdkY6tGA)6?Io{FC!{duMAoWw%B(f))EQb!>#>t|8+JD3Wm7Xyn zus3NcGLs}rYYGrl)n5h#9WJ7IeG*iB?-d&aLhCUB42Jp<)^I>24J?FFG;$T$6DFvD z23oe^fN(>aqbQgSfpa7^;^~W6xUYvm;9PNQNIfdqe77f4P#*oq$QK0ltEC+(8VaE{ z38YmaBgpdTKS*B?6jZD3NlK(JMj^z-c;D@G51ZgKq&z7q$!Sjf8~% zrCc~5z$JcgLjiqJ=Q3a1NuPt{>OwTtM?SUln`@HL%u?<5mhH+!qOYlVoa~f0$*3+d zA!ZgymzDAk`3+B>d!CM+N30QVWqBoxEt2)u7(T0Oc^OI9tNJft9@*1u<4pp{H>HND z^Tt7hKAlL~CtWL7)qSXWoyndU6DoRKAP8Mj1Npe7OgP%kj96BX2IDlN#58}A5*^@& zZn}27<0UDO)@|o0JU` zQaOKF^55aE*l;Ik1my9k(1j8TFgk(a{X~WKU13pK`-(!yv;VjN5vXJ+Ay2;VcY<=m zz@Il6IC7m)#dc958@t2auV3w({3KdUKH!yxUUh`#m;lSY8S`OO!+%HCeK{%QtGKKD zUnk6axy#7_4pi=^rx>g~Qec148w;a0U6`NHqySL~Z8qCT3NQmABzDdT8Z43kpD!XC zwVL*7PTQgA?o@qRNn~P_sOnsHKovg=*>qyUU@f&TF#{iJ*kV`!EQd%@3NWKD4lj#q z)R+4HsE7S^|_AOmL1iph&paZXUb+;bT0^t^2mS=2dz3B%I8?5hNF`hT#bC-ln=>F5_I zy;ItQh-WEc`$ek zp@rDgsGyI*+G8RrpJmvg=!BU=S>sn{?w;E3R;auPLJTU4-5NI%x z6{A%|UTmhl1g@!#DJ#!%a&CMt4YGezfT{>1g3F?M33+xWbbosIXn*0bHp*QkCSDQD ztaohOhR%EO&74zECh5Iq6*|KT_YrAGPPAGQ!9tVJ;;4XR#klY%1gq%K+vlF7bi`Y+ z*t$kZe9zTfGR2I?HFbl*LQsJlfH@?TJ*0v4lAMGk0y`CTsquXv2pT*P==bw)B=r{X zHwnCc4gF1nl^I|Ngj!+~=Xe*v0%(84QUVDNb@8EL%7dvF zb`l5{&UmjOC3*CpjWIB^0cipQf@z!!LE~I_8s`E*RZ;Ty`rJv2J-r}WpCD+Q3s2v` zk|X~A!3*}Q2ZF}A@br}|5t7*dq*8=5Q3ZMQAJH)|)bA}2!J;cRm~$Qne1t54z%LPL z0>;yuw5WDR9t|*Qoc=nA)C=?UCgncp1^=g8VFh1kbmZJq^&=^Q~ zsCO<7*hc}lUd&uX(&e0jppx~0sZ@@bm`ga0W~?t?P8NhQaagh#wKE~p)7Qaa5p=%i zB(gF-y$??hSNJV827>Cg4Cyq@{13#0RW6`^;OWn=4*WTCApug7)n6!Rje?Z~X-kES&tN zH^EWXAZRdFO2;MhG6;G|i>RK>rQQ?30X@-45JuW~gckw}kMWoI!!L)%hDvRe9M+AR zz+!DJ@xt)?>{&U{DjW!{{_m}*tbGk>1}K6>|GN}d8;+30{cvBi?ytfxAt@(?_u;Lu z%&x&SgBzoK+_X{9O03{SA!R{^IunuR)2|wEg*-sMqLqFeX+7c zR%v+!^nI5QHC=B+dxVaZ>(Ux>sJE@-C8#9hhNAb;yIP11D|8^z8Ao8d_P}I8=a&*g z^<13vwyYfb{ zexpDHR;(@?LXMFTmPP-m`8kBo!Y(f(haQLtTDReV1|}sC_&eexaw)WL4ByKPYr~f? z8n*sGgQ0mv7o}0*y+in{j6evOc~oEw1Nm`An1vnmn$h3EiaGO zzm^Jy<`F*QfHue^hz)CgtgMJU+QS$OhUUeZ;(!Krc@Q|yuO56Bo2CcJZU!xRBVtx|c2Cqj&iIu}pShD9qrED#Rwj`{zwA|%;@wG7 zx*PK7`>IhRjNu?C+XI|WduBDnhIODWCrTDg0jzGTH{z{a-MYFiy})L)_T7mJb;Zy1 z$g<~@ZkDwkyI$2f=IgT6oR?91B7x9q=Gi}w&|+5hMzj;7Dq7BgpkFeo<MJkohtB;#z ze^;ds9hruIc!8=$rzF~y0CVDS6vAx7<1c_sJ|jyEb}EX~n{)3uimo1A#V z_9K!1H>Q9B#N2b305nm69X(6c6}d1*AEPGhmtnNFI!og@87l2^UOm036i3=Hgr|<^?={nh+~mL3?a&D*(AI(-Za zpdD&UBY5%2pJ>utdzNS&0;5Tmn=~6BMENon#<+NnEQq53S&8jn6+I^?K-3$kT0}OY zXgh1f(#kKnW%A0+xk0+xi1hj)VwX!`jGS}-u?1MFPI&(Ex{(A&)D#~m4Muh_m_NAt zIQ27(F^cP0;=hU?>&MgQ2yHjhDkd*x($d-F^hP3m+4avLCJgo_tdp4aL$idO+lO|7 z`Zdt_(qPT)QacK8QeW!GWe5#?vqS+p(c)N+g#YS%+Drrmn4#Z&P&tDBa}`PfDrtql z;8)@tQF>uQBWR%Um=hH6akAaAjJ&8OJ%|QcjyXZ|Vm&ZeG1@p0t}omU)$@F=n7p`2 zqksPwOQWwPy$XwPKovWw`)`o`4^R8uBNOfY^6VzRBX7fSoy?Om@?T|B`di@De%~}3 zwWJSG$%YglEd#t$O>`PB|{UJCH?s28Df z1P#atp!-!!zM4V`-0;MFQ{iaxxenYePUoJIO`D6~^q9AuX-L;Us?2K5@@b?+zW;$9 z3=1c@;Ncw#P!&FiuGGCi0R|Y+GI#;fP1AzU6d)=|0CZEEbQi-i$+Y#Cw%KAnMp&aX z*+aT-IUpv}SGEOyoG9vSd4L{n+6tjQCS7|aSNYKSAK{u?jAGyZqy7u-KP0`1h9;qd zg%G7KShV)0>{bu8N)bwNsS66e;DDANAXbn7y{9(eW^qlrJwy1*%;uNNC*}m!kRN65 zJe>R_iNH3xKdnS6pdA#$jHV^z(Rn#s{7pL2f(RC36*EHX--xKLorQx2`rS$7#~!Ie z;_0EpAOF3D7pw|s2ib?(|F+@L2F;cAiD0OwLD*~LC?4(3r>rw%-pcJo@|YV+|F%4Y zN8b}|v_#~(~5qVIXQDKq3jsFQ6r{=OjSJ;CWx$*0H=2sQRX zHPTc%BLbzvGr5NR*mw@iDUJg+i^|w6;eZUHcKn%gXb1VPJq6O>A-ok}P8imPw;G)F zDu0N;W?m_a?FK>bnF-5mjDn#|+Rkd33h0X&qdg*Rcy#H7VX-J2A?b7M-OBEm6qYJH z&GXQA>o^|?q4KqNo{xi&t)lJe(3S`+9bu&IA`YnD3ItunLCDSq8L_u!-APxi!W{{* z5IAf4G*WNPDh~oLN7>_mF#R|PRaNXE$p-{ADO1D&VJ3MH`1|-pq+SkL0bLes41$`_ zz=r=9+>?~+S|W%386^jTRu@?#u-p=z_+C~UTZEP*Ar%abi1T=_I*!kxBHc-LdT}7A zU+hI3@SYJ9LaS5jRa{gE1kM`QkI!Q4oJQ(}l4a3AemM}7DS@Op;6f5b$AO_H1*SM4 z%)A5wXC+YaS&S%e(p5C0r+fk={D*=mLaU!5sCrK9E*Kio);Q8MyYF-G^!VhlMf%BG zH~*`@c4I2_!O+jz+9wNETumOjV0 zx3+=uY}jXMUjqfFkRSJ12+fRL2oKU#fwvIG$AWXn9t|NmbeSa-2W-Y}R!J3CE1(IQ zA929Y<8P71 z!5X!1_hbTZ)x*gWXMQAcA@!xvWlz%8O5mkyR<5K(hilVSDFrwn=APZl?7!}rZ*ENi zgwe-n`o|Y}w6;3dNMcvVm1HEVW)GlE&~5o$`#{JeU%5Ej2Z9ER$gIhNpxk#LV>rGU zBwb0P96A`~Tv{!U7B`Yu85Cn3 z+mNM=v6L;%AY!s*iA1IB5h`XZA!-*=80+;^mNOzCI+Tr(;s+KoZlmgeWq_)Vv83Kw_fc zMUlF+XBt?KlAG7hIih35Dq;|Z(8+OP&P2y4bgW-077kbeXinV;UUaOOQZ&LaKII3- zVqRL~-`*J~xp)};;pA=Ve}Z=lQ&>3Q0r-=c^1`RWrZN4w{(+>0K= z*}b2m%~|3h=eC?CLSH8W>ZU^!S$kNYRq7w^S+BPSnv+9sgb4Oox*eO0U||}O%OfB< zDAFK(9-tfdBI^M(zX{*)ZmOVf}9cEbfZ<(pomO3kYG>!ta#@n<^zduqGrI zKWj*hR_Eq?myBRhxSkg4iZ-@-RSu0Cy>y|U2tLMgpkv|+XuwTz3DwYXy1Gm?7m;_z zGZ@V&G45bZ9q;cawwm{UCi2nPjAlDx;wCByux{dyB_a0vzNAw@OLFmVQP^WwqJpFr zGVhRv^XFoqbI~&x(Dini^1t4Sy_^K<~xJ^Da zURD#!WImv;??XD}@7Hu92hjn}e>XPk6A0v$12P9erOEsy;L>9w(+e0-xvu$y{vFaO zyx*DmWeiBIeMz9y1BT}8GxWR7(MG(aP&nx0OFCugcb~lyk*Ale==JXo*l1^0%8zBYJu~Mqz!~t@mSwTA>Ntn^GvU2f=JW1J1-dh^jFIU|O zFhhrMM4rk=FGBJ=`6dmlu<<}qeL2FANTgsaNdAD<$BH7Kms_HZ+lsT({R2tz`hPUA zu$MmxFjLzbXk$px)T*lwX<-_xvb$hD07u5YI!1sQUU`l%M)ULMqOw@Ot_9M|@GD0eQcRcpz zV&bnK6-Am_xT870=(S&y7Dc*pdZ9U)B&B>@2mp0QfC-P?O-5;9h4Eo1?5&LULm`pL z&U~}(M`Txnh(awW<(v-Gi}O{!+RuKh8KRUTP%eQaCa1K@KNfJLE=G(Gv(y)tr$k~x5MBdewZiOSF$l9RIM@z?MF%=Kjc8;ek5|Wo|Iz*A&*y10UDWGI< zKywN;m%VjV1n-h8;T1x(H0BU-0xT{%DBK&U8A6iPoq;_7w*4^%NrEF&j{2T>y@EM@ zlUM)!Na{QUTDXjuT#Yu}l}d$iQnTwy=_0@mw@w{Yj~lIx1lSoDph&!HLa-bW@IxQ_@8Ju` z?#Tsc7cr;D$d6Pwh!q7?3gK^+@>c=jA4wJ+FYx!R_1eqGL{#Om*LHzm88UiM9HtLC?flyKuneVCB7n+^BfT_6fGNSHb z*DA3Cm!8G!eIjxOVmo>xGY8R;`KW0BD`U136?!>)7rNT7Re)eQpkCle^MVkA7)>lb zCuDoTk90(0CVAmA5zOu-$m5RYdlwm65HoY77UFv3d{$m2H+eRlXkq87aQju2q5aMHmq4-k@rlMR!f1#D-*D|L6 zMnC|G3EBFi^_=k?(!x}w7WUY^7B$(ZSgfL(gUKJc~!CW<6iHcurIVD@i9RZmo+QW1Fy89|{B?uacZep{?H7iLE-1ft4s&4O!5~fZo-fH(JM!-4vIh3TpsG;> zNCS(%+`ZP{LBzf^%v!0NRG#>sdnbkz&>WJgaRYP?7~01g8+IagtWj&;W*cb&PH4%|B7&(sqU9RmV>k4zE~9eo||v@`7mVbobQ zrD$N}K{mPy!SX?RNy+S}dAGoahZByXNI|aE;5R3aj1$3+r^SSB0Iu(6HsB(XZ=Uz~ z11QWbBL|1!*^EQdXwJSaZf)SX4!=bS8#E`riyL*&mfgL^jE;3PnYF$KJlmfosj`3p zX?OM9IcT>|=m(ORa>gD@ps>f%vo9tc+>n_^z7xTb7ruXei2}MVER@i@0;&c9bFXWhuaaAe0L4AAMTojDhc6>*NxPABAoeB;{3DD zfq}bbvbpO6+KTM#HeK`)!w2-MasqYq{B4y5W695ZP|s~e9JBdGr_Gu@lNA4qKbhT zr-fZe>q6L0@FowntY3=v8?)1-Vc43|g;vnM@ZnXsT|ORX#m)cq>S5Y9E8dG>K`6}z zZj>4(Axc%(KDPZl3&OB6z>a)w4F~ND6E-=BuH@#N?wh?A6S!z1n!$G>ZYLDIH>h`W zSPcR*Sh1?5%VJb@SHF{O8f4dc<_<^?XkYk(YI+YJwd^Ph|Kd8n8I*jVQ}Z~*dtvqm z^BIaa)0`|DlHydw5%uK}2xlOtPK^N`3dMa6)4n-7o~8WeJb#!bqT7&#V4>fxwec=c_~ey{65a^L6j}SIRc{a&3ehBkBDH#X%T(X#h&xrz$=!zd8q1<7Xta9aulaJ z;~Cz9WQXxO70?+C}j_gIcX)Hp(%KrVfvtd+H(+2$?g`} z=~>bds!e_#8=bpcpX-?nrHrLpwQ5 z>D;B^$&rFo8bii*E*5QSX;C(`%f6f^bUMQoK%PU_EHR*pp@YuI>;X z_c^k@$VMSMl(81SU6slZvD6(euYDm~$Vzv7mbRmkMO(3Uda=F0Fles~f-pYhq#Ml& zQ_Ft2vfmH$Aip>-$wQ6j`;x0917Wbf+`Aha3TCkOC^fu^+dTtHbNz!?P+k5@nqHp<}`mv+W-6{)vzI}qdGSj)vu}C&gcwpFLGNPlf=HM;ex_WDZ9PDks+^pa2wSLVr3LTjNi3SjaqxUJG3Bps-Ad z$$Lq@B-<11dJnap0jlfot}-QN7cEbKIVX2ZTmlFlb%4M;u)fi;EEncdTn}iSyEO;J zQAzRd8fk-$Wic2#J;in2pTtA~j0UCDYX}YOG0~;(Pr?Z>a?zvGjUz-1bc{Y^uHLy%J>VtDXF#QHHDD1j~ z1{@h{|DuJMgEqNCVk*$b#I_N0%uE7E3!sxI?7H*O>nWTv9mJduU9%XF%R@3e0%0hU zvViGt%R^!B<@{dwYb5ZXEP@|%_Uaa(u2s;6MX8YS%>Jx%m4(rwx&xcWnLX!w%OCj%J`QKWfwo7IT}u4j@bhX7MxX+m^>C9`G_ z^Oy?u%e!(i2NQuILB<-`V@y(R1v8lAD8s$Scl0 zKSdmS9{E#SVH=nUJb%PUlL`!SM)slnB@%r}Ot(sbmSdrq&?_1+jafeA#h#G_V(@$elvh{Ykn@)p!I78hlz z5k+#3IPaYhRMmUjbmj#Nqlslw_H@cc0X%iAkC=}p_IdG=i0)S+819(Gq`Zi653)Uh z57ESW82m9;PD1;TPWk)+3kes@4`wkAGNbc;hShO!-QQSY2gUAC(k4GS8F`!zHG(vKenOTFb zFNw)fxlqdY4(XJ0_^qSQ0iz+E7s$E!VYdK=p)~tU?!_MB0Hy17hcY2K%6M4<4?qg- zRYMuPnW9MU^f3>h!{hNq6e>y#5eIlqJVLNeW5TQrIgJpqXk&v{JWmpQNb=KKTG;ib zv{_95GBzAx$U8Jn%xT#ENl1>emh5Q(Oa&PY?79Y$kQ|lbAC&tOW5Jvc@N5S+P75)o zt7`_+FD5+89zY zl@V~4w4k3TifpWJ?>=k%U$JUE<$q$;d0zI*+cD8LKQ*wz`I}W9$j;8(;r*GP+f8kE zJK4&mvMTcCF_NmsH=G!27puCePLMa9@bwyT{lsvU#%Jfu2+8?oy8&KCXk&x^vzf_o zq@euzs1BI#4(Sy8l6^iKnv>g0%Q!<6$;4BC)OA=CDJb{)#o*VJ-&=zN@!?W%Ies0n z&b7sopQjGJH{yxByfJVr{>EjuOwov-?3Kg_JBfcqM&IXWF-7Umg7s1+Z(f(4#e~HM zM;2iMNW(6&+I0nKfInl@f9x?J$f#Renq>%?$EAW^v&UjuUD3uFjwK;?4&!qA`3T9` zA2Nw>QRJihay+8QZCO)Mc>6OUsjLc_-X2cHqOfSHLJb9df|^q)2%s zi6*e{0HKVYD0(8WeCj!x7+06@*0jf`| z`O`yM*kkCgQ-nhYrp7S;y#SIWch{N8)InWyIFd;&)m<7W0Mnc&pY|a0IQkTw#LYG^D=*eudj8JYs0Liu-(^E;NU_j^fR=N9$V0?HYG!{@-%l>=; zB&O`KBhm*fsh%WI)s}Vf2R^xrJfO=siro(A)0+>afUm)>>G=SlDL)Sg%}MD}bZ{t~ zRYQ&w!7<^&LV@U5x9++3@F5}??>CzI8v|snh>qIgdV{~53{DDkgqYL$0f>#TvHu^4 z6g1MJh5e#oM1TbV$q_WIg_y&3U;Vn^=f5k?bko#dQf%D^`8V8iP;~xnf=8X83S3YI zoWTFf#9;d4rST6HW`$`)7z(7`{obkl=?;zXZxNQm_K||=kI2)+;@5jnSYTiWQ{kz2 zlK{Pr=2X=PvQLAH<6cj`xqqrD1}HR!%-D7KUxj8*yllFlrAeT6U%&mWX9fW#KQg88 zEFHnJh#EBtf+JJFQ_@3zB-`v)qL>4|hwWtu3X1_|5tcC^{N}}$1G=YZMaA7@v$xXE zTDjF9ZkFf*znH&WFE53IOZzTdG{*4>u{0w({s^$U>$9CrM0D`vZDsC0=Dc9I zvSm2JQ2gsS5&Y^+)%kW}j@iGw3ntO4cUoUUyNG;Zw2D*-f0+=zGu!r2Q{r1Ko6i4e+7zF4bKpxVd!i>^Qz`I1EE$B+1^bw1%2 zugkH}!_3a=ufQrQ{Tm$a5(KGi+6})Y(PVLnxdH1Q5YEKYh)zL|w4~vcrIe+*Qjt4cXtU!>yg_sjE<4fw`~nE43-jlxXQL~LOuy7x z_Nd0|Fnu(=>_--7f3_SfVoaW_Rj{g`Pf(+!+TyvJ> zjb3y05pW4(aMR=N_u{)&o#IWBZ_!D#by`nMRoW_c|x`}GkjTmI(7u=X95Z3gX6 z*%pmqa0ICVVVGA(xb!GdycbFAsl=OParu$x2y5~r*l;=RiAt$O7OdGeIfIj%ks3q7o1 z4`Z%F@$P4(!|rKk-y=`vaHA;ROu56`@3i`XVeyfRg_psMe7na-&L13V-?k(l@}!Tn zqkkVTH-7aI-?}hWo!f}zg|I0H%^ualwv`BGq!0wZCLE$cGU-tPhfZA9Zn?4L$4m!d zRehI)sca`&*an(d((oo!$#vjp%F-}bqwqSaA0O@(f^bZ*$9AA6e!dM$Di z{9p!Iwk|sX%+QrnKM9;#kt^iMZ~;FEL72*>+c3@|N%0;&p*PNnq(J>iow*#Wbfv89 z?sg;vY8f7%EeA}Cdx^?2)5A=?K3PB*f~s&I@{0(cAP{a;;rSs$^?w}YlEC9il(MnB zWV}xHVJ%7-TNf|&Tkae??N|Q*n6a2a!Z&Lwou)v^fe1<&Fh$8KPPLhJ<)kU{4MG?r z<-nm+mZ3nCAyC@Zx$nm*W$2$`)Nj{F|9O7dzFuXcZ%7=4Fz)qnfpCVlTli)L=`)nF zzgvGU8st|d2kqA?AB1r)n+t?HV|$D26rU_dfxda+MxLxzf{yAmUE-qI%eKn|7U6f+ zx4Og!XX00QKmpF4&8WhIvpX1C#Mu4yr>P4%Ne>QR)@dIsM*+vTE%NRm+NfqswEH7g zdRXh_lVmZfx-QG~-9rs{9S%KB3e?AIGi*uT7fOSW%>fUAVt8DZNxLga+i2QXnB_aF z#~7#mY`4nS_SlcOM=zeb2~*kVO5zHX_;~~iJ*?1KCXs{oE0+(<5UdR~t`~fDpX~Hv z=rjdNIwHyfMJiFC^)I!R?~}#*j)!x?b`ro0_dUzS>d;Sp0PfP;DrFk$0Y69dkbbZx zw+orFYAZ3V$Vk~#U7Zo1_~~~gD=7VK+(Z42_Q8>U=PdWYnrDwDrIky#_Zbbr`?0H5 zHJ_e*!WDa;|24PJy|$P>lv*+{w(Bq{HWTvd@bbzE{h8=WS52%&zDtee+nUa?NwKm| z6}=^kPj?P?xeW8g!(s1Bhk zS!Z62?lrzseqgchbOs+JYeA8{cjfI12B5SYPX3?et>_@p^am$R3~T>h>;M z?%qDVu}mvmD)9V&j)b?r~i8;qk-R50e}54PbK0oI`(x+ zO<4?pGYA}CeFzxrKNB000!U1~|t~; z4OE)ZMs$&6F;xC1xNW{RwScK`UPki?6JX9T*d))v{Et`?z-F%+)tnRsTDk~-5>tAH zLCh=$q}lKcAj8M0aHQa?1TFoSK*;MAsOLjEQu`Pk@nHq{lH;8|v&k%ew=0qmEQL`A zz;-|y1p}&`(!?Hfmam&2{tw3>g>C+qZ-Cdi32ga^Ll`!gAPKNDmf;8%iIo@tVQBly zH25zIgt#aEYTqoR^R=+mR{_w#;cq}2mJP4}PZRxlg6m)W0{B(i-M{GUfBWZucW(E- z&zqsJ_>G=_q)wdV#IxIjzN7`>pT9e|(hmzcW?^<{$=AXsAW4mUJuV6)A3TP6R8T?nI<+XHP|n?EUm zQk&a8P6PumX(tgob?!=x(WXjxw!BcNcGmSS7kIDSYQ22bHmY05NZ3@csvL9R4xn$I zJ#Yt73UzP7HF>&F*mVmI0)WM#Im`d_2=o56G$<#!dl>^-PdgNXAtci!9(%A)^=<^i z&kgPWDZfi{eeo2LckBAJOwG%AyOSMor1|j{OKB}^u}hH<^KNT2>@ z$zG6t{<^)rTE``)K+o^6p4Ep9Uj(2}~r2E}QFBAK`n_^J8JDW$fX_N>!RBPZ&dMyxhfJo~0Mi34#4h5Xs2 zIQKX;^FS*|T9eM=m7eD6FcVc5rD9xBk#*!!H|q1GV+wtNmCz&nyX*t&asl{CidFF zXObZT)7Zy~0{zX^hRIc-4Lhl|9)2QEXPrj=;r3z2(FDv=W?=K;myFY@W<+s--&zBTuP7pDF&g(_%!*tP_?c ze<#)Q>e1_hKZO(PaYm_0ZOtT;3+pNyOeU);MbnzQscOlNXWQIf{+@PPjIWHt89g27 zZzg?Q0Iul2q2ehi;NN1opQwE7b2F)Xalr}uaQ2T%k^ZGUlg4raBTtDGCHeff)4=Ao+%eFTc)>VpH6X_hoH|m^KHc{ehIO7LJZt z=)Ki@MC}IDgUKJzkxI?_oJhKfv}``({?d!tc$If?<$FR12MZ5Io-%;awKP{3Mbww+ z_B6Naw)}D$VUmnA*JtE$>-SH{Ei&?y9Cj~9$Kc!#QDpHhn<}SK!F#$hYPT_b1ygTq zKXt}kwj|7;xZVdZ7R;zVBlWwUO}}yZ({HB{>QFf6`Lluh11a*?Wca8mo)&k`SlaAk zRe;}+`?l!vSC@c4DxOBc^K}V0BZ1U!&DDlzTAR;8#Hxywr%4HdONDb(=Qkf9W|)2b z2LX{Z(p2;U-`rxtTbDoi7T@e{7lj%UGbf9ujY17KAPm9Rx@ijcIEx_8yXIe^G?8jJ z5w%#e; z-okv6aPiV-?8fBD)_$+XltVQ8#tRcqD1cyM&pq?BN9hA|ke|B)0pI-a*zAF1l1Q?I zqJZYo%dDQ2&Tad2bP*%iA(c=^-K0K6p3$Ih&^}3UU zZ_bOGU5AtKI(-~J7DOQoCfT}dbD&*J8GXVFW~7*^DqY~D**7Z7oy+|B0c}=6AvRqi zixzvFd3#{}oTRkI*e`%%} zM-JMrsYD3lUSTA@dHTlPgD zZnKLmfN-9+E@UTYDj#*_QRTJ2G!yQO@fKc3RpF+bJn(OQvJ-zI9M~enMi;m|4q@C& zcnrcpGHIOCTv2!(@?!fNM+k$dC?VC2gH|#){h8pNW- zSel>u4K2e=SMqqw{`pQzBO86=GN6`XF`rHXdg)&MZxBv5zP>LSM(u^RemKM<9>U2+ z=VtryV=)xWaMk{fZ|*SRozD$arg%S3VU>F1Uxe4mx_pM>-OrO!`%ZuHF9RTscRB3r zuJWDcSy^Ffoe#%#4mWmM`Ynyt-cDg^ofR)qXbRsvCEw^Ul)_C5oc}H^vv8W?bXt1z zK}(E)48?njFBnyTZ_YEB1Lpx}uB)4OEc}9C{3eBK6|bXvSd{{GjW|Sue4~OHg2z(t zv!V?CrTm*&&S9b)Kt>>3_TYg^8hK)e1>xERX?`u(1MM&90z5r^3URR zcutHPqmuE>o>nL&Y zfGmDVGvauzdiUUU8W*nfHL%b?ml*ZijR0<1NfIFX(?85qve7|j=Jds_$vO&e z3g208&F^S)CrTMdw2HG?NdNyvuqbM)YGeN=l97Y82xITg8be^Fv1 zogY#!QL)Ku*lKup=KZd%=-A}s_DQXYq}u|QE_B=~x_3Liq4~^}XV+fX3@L7iCUzIA zEPjKYcbYg>o38aL*6k?0+}dud+MOIu(bk@90Kf6G>tu> zgXmJ6N32A!6xUvS0Bm6Wpc#pD6Gf8mY>6WCB{f<@=Y$n=QP^V{lxs@|Y3ZPH0vws5 zoVcP@cTAqr#lek&Bbku$uYCeYww1QZ5x>(wEXoaiO(lqqa7i=yjFyE1&Gd;!6e*Z+ zSEnKafW6;~0Sqq5~_r%;B4tqlIPiO#n8y9{vWH-E6_GXil~%DzJ66dQJMD0Mb9OZ^5TOiur(n zQ$G}T{iX&Sd9V5uI#yGziwLgfba$6Ub22GPYaN9n?;VWfB~D{3W?lVAOcnahi7kIE zUi~-!{Vxn@_0m5VuigampxAO(G-r9DvTodeE?)gt&idvz|Lx(ZuK((MAM8#bn>4=k zE+i!9mKm-7q;f_1N@~?y$Ai=N)@w8Nu9l*|s^NE=nmupleY?+n zxqG72N%V&0{j?qCKMP$|a`6+7&&OX0-JkxvM~Uli%k2+HD=2e1D=zvj_|F4>y7M}{ zYo;wa_Y!UT=Twzk?jXy9Em~K9?ziZ&3HZAQ$2Bu`j_grqa9*LmZmutv?Z3iRh32a3 zG~YW>tTq-uc{bQFWJej%6d&O_gwI9@Ez4hH-=Y}pzi%BqTKC5F8qYV>^2EE1b3^*HR@TqKhyea6u-^xVuWQo&$B_nKmfGgz|YKd|6^od^Cg1I5% z4Ml+uqizAOM$F0T(BiP5xs7|ur*uZ2v2Ao$?SztXJu_Zww7FlM7q7&>V+7xTTb0b@ zE(*O6e|OZXZF_F80DW=d%WlNiqfRu*d+%4oc*{JZ@}GNb1P16E@N@}PgqT5*7eDi<_FVIOC#Fy1A=MmeH>F%m-n~Ph@Y0^) zf~C6J=8_Y>a@jo;?!_5VJ`4eB6mj7Vao?J=T5UAk*LtRz~RKaz3{%URxti}>s zi!8g(vo~eE9kT5H4R2!lJLU)ay1Z@WZgWxu{HCF->CO;j!EfIh4mH=rECV6?*g4J+ z4X5~AQywx{ga7cj?67LA2-MLtoI88$LH00nm@>>?bif1naQV}?EV~E$zxl1o=df2*}b8GF?eJ;S+k z#*zc-IKcL@brz4*T3F4!B=bj@MfCIW)SSOoo{SRx>@rcN{NN za@-0;w#W`=Cn_&tMwta9k9vg1R&%N88QLY9n!iYL;r~(~Gr}z5%mPvvmn~#J*kZ_` zre|swRF|5bsr8NYY$@p%rsZMQKE?x{;m^x^)bt!Nm8RxGf32Ik&hKF>K!1Ep zP0#Skb5nEPWG{ZpNSP640i(+v;oK@e+n$AuFnf-=w1571Ppu`YNGF}*zt|}*Rvrim#_mCgC$oJGE zTu^>>+%oi-nx3xig;6n&@Kg0m^1ihkYD`OP%?fIIaY^6QCe_$dbd-eRPNWgmE@wQR zdFAj}&PZGI;x(PaAN|f|qFI_ud1$w^WD(*Wvco)cug*!SwXh1jbeYu;sN>jV`sKmT z{1trl?5s;bHHR9B{fsz2vP*WDB9;8W(!nd;IijaF@$d&(+Zv_SrOR`GDbKkkeYn*X ztwg>2f%(^U)+@Yj>d${3YG8ip0(VxxT7?Hx^u^6Q@IHPfNupeOS43H!ue#%ji@@cK zE(H6-wIhyc7e5WWt*w`Nn5)%jgmqP8dM{~f)ALs1Z9`VPyNVkZ?>m=m>NZvU@!4;mPGq~T^oJbfy01;~WxHFlb}@jW zGZYq))~>xszDE_h!n~;4?k|@PNsm>J$5;sXPLn-)rzPky)zDw!(vJPF8bW?j(y6v&^~I!n|sq z9;d9iTD#p<+CHmsdoqu!`NyNDCrf+CAY}s$h{5UiJf{L!u|um?LOng?FvLY!8va3M zSo<|8*t4Ods8b}L_3@h-VW3iKm2Oc+*tz)|5*8W|h~rPWr~Mib4)5bs=U;s}ig6~0<_EI!<>sFWzuAJ&2>vr~ijr$qdqYJ*1d{iMbjCmo@L7<0B&NoTg)_|}z4p{Wu_L;Mk z&B~PXVdWc2%nXZMDA=(^K_Kb5gSNl8q{%E=5GD51c`mNkADLS06>muJ8ce@B+@lMB zH+GbL#U=A3&sd)D4T-~6Q(Vkqp^zIA{lX0nZ!^PCc9|!6dW!s5Tit7Vbm8S;Clj1O zX4VlzmX}{pkg$yNJ-QMV6GvmBtTZ9M^};tKEWjDBgl|wTB=cv6p^O_&@^lvZvQEnj z^yr2&qb_h~gz4JWoaFhS5Xd?$!JQcfva90|_;i##@~IZ&Wx~B15-^+XVYYKc{;bos zo*7~7C3eb8zO2)_+!t!x!>rsoDX|Bkm#f&hN!<`$>x+|UoY6~WQtSN7A-LC)ia|mpjS!4D6YssnJN#qUZ zZ=Z6Sj(U2Zu3GnFjfyqwDJs0ndbi_a09H^g4wMl7d*DWtxef$jtkBS zU3I&D^%AnXeS{tT5#O%E!39Sy#_Hd24!M#%j%hwa^DAp`GdA29n(-r)Lpn< zkzudDy0iH>w77m~>;$Yd@wwx$kJmGYQJmND1;V2{hpgW073F9!y=utQxcDnT?DNH# zUxLLk4PHlM1%Ceg0h&+gI+k+hkifhX`SNF_mp)#*6oxW827y{ux7S6%J5uLUff^pnw}PpI!xt3S`&Jrs2MT1mOq{9TdfDRnd62|}}X z5kFSz%7f}g@z$zWe0t{OW+1KOUZ5wuho9B0%=&?C>PDG5jmnAZH}>C0u8ZhtRstd8y~D4XPH7d4&-w($1U6 z-Ld7#dU3t2b?JNpp;D=6R4L*3CL=lN(U?H{+>LJujpN`W2HQ%kgH7Wx{T4EX{!P5u zlR@SU<6zMlS;Gqj2Om?+>&Ig}UwWzM*cXm^X5SVyw8qwngl&sjz8+O#I$<4X0uu02 zcUjY#xAdyhE$H6WXdDM;Da&46xrk_Lroz+V{hh-S4H0Fh} z%N*tIDe`M-aj)@GhnI(qK5+(>q7Uxc`f+^5c`tQ|iixGDD67VCpL*ds5eu+ovT)t# zg=GG5FO+e^D0gR}Z_|{#fS0;7X=LeqIny9%Vu{uDY16n*LX?*}ygF=@n_V%mNl$Xa zOI^NnvjVESR20}WW$QWSg}RbA%FT9*a<{GJ*0>i6D^e%2T0BvtJhb$+M#)j`@f)_j zCBd{K-pAjJxppy{#>Mxk-*CsrT-!B6x;qtGwuU^iR>4S6JcX zm$rltolEBnxg4Jh6t^y&FR#6klQhcRs}Rt{GB9C!SZb(YoLzmZ^K@VnZ{zn1H4=69 zME_LstCA~|cpT-!5#Ef;U zA^8dSm|aN={UDtFwMg9?OHC*E=b5!33}&0FV8%WvUr>Mf0m5jv-gSHld_%97AYAKhdg(=J&^~4^ z5HkeYk68%hmVnauMGQ<8Zd_uaBeEwS6mZkhd6bnY;&wwP1nUcJ{v2hYBaTam{N$yw zO~*kPG-YcRdcR{aI}Ktn2ij-nU>6l;riWFtf^ZVA0odUSGkvxCBOb@h+Rj)9;Ur$Q zd3E$)TTyCs+aj33E0mCTiJ3m1lOafb5$9(|7RMkD(;!C$Ksbp!ZHl-g z$U-B8EY4BFO+#Gn0O2G^j%0E6*H9YTc4*Ll1+OEB6{U7g-xA+-mJa~oBub3P;vY&5 z(fA9!GLI@yppjOr^e~bv2**G1kd?k$>-I~UH}dB=m_c-Zhu`@IeZgSN$l>;9f^^SR z5+-FSU`N8u&*E-*u|rGR*)Lc zUdi5chpf~3q!(uf!hxMhza>EX_@pfni64z?%!X%LS0YVTsece&`Sv$(|C;LC3AR_Z0ER34n1Qpe5uOTD=r9C)m> zHrgkX_NQ|rw`6b1FJx_%FFQ?RRr_+m-f~V^-O2c8-OGLZ>p#u2BOdgb`rl}4ZuIWS za>@*$l6HUWMEq(i-)V4?s0#h=IMKWxrz6-y`!kfdx0&%caol4gr2aR97X54L&%EAN zQYdlO>YK=3Z*T9}ddH>I{F6>k^q0!J z{T(NP(#TMa;rnT4%{v}`ckoyDY(xEC*1Kzysv>rV@%#Hb@k{&f_M7+5?kxSd7Z^15 z!fRi{>I|-CW^`Ribbn{EPjrU5l$yR$+N-v^_qMy6(Y@6iys|yDGq<+>)*H8x&i`;w zy4&gX*eUu)oA{LtrTrY7HvQFpN7bm#_q$uw@fR)2V(qTHxxKmfvP7Ig{o`Xge)d5~ z=%%Ti(8bW5jqgAAAKWiC7^toIop3duHruvV)v2PaaxuVQ`=0uRBkUR1`a%MV{qH1| z^fd0jzV!Y~Zk3W7OmUVN3EDxPwycHa z-0^Ra_e6rqsV3=)9?13|fy+f6$aC{s7A2K=%$jJ^_%A8?&V=#&>O3BblqWLm$!g?< zkNLoJQI9X16*T+#S$$^rK%Sc`l(HDbviEzRllMS|eZHiYTb;*~J-KQ%ijC=$H>iZ! z8{ZsC_KYTV=e+hnqFjXRIS)H|nS=#a^rQ?-{rDvfI<{mpiuEbvwC6lL(3+xOU2K0c zLx}}lQk6&6u&RZjUVlx|Cl}vmN>gHCyRBL&kt}S_nK1Yv1uXq_6x&yBY|l9b_IN04 z|M5aHzbEpfg;{kTMSiXp2G4z;qMyvPk=>dS9%(s>#h2f*=S;BjGLgjK-}m;Vgh!~A z=P_NKtA)vvCQ`s1LuWTZL96SvFnGxmC6>;T>O6`rw+E8>P+ya~ClZAfv5#e0!FnPM zMhE=qFt!nFj60n>EOlvy@hu(af!kkA)p+tsIE-GhC7Es`%c*3{)OzYCv| zNFx_i=J60ciX(>XjFK>Hw7x7qHC+CTibX>9e0C)-4r>Cv~2B3|AUrb4Zb z%9Y(A>-eZGQBU))bG(W+ zCX06#a?to?$a62*X$Z+7px$aypsr8Z=)MV2ARN3}k1Wov2&M5$>!!KzQKP39#dfph zpMr4k5>v8xX9*W=Mwc7RU_RtCa1+8nVbk%=EGt+DfSt5~zUexmmgj0GL zOx9sKPUj2-;q;F|m;rHmu6kmTpJoZdNx8Tt$nk75(^m^)@j5LhzysFh zfABh4R>D-d|3k=|heP?lfB$As7-b(6O-LH1EG0`CB_gs-hKoWOWKWi;W*9=YW=t^^ zNs8>tl}I7G%bF!yO15O*x3S;Hc)!2j@8@^_?)$j^8i#Q$T=R6E&&T;^(E} ztHkw@jrOh`&(B}ieK<{;U-+BfLxqhtI0kfxGm^04J3Ewxiz+RN2X-gFsyz(wU9EgJ zn=r+b%TwJq7I43L!@=+F?vDHppgPdhBYXB$WWU)8src_+sqEV;m7#y|J;MK%%IW8y zt8)STYY&_9*pVR?)DCfa?h@mc-^-4E(dBaIpNVaEwrkgHceASSZSwm!DsTmsXNCK> z`xMwpep36pMx%78nh%TS9)B;N+npsMU66MP0tPwq)e+*KMz* z{*@n&<>h|&e*~`YsLY+$yirbk{dH|7FTne)N4))dR_NyFspcwXnVi{q8+hu!%cM#H z)mShu(t;ZBU4Gx7%EnGxD^H;R{o?mn)K{$L=JA{;8O`mD>E*z$=exw6e}>aMR03W| zVAtojrWc}`nyw|9G@c9cX8Dq5g!l01=rk-}!&Px)tDeK{qeL&TI33aD8fp+a^_3C} zlfSL)H1}G{!O5!9-z)#9E?1K1tRAP+D}npRTOpQuOR+G&6y82BaWu^2t7m@~O{oGO zU7r;p<&bWkHo{Y5D2`5+ttL!P>v4)*)B5n0A~>vgC8~8&PxMfS!tcZ@1cyVqjVwKr z^jwA-F8rJP{!FhFHg&S?WjedGM)}gWDwfPsJRm zP?qhQi!Qp6>_dxU=mT!$m*jNyS$`$9R?>p2pP$1GT4#HwkW0))^eU5O#n2VicLw#G zZ*D4Esp338s_SySx;Cm8^xZ`qt$LGHm+N5nO?%i?OLbjlQKZ3-b)RDvMP06g79)Bu zBfQ>9IUH|{pa{NYW0d}(FpDBB_??+5E+)oBmy6eYMi2I(Q5+3ZVjr*N7DuZ(u`<6q zs~4m-DULR|F-k!7D~ZNRu|qB+dX)>Z;%HS9R$Z=2IMK*Tmy6O_N3*LeG1F~qD~erE zMULx99Iv}vY($~zS|kO=2ki8eD~X|k;uLjQ#L-^I-|al~T3A`xPI@{vy+fD|BTVQe zzZKZ`v6-gjTQ@NIMI9F&oSY0(!@*7D1Y&ztaU2)lODx<=;r+$f8XMIUWl;(fgo?pdH07AWp%mu?rw>5&)&8n!|inud+a29Ifqg?(4P~TJ?svuKtBQ)gm-mIuTo{C@Cw6oRKA50hJ zQEjb=d)Om}PIf*y(%@AWqtKWtn3&>>x}_Cm@&;Y2BE_}gQr_|KMm8;|Hk z_yowjuA}jfbM=R19HGQ={8gDii=+A4=43C->PZxJwwfp3P2o*^cdTWwF{6Gf7b{JP z)f-i5$Ph>KwcpXM7f097&)rMfyGU4eH;te33BNz$Zi-2k57+5*DR#Z-AwAe|lsKBk z_cW&M9FAOUrOU;OpVhl~B%n-+{r>U_B~}l?k(4fm_BVPW#crJClfn|TLJ>SV-7jJ( ztIPHDdmW8G0`{!`KRl!Qmm=yJv2ruAS-vSMgjN4?eLh+dGE zi#S>j$dO`ikiD0}A~7R|zTi|QC0zwSR_>+Cg~ZM1RsL&DDS}6{BBa>sv%FJSF0N2w zIS?^Ca^mOROky49_unORA8Xj91n`*yE`{Rq1HF{eiZ(z zXR08&GVsVapz#QHbJ@US7a#2|$SsZr$3tKzy;4lXFTZH~PUA<{8Sjsg3MF&BWUZ;8 zX>mjgr=C{D`7qJwv|a?u$ZhT#8oy^>n1poi83!wCH+l1){k653_4n2@TTaE}pLv{$ z|6_N3bfxP!F|}8^zJ5DYs>~>1(ntA?Oz6$u9Y0^cjZJ=Fj@+6we;=Fd_4=yQr({`aLE~pvTJGIT zy?PthgiHT(*zrt@;>{npz9l7V;pmV(-<2*}(%woikSJX^_v5}B8b$^E*82-mxeZ-K zsh&uCgQ3&a?r7K((8KN2$bJ|_^J&FT9NcQuOV(<|y2wcXZgEW)U8y29db`ii4SlH) zb2!h@#efsPCzpg)+E>qgNT$svBz2fO+e!a0RMsf|&>EecF8xs~c`+L`C7j-&aGsz@&mE_KQ9 zy}jU197mh!S~8f*n`ryux~l;v`GEeIqkh3OAm__yz)Z@+rUiYu#1Ue|x4zn$B=QY5IgeCBDmJ38p((qTR;roAIw zO?O8lLz)UwL(C_RNZxe+$8-%zDIKx9&P8djp~Gi!PfoHw&dg6`;VBO~$?m^=zzwa9 z2u~h&Lwg#&K)AE(u< zt=3Ktei!=_=NWzII+y98b%R7Vw4Pq!Np_B`qEr@3rV&d?KFPk{{icBg&^6-Xcb!WR zcgG;;AH~G|huvZ`sdMq(PAvaT4}R3uhx1H1TO^41o!W;BE5g?@YE|3l{4X3orV@Yu z-komxJGsXFb3dq5G*|lr_vh;6?X|@wW54HyEpz4fwzdjBrshZe=9YYUGITOJuYh4^Q=ACCR-h&tXHKU4Rc&d=D;OOr@yr1OU>g|*#+)2JsyO-_qt zR?&kW$Nj{GD?^~ZJK7VMcrw%z5qQJ(H?+pG^bg#ZqOPe@RDyAmW?EuI?bLvXD|!Woe#|xbBRj6gHustbR<nO~qiIq}lAC@$ zDPs4pN=7cJ;p{HyL*mV~l-R?n(8=aeO6=h;Rq{$F{8Gdw%v@tj#n3ALUk;=Z#nHNY zYcB(&p+oM{<`c?oF|sZX>Ouo@ZiM z?rLv4npdbSNT zeceVr;bieRbsQy@h4#FI26s^QOu2AhnG!2?qGBw?H|4@D=3Qh2N!5n?(I~OKMPCk; z2WzY0C|5`1t4qaDKh`F8CPyD_5-3!uDaPj@uaxu`(HnDPt-HkCdFMr8k6q>HlkZx@ z(9l}}n~R)cXcb?PDXuLs^gh8NO6=jIb&PvnDPnx~@kk5ZNaU%#|t138h~f9(nzR)4rWJO0?(RK=NQD{>C!!Pq$SR3hmY&Fh$p zVT70%S|y$R5}6Px>zg8WOOzrg2P8|4);B6p0WeYw?Hwa4hSt3i=A<3vI;E%f*1~)| z>TU|Rd((Gc+`bI$gxiLeDVCr0m$v%vri3{DwLYD7y(RmG7#fN^nfA7irX&~@9Ihht zFzflBxI0EpdL0(7$_@>=Le!j>)YL0Bgbarn`Z^GB_j8P}3KnT}HD+c9%AXq&aWP8BNoB=mqS=O**sd)>2(vjYJbQk06#>yIF?Q7Y5er{-U`U38cR*h z>dt-kU<-J;0i;h(W9dOs4vfzfo)&8oriSozE4~X-I+IeOaXp19U7@xw5syW_T~c`+ z&JVI$zh;E)`eKn)OH&#MlmYqb_F{=03PD@fA%xz)HoVw$%YcM|VZ6fr_Y!Elr17gd(3b63JJsZ*UzL7OJ^B( za=69-!dtGP-_C<<>UCDaYKc3I5un-CD492Qzb+YMEMt*cH&Upfy<5vhRwx4&w=L}P zPApRLOr(i?EyQn?T4D@s0O2CMp56=K;QS~S=`TD*#0;M5VgR1$Tqwjvcn1NAMT+Hw zQrUI0mcR-%JQk^%(+vf)o}CAIF-KoJ@9{tsBIbKN1V7fj3AE+wqYwuzMhFu6Qo^av zsZ8^OMZRUzP8`J|#opCH!SBo%KunAa3Q-BS>wSPi#NcNMl@||f0G@68u*jp?zaXs` z4$iz#szb#|{^$wqwchE&0MC@OSR~?i>IM*2gl}ME zDZZhGz7R>6RbSWs`f&LYX{9GE(1D>Hjvu5fEP?gR>DgNA&pE%=q*solvRx7EQ3Di0 zDDi{KjXwlVUy;q&&l^Cv-4>d+5mI95nqauz(nKL7qLc1imG6W~Z3avn~WZ-6>pjgh>Zh@6Q` zE_gyTp}a+4-eP2Z-xli;Le^A?@H{HsT=DnOQzr&nZ-?#Q+W)Z+dVZ51H^y4Ux&wPc zdyw#H?vLDKYt|js8`Zt+)yD=0UJ3X2^GF3b?9sq4Vq@gp3Yt+v0b}1*1axdl!L=gL zNpH3|?(^|M)S!=?$Tob3)q1G-da*1M+HopaS7;8ZIgXu#@32jXz7yWJ717+RE!Gzvx;8yzP1wfDE6G7ruWUILz*lItO>{4x+qcEpI#&@1-_agT6K~tU zwZG;m$g3NDf`=iHFrb!pfC&SI$m)Z}A(vf$)A{Fla`a(x#yRjERlDb8&4ac_=4CF| zFWNjJYaX-^pNEoV;X4)8cX;RbA0a&=Yf{XG=Vgp89IKTatbJ2Nd$OXQ0+SG`f(W*F|7n#->a5QR=>}$3^&PSB{%ZCO9DTfW1>?z;<_K%J@S(pNWil9pzaux%^c_ z`fe`v>f-I)ji01@yNdU!3%D5t%*M;IB5AT&Y+fdElx=JOoA?EKhy0*1$H*Q1zZ{IL zLCWn2jX@NP2&(M`Iq>nb+-Fbu-rR}51ZEZea7j2*o9VP<-$h@kmA#X>EzvLRd#|@| z{}!ul;)_M*v$Ow6z`_Exay`emNoLDqa_COpqm{A9)xV%y?8U#Mk^ zt0+G&-#DB3X|t2Kv32G88F9ZKW?5(QNK$(9S;l$Sgo9~gMJN}WcwR{0cfZi zhTU3k(eb06d9~Uvy(>3Ry1Q}bfQ8ybR$=^(gJ^-dM$zN>>e;91C5VZ@??s0-?rhfz zUgM+i&9q3Ayl~jCd(~X{>I0H1=)_b4ih9w=?jec+;u?fdn9p7qTP##5qQy#MQRbIf2cpZW)DBw2WdG85%^ye z%n3G8G)NdWi@W+68zYulpl6%fC~_F@SRg`nNfJg0sCd;O?WK5ZS~ z$oBM0JVC_J+FK)%mqG0dUjTXgV^c3bn^?9miWbth(%_66Xrc5${CwhZigqXj zs5Yq>hoy6jtMIv!FvmlpmcS76!tg|vCqxWQL>SQx z1en$o38M#qRCYwhDtPm59F>IYI_(}qWv_P`CcqXaiI@u}WmI;KtYwhpiXjn0vskI~ zOUBZ_x=*Oz(ip5+2f`!n;-^LhoOic|pUr8u>|kdFBCfmHrs3cyK$%i zgYwPj{=jG(ePmAaJPG5%<%&YU6jwKVmO$+goekh0{6Ualsk;a9 zq6{h;u7~AG^q;x%y{I-b!>=~8KGU(w?8;XcKL@<+AKz;hiivl|2%JQnMw7^^gWnl| zBwzJmrJ(v+h~M$gyyKU1or#e|_2V#bfGO%g0qPB+~Gyx2y{ZLjsFRP$!L%1xsLvmBaln za(A&vy9|EI5??Icq;!(?MDy@vEZwBoXkGW`H;CUg>9zbImOi0pg^JSANv29q?c4rh zSU0e7ZWH(raL+*W%IbY0CPXD5PlHO89#njN^4ED(Q`^T^65LiX)X-M$HIVJ2%htuE z8%8u1j?*Z^1(5g9o?0TUN)Og{K}r>I+%k_Ns1E7or_X*ny8?z}MvIgbP#p%HM3UY^ zkN@Ukk;95t0$L^s4Pl#-{YuU%C`9FYlbtq}eqp}?v(MBX6Mo&f4sf)}`?(AeIAe0V z1(K(r)?i2P+f;wjpjEM-3gnD&~BO*+GtLm@+%a zxnI}Tf$(pwPm&iw-o%01FFPUr3gk3FqC#f_fM1`<5Hw`~aO)92XS=(joeb-wa3V%~ zROQJy2@^7&w)f`YOHWVU{=N=yw9RE-+XTYfoFW!+M9ihE4sGS!d@NGZ+`8f0IH7W= ziG-Oy)x8c>SZ6MPgC;^)WH6g*Jcfu-wL+i}6!S4ckas2#<7t&cW#8{MNRR-?bB#Cc zFP>1wB9ErgsO);#Yv7w(aag2SZXKi*%QE(|4pI^r)kYyGxLHD_q6-nDsvk;~z94Du zxYtkiL2&!HO@M>{=m(``aPSRNp}=P02R4&MpFsdf3D8xbY zDS`x!Olzovn1CJC)K(O|kj*hp;Jg}b%psptZ($u5cz$b=%ALdka{W>az#{GVRqdiT z=4yA(Z*DD){e_k6in!WiNEkDps?1uRwpONXUpihO1hdV`0`>3C8z31bEhnVD?@VRn}0Nf^g* zf~X6=b|RN;0|?jY^+=^UoUj?=EGdqoN(*{0bSv5*t->tylY6V+)yud|AYF3zxc7J-#9FzwB5YAEM=t;1FvS z2?Kj_TDPUoFUNoMa~9QsoF)40gaE3^B~8ZmT0u7KPY@g?@+T#roNhSY)t|puHXmqy2aMJZ;pF z)ne2spKk-$Lxl_gE>-<2&0cU5;Ap#3wJ!| z*#O|c6w{3%;mZQ{Cx{r<>+uZh z);=t9uw7*y9CY%aRO4Q?2oe+Kqe;q z51DXAN0;5`j6TE^%@ECJ2>5 zhZuloz7+~V>8yjaO6O9j4o=tW{OviGLAK*{G9k~6Q)B~o$2Jz0=0vf4cGj^}op1J? zfmF?s-SM5#XXOQ(RB*MS1FAap;B>e>mTvMGRPXPCv;>nuTH zxr&Q)1xsHoP1mxUnOKt1iTB$*V_BxZ)LT0jMoo2f{BB+1D3jnkOGu6sN_*P}DG45i z@5n@oZUA9chUKk_%s%lT7HP*&Q)^K&E<_CNoBU+LHsb&DiO|vfzN4s zZJ9CR2{m+FOy`0Y%7D##`fMUKlqErW0|?JqIm~wmWnkQs`>f{9hikTKEB*nOw1x>0 zW&tA?X9yD7{j-mau=M)BDw46xF3gWuh(!ui)vg2S_~93#@59LH zUMPbQwi=N8-##qHe3TIJD*V;6-%xNGeghy^JU}6MyO|Y52#Y+H_Jk_U&wAFNXA>CA zS4SauErtj$qrBp&4yPKOvB;yTy%4`BelGnN33EYzuV}1*M{gBkk;k%nA%2M`$G(c7 z5WH4%go{;?Bn+TeNM(=8B27O^?py~X^N(!MCtT_GhNM)|$CrHx^qpg_%R!HT!WwgihM@FnWR6g$eV$#v+wN z3#rm94$&)c4=g=g;p-9KeW<8|v;OpO5{512iKb341ArU-xMP0Wk*`t46^+nxIJ747XNBE_XtTo zmOx&yv^8!bCfTaafVN2P6~An=Z$mlh)8p4iBPAYpHW7gdpO!lRSv08OB4OCf&)s{` z2l0#cCo1Al25ifRc@Civ?S}U^v2jLYga|UFUuG5Ly*|}*{}9R`Woh$c8v~F`ObXE- zCve_O-CAw<32BijaQ4sC&|WbIGV|bxvwCYwAn$9B0ZVU`L5f#wq7wF#ZlS&Q@KzE+U(=h zKUn0^lprd*p6eh1){#lX)L63CT-=96YUFf4aJKGEfSkWU3r#j-0Fp8l_4a=V7x}YD z7*8OP${wCI4>G5?5I-9BUTV*~tb#0}LQEO`g5aT@8-OIHLMUYwJZf3XyiOvOog-%r zJbJ4ZOXuM9+;7PM@{1cG>{0#1|C5W+#*wG$YaqL-5)+hx)Ad^59eD|4d#}!?Y8OqE zluz2-Y1It0Q6yq$zFS-{rm9YFxPASlu|Bqj+D_~bj82XH9|vQS_*5d{?lPE;&L&~l zYI+Ygwm@1O+y~mYu}JY>3mCplK$3j*1!0yjaks;_BT1fs3_3edWcbv>T&HkqC+__w z{`+(X-n#qhZS$~) z)sgsdLIjicjS?b?i@x*KKw5jbsiP=^o}#ya#dccF_c7zhGjlu6A0EX0{ZaCVFp*%H zQ}DjHtGTKH(h~X^a?Ny#z**af?ely4B9SWH>C*SqTJFg8hbRO442PNSbzoz7<;lq7 zYrpCttx!Zj)(jEz-L+&Bki@+kr#NGgQISe01KNrVu@*Qy%~X)P`VPx&B+8)X*_7Q4 z5+(!+PheJ_fsY-q!s^!xCwMq2I8-Y90Oa{=S z@J)l2S%Q=@EjIwIu1JvQc^*~T^USrUskvBsz4sUilPr;QDr|U^NdPVe?Gb>g8s_I& zIbi5Y994QaBRJwWmd-JuqMCw5N_uz7)GxlLHJPF%m%!zlE$X*QjfEd&kTA5n;GRQ2 zat4ch>unZ=CSmy6jHwc9p!Qx4f)L>o;K9wTKuVhHA#D(UC31=oK@Jdz`~_*bg$~I4 zLmy6IyDW^vZ6;KN4Tu<8qv?saE1dZ7nJ91}!3{Q#O&4 zXOa+6LnbFYrm~02uY!Xmz$oFOUq%%K*N)o+B##{(c4X@35Cb58ltm$4UmGD*ewQU- zF5E1mN>{<{?Dt6_v@+(wqiXI%jGkQ~mHlAW5~zLS!o2DKul~U8nPN+&NpwN~h$x2H zbUnpS6JT0>>j1g!0SduyK23;-VVMF$|D*npvyWquNAtcx{1;)T9RCvq94|$zfFVVx z(O~!z$SXkLOL%Vlr$m01wDM!Fg9?!(X}(wfsXf~Z8)MWwP0|97$F4(kjtMsj1CEF6 zPmx3!WSIP353g!GLA8bhfoMU2eCPL(w_f!=gECWm9BDB z1wg9w2c)WbGL_vpa}i{@n?u6zC5{N#?E!$o989E(>kF~UompZyR&jR9SpB7o^9mO(+%Xyn~zBhv&?)iYHBwkX8VlY!FK z5dx?GTnaVS)vCZq2c5L^YA=N=4x7CbSkg{cFN7fzv74^sMWE1A;kTy1W6^ zKuTs!Q422bv2>F-2|KSQq)j_>=;!jUDmYyu(F#E}&v<}kp-1#6vqe{1_ z?Yp-(Xo||8B4S`viKXYmmJEQSL!KyDn)HOFl8X^91ktw?xA1xFjXnmdYBbE-! z_xfm$CSq#-Ix|@TJn@}o!(d~Em*9yAA||NO^i|3l$oqC+`$ZL`AvA(sJo7Q3Jj2k}D$*pUEM6k>k-Farp`w}3^erZqzFH0D&NVjmXC zr0`ayRQ7u3KLiOoa$;o#6dZ=%06dEip_;D73Xm|TO^ud8Ug--*kFJ4(BFb2#SZWm% z9Ad_-Q7-??l_d1YJr!+G@T0TyAj_>$Eb>IkZ)h)R{#gKvv}0$SbR%L^ZBnW1`&|YJ zm6a|ej4JLh3bBxB^9|DlDSgaw%zneN0l=@F+iB}+l3C2#NZTLS=Y~=sO4g|FRIrZ| zXs1P2Z0Ug;RQd9q>7-vft|Es&GMk9;khBdY6r$_p4@gUJWKHP=%7Dc#`p%x`*YthO z^!!TVjsMu1n_I)eBZLXhJ`bY?=p_8HhJlI>EpIZ#&0NP9v{g<>x#7;^j=vG>-o5>{ zj4D09QvEB&c$6UVwlB6Gf`WDN3}8Or3xxocFc;0UaVW&A|DGk46>8Z&rKV~aNBZ{X zX-#**?{~-Q#K-L|p+p(pG(-(*&42%kTZ10)=Le5Jg`C=$UlISc`7AvQ{)pR&`yZo*ZF|a2KYWu@9DnCerVog2C2ZGl28yf|ikdj;e^faVW$ ze_R9c4~iI_JNc>}((>6U5;2uL+vc!SBw{%mJHPdIm}&o)oBSe4m>`S=`=SOBQ}d7g zhrf8I(NFuM{2^EkBPbNks8Xf zP08zn>@F5-pqkn;mC7z*k&<-{T5cqaD)BRw{dcW83NZvH!v2*@wstRUo2AVcDlX)c+(;%IPqm#Q~6=x zh4y2Hrf0ikTMGP|A>5y-V>x$}zqdmj7R^dVI;t*3E=(_+e zWAKu$pZcH`_tGu$${ZrbkZ@vtF zF8+A~fQuP?Bn%QnGw}@!XZuM)a^^_V`)^o!MWu-Le=+2UmX=9^=pmd+z`Euh9D5=o zi86RYDDElrd8>_SU;nJ_gQdgruKX*I|KejRN+I{ZSEL$neP#e~;P2Q~g?W(ooowa! zBokd-%-UncPF+k0a7W*q0l*X94PIeZl5%IAy%z!FNrHqzxq#6Wfs_2j{c9PsjP%@R zk~!~X8Os}LaREJhw~{I5Amf5UNNlwDFmnSJvU02 zh_}p}3NY+Y`3EH5J?X!Pn|n=8Z4+Q{S7Nc>!(MZ9mu+ zsJ%J{CU@s>ZrGpfx$huzY3c?uX)jjc#EN57LPwjGH>uF#E|1sIy=3{t<6g zmgf7}ZaoJ^EF%gmGS!Tq9t{kZ=U!X5I2~#lFBHoYRHT+m z)I3auAF7^Je115B1rjC=m*H5gzP)7oJ6-Z>!Ew0eC*C5jlf^=zXh3N}Kh$M~aZc%4 zo>_V$aA0;i?vQFD`ZB^cPjDfobs8Jr&I-tJi*Eg`up|!sqn;8kA}zMB-J$+ z$lv62`~&zr5qV~0H*paymY*yX98muDOr&7cS}Z#l8D@WhPEiyPPk1UZ1_FqqQYtMW64@Kn(t2YIxfT z>vF+7GpD$3Jz*>QL*dq?xDcK;+xjw`>ScjE!D0ht8?;zvl^naq);=5bTXw)0Q;Jiy zO0?kmma(W`>0D@o7RwK^;M)ICPEJ{1R6DQp zYOFvRPF4T2MWhI9R_=)fmu|+qzJx@V4f<$ypB%e=+{0AQm@pf3@V?$!?o!+Z;Ijo6 zSH_zD#k+-6l0vM0o*<>GPEMzkrQk(}TqvvZy#CNl=G8NvSTqLvW!PjSyT2lZlWj68 z%v#3cz9j8zk(P!A0_^>1;l8FeXuJ4Uxs|)CkLR#1&mw;Yx;Mrg5q*%FJ{}UPrj%!v zUfuh#qf1Ula3t+Sqnr>~q(|<~Q}nnq z>^qCh>MK3>Tk_XquXwAPIe(D-Ih-$h_R5F8vxWtIY3@a>nWdSNiDecpgNNaoVMN(H zLB1cWz1I`91X>F%9MfWDgg$8D{Zru>U9>&B+?xK7WvUHY(xI0Ut>u3+fs%L zd3`-uzCo_5Fb7a^&u1&#+cje~GB$nofl4XPFt7CxHo?}_vGexJO(5N_DICFlY}vN6*^gGjGgq~5gc`5S>fkPqQ4M6| z?N2hx*J^YAf+oeq=Yw9$x~Wo4-FMp8lAz;K8Dol%B0t>EZZ5gvV}1H*%U7F>*YB)l zqW7;C_@_!9srESY*dlUX^{4AEnOHm8@hBq%iOyzj;t~4HcA_#*7-gxE- zCM&*{Ywh#h#TY>-+R zE?J^6hf-FGW9!){p&hfyxX`1NO|(+K@oxKxplM(*ODNpdCdgRR%n+(08GkIoz z7O!Pb+uNYw36jTOalK#F*SQRR`+h%_Lt!mp;fzg&-k4u_oJ|HtIKM4tub7U)p**vc zC4nlZQXK8uoRZFXoTP#-Uv!6@5bXJQoaEU=k;)c19iPI1DmdDjN*5;uz#pNxa^KlD!}Gwkxr>_bu-mu=8T zbGzl(wc{S9#yqy2i_JE3flL?DtX}Go+xeqhE1=+$+a!Z|0vWDLrVVKN}Yr zL*_9qv*6;)TG5xVthGTOO-Z!i;>cOkm$+4Hlfl7R{I_6U|JGe0Dv2Dco+lX7RU^m# zr|W(y`OSko!OHiUDuKLtf-#*nat>7`CV6H~*Bkx)9xv(h2@t%s<>fn2lhsT8wa`ql zF!}?-^YF`Ea>yWs~IcD*>2~>?lJN*9N$f%$%8%wphh?|BZYg2X z47MAT>*F(#4&TxKRLWfID{kgqp~><-ikq>&sF`XgG|%Hh>6zJQVU+sLO-FE^$BeQj zutC<8cn~&T%RI;CR>!_A);A;PsmyR+CeMda_N_}(uLBt$$hVPi@J5-<`7lC7(If{Dz_ykBSI?AZ)FpK8D5a2_ zajl)RwZQ?ln`~4qaw8WO_*NwFIGKyF{|%9dT|qDesu&&FYVaL4m4F8i*tb|&2q5S2 z!GR(BDUDk-U0E$U@yG9j`c0XZLWb6PeNjH2G84YEM5g?9`*aYtThVjpw0(UV7ejzg zs{UM1z09C-`hkOZzImBm=G@-Ofv2Z>wDr-f%(i4hR<6oMQc@Dhv!qY1 zz1q=w?IG9e%x?KHf7QXqwi>YS$MP!dBR-CnWHkv>j|8r|4e$&G4(tvDqPFjOmz~p?ZJpVD zmATWXbLpqGBYtbLpHt}nu_V|t-o7>~u1STv|5{w$k%6h}J370V4a?oA+n)kI%y_&( zC8gU4Jt_6?nclo)p}93QIu!qf#Ms!HPWX80_15xK!S~(uRkO!Fw})J>etv%`%z5G& zPiyOu^5RmBt^=b}{cmLNZ_4yrr0J`utexw+Z2eQUGqUK>kE_8C8|Gl$!1{zdEz3~Oy zo+n2t=@Id%S>`#<>`UV_;zKiy1gcciE#dn|MkX0szwj+eiDNz-Zsy8OZ85PD({b;A zoN=MEj1}{N`Yrt9K21{Fofz#gc?wO(gtJxtuXC~^XAE~>vLG(oQ9?=cN|?d}{Tq~f ze97?+EKNSwGw)|vhTr|{f`l(`=9^?#!mTHtyYysQa`28)X!%Ssql{=@Ct0O;VuDV2 z3ql-S%$$6ca~B4tu;z_E(TS-kGEp{o<89_Iazo9bwyG1OJ?f{o;)2v#PIxTjLa%Tf zk@7Vy*xNZH%M!!!Jk60a$IdM~#paQFs$(TzNxCHmDu3KM*RfJDKHajyI@7dZ&_u`u zIXI+ZQous}8Whj2n{};#SLWuLFG3md$~?=l5_UM<(ldXL^_v#(%2c#CraHp-r@ENA zjCl6PS*8We+G?gP%=nTr!W90;%}W_UF@5J)X)4=^(bFr8XXnT=FJQ48bwLKRonUZx zVN{)1l@Yw=xsE|vlU*2-8#4qHcPg%hA8;irv|Vn|DrE_Lg=-0BRW>OYyjkeNWHj-$ z!G0Mo^kjFn5v~2{j+|Flm|MaX*3Y4KCMJIjJS|t~l&q0$d&9`E+*`)AJXEwjY0^Ko zYFi1|+GjW>znyhqhBH{|@hxvt6)Xza-gs?oEnh3(ty1d3$QF-SN?H`~RxoPiFSjTS zc8UpWed!~1&7GK_yxmfzKi4uW!^0&? zUR?O|&av{HY$t}z_2F^#49od!oHF8-S(>B7jj#djDA!y^37@bJ%&jt%4Sa2m-8#wm zp9xtPX~F2F<8}JEDS;Tlh=okDxvbOFnzj73HE%<{AH%ol!&^h6G2T?H_lwjww8$&%S@w6dvX7U zM%Mv~KerI;Z^-M>bHyXd{U=cdZdR54lO=+vCfdt)ugM#LM_XU0y3TQ!9SOttq=0oh zn`DtXJW30|(hsZFO_`QarOmUVoE-%dsiW^*lw0708A62Efsyvvh2+mkRGk`c^>yVW zs)Li>>+EOE@;&HP@K?p?PTjK!EL|$GywUN2W?(bq zbvHfraEYNCCclhv6esCBFx)=8oa*bGDu>MFjdgS=Sld#A5(omc^X<*QlU`@_7A z0kzwnqp^+w*ZO@>O$XvewnI7~FKr*QIt~&>#f|b*eDCyr>~EGp5x!H+kk@xvxu)n6 zD!abh93f~JLBi->zVOkS`Sm0bV`7#;Wq-&h#3HBLdp3YKAGuIXG=Ug*i%sC~8~i%( zrosw^5ZnuF)MAmCv`{L$LH05zb_=FRDxOM2v}34wW4tw+#NeV9p}`_%{?>hsUnMOkIc=6LVG+ zBGzJrpcecahC&FMj}n5O9%2B|{=yT42=^w@C?QiU{qXU+*OE3^+L&!z`R>r9&H29q zY9CHPkeB_TK*XP}z~{D_jPd#Y)-+4oxpK!XTH#CTDBx3dS|*4(3fv}ouCD{!C)`LF zzV!M!&D}&IMmJ;uEj~+VuM%(NPo|D?DC|tL4iX|%eZS2sch;UQapGCY?6aM=s6EwW zkq|Xg>yjY!ILaLy@J$-7!_p6bRZ<>(wh2(QKci`ns18W845|YXNT!a~H}duaU?dUa z9V1J`nB2IIYU1m6J8m6KO;w+pf!C@#uq6;#cY>tG{uGaKFmyG{>>t`yXbuWR#|`+M z8brmLD=8H;aWaf=Uf#dO`Q+}H>NPzQ{HXe%u~SY{hEtCp{-ll2u;@IGr!wKw%_wqU zdbh&8sUmL1HGJL4H+L$MRQovRxNflS(8j*)-`Wg|o$9>XDVG$_W}9Z%M5pE!Hl z8^I8;o_Q{aW8dP;v-GLZ)Y=fgJplF=wjvV>t0O|n!% z6j5KyP^d{p)-uMDEkkNhDQc#aZ9=j|ixILVV{Mq}anCb-w|npJp8MyV=bSm``RDnZ z_vih7y&fML+6~#TpNv*75@E}c#{vn4O!!M~^^Oia5@HeTH4ed@ywHSy4}vF0+Wf*N zvdlM|^Ym=D3kC(b+wP9oRa-J$Ld4N!`A!NfBAy|^wnLTN?nOa?V)P}-=DBxxWcr6~ zOsUsOLfi%4YEibGPd{0A2UF@mwh%WVT@ZpB?->;0Ce%~#$O3mcR%@>g43NP@ja(OH z!~7&{6|cI5z6ENj6M<4hmSG24!u!l59JByhghQlgN`Nv3>jgrDn5p#gEv)SwgRL zV1Tt-CY~lTj{-$A`Jvy!AscPm(MC4Az#v8sIid-4hzfpn5^|nGa4F&(Jyx zA0QjofgZ7o8%Fu{$&d|9-(Dts_N@v_OElRp`YAtm8)fcglqkEK4}8ag&8<4(Y&%dkBg%$F+=l@&V6Ejw zr~Q}^2(k;&Mp z4j=U>=oVUPs}L9F&A1KT-k}OyBP7^T;rWxgJ0KgbH%C7hO0ZKXsZL2`7yuKRJo<7$ zJM}ERFy?TEL!BTuCAlw+q#6tbAT`S)Nyp_`S{5`!nhp~_rM#4sY|V0uQ%ifJai-Zc2gt zlTnyQX(aem&w%N2f+4|{!rv^BvXx$pV#~s_a_w~T{)Kk)|l-^(%V0Hq;sCb`Z69`VN;s_mXUoFL!Ls&Zp$o^+I zN=EH%4?i~|i-1SUx@~1?VenfI!dXJK9WcPmz8Q~{99CnxJmh1fXfUOc`h>W};61wX zU(reHINsD|6RS00^M8m=IT0KQakiYCzmi=Aetk85ssBgGcg<}@&eda+f`&|kQ^qs2 z?v3U8_0KL?A1a5aUi0K{HVWD27lNGJ%+ew(sHcgtqpgQ5WWuD+$+4t_*NkBRzZPQ< z-YpC2lnhmMPe1{f^Q~Qv-#S0$)DDIAq8~eTpnLj;D0));6bi)@+1L2(Zu=YzNO%x% zpi+xf2?Y}FMPF8yUE}A9n7jdzGk((Oi3teKvT8TerBr->vP`8u(*<1r>|nY~i7yCA z#rZ)tE|OQOrV6ca?-iZi%LlsOf8WHqd3A()t2cpFGy1u>VOaH5;8zBGkXW;t!dU67 zaKxW%d3__2yu4aCn70WnQy3`(#wHrgKlYu|h5w)#Tz!gF1vP`8erpDQE`giW5HtaDWYf zGxdZt`*}11=e;rMv4B0UANgkX%hL|T7qa8%M}s{hzJ;NyoIi4Nf91~OmcA@CuP`R3 z{}=&g_GrzNJ9svEH(10O(||ob`(6w_;oJZ=E!P1`-C-L6A3vKRt&TM5Y>t?#>G_!w zAJ1-PG@Gofu~ycG_MGvxT$>&|Rk5)&SNX&KTgC0`)OSM zjYV8@Bzf;!9jEA*PxY4;RN_yqG3Y&|=NkzRxJfLtZ;9sP1`r~(Y2imZc6e#~w3WYa zs><<2rIWT~J@;_~$o(z2ZKk>Ot5U+bH2&SmY9|I>I8H#IUTcnc{{y;C;|4Y+T|v#| zi@C1Zx9p;GXyRMUn{6(~qULpQ>Z+Md^VN&F#p(y=&Kcfn!$R{*B=&3K)GcU;(nEmk zls7IS4KaQwCVrBf#7$D$dQ;tPjbZJ%Bi@wXuOYjEYs#0~frToMQI3h1=hCE0O3dxF zaq8A~&k44G?36LhraGG@T{OaUZNpycoPS3dH#jhP+Nh@u3-$eRuUw`BEBto8D0spE z;+pHXkU^9Fh?+1+NTVJj#FSX5PH^efDon9K-zO-vH2k4uQAIs>2ugR5C&Jg~Xyh{B@MmeM4yfZbVj z+{>Psaf5K@Y_5QFRW?nU^*No!JRxB_@O8q#+IicWSUSc4x4&6u!r*FS_!_tbnK0OO z#5s$`+&QtRw57xrNG$Pa!wN5kwPEoOgV)5;(fYWT`!{!B!`}%mDydcb0H}0fa#|8s zfKV0t9d_5F0~=m#x~R0}fg6wrGfAgOBdj5%*)*no=(vGmlVdhbJhV!Ab`HOon@}0_ zy zX4CjZTaxiR*BE?#rDG5JxZ?(p1nQ1($u+T;omx0`{KAT?@|t5xW=&mAv9ZI?OYyhr z8`1A~e?(K~f&Se}Lo==eDY135(@WL0xis^#I!aQ`o8cPq$~W_X3)ADostq`|~JPzMZaoy$V%>N0*Z8zIjk%Vz-JL z@@I-2tMQ;qOtolbjMt3kei!XN!jO(fiorm-y1Je1+&EG^{Yf?4_1-M)eD(1Kp-fO(*Mv3h9Dp${?byl^8%DA(A~ZU7q74bNA+(4;WL0A?`QxvJe8_CXVPK$fH_qxmR!6 z%kQDD`A*ynaJN1wH6$;MMUY-aQxBOcM93WL(hYRC_IMg1Dttzb1a<=til;i1_hvfep%hy)6o!PD@&{nQtWNlB$b$aO+RjaL5gEBQyn%F%AwgUm~ zw`7ZK6&U6CRM8>%bT@rcA}To0-5zs4gn)6xpYYHpK~cl)<@k4(hva2F5u{frN}xLi zbeu_-Ih!gRg@}5{4aui_X_26bGlA~bn6eO2`(O2cW_J5<8i^x)w) z?6px%$3wU#_Dt^sT)eTqBcU2WyDi6FfPJ3DF!fv;)+buaR2wG@s zdhoz=$IkSr`*(}4PY$20laoj5lL!yRujeXMV@yq8Rj#Hmm9tYx z3hRjh?!F%vBgxBz;fn6U`bz^c<)A`i(f7R5O6S`FbVTEwBgcaDgMO1I3}gAK*{=ac zHhPX;y4VVIGicAalCAkLJv;DxW42pPaf~Nx@2ie>Sne?i!RhD8r5NE-`KwC?- zW_c*%pEHyUqlW=zs&wgyj3NgE zy%Sb>C54#Oz}>yq4erRdB>w@FW;FQxLo?=p`;s(gwItAV(X|f*9JvSq-c&0t#~}^@ zsa_y0*Mo}^lB>K`4t-$#re`)Wu!te&VzH9#-W9W6`6knKjU0BFw?WYqg*%?O5XpB2 zTulBnD*k)RSe7a57l$xAsRS1rEg4tWfQuPH@6PQ2$-3cI)IGf6i&w3QSL#7LOAq83 z$wYy!%0;TQp4#fpH}(TPD>=~*?lg$oVP=P)ON%n%t}- z`!Y}u@j}^u(^?eL3{RS&kgB9=4Hq-IiBw%)yw2tNue3IoEQ^7Alqf3|fAGo3>K`j3 z#F2e# z%2n5lK&WwgPoGMiu+O-|^Uvox&;{}&^sOh2{TF-V&}dF+*JF;&=a!<%VxlJNQJFSx zkMelmx4(%k!+9RD1v^XHZsu7@OtRl}NOhYVnV1y5X5$2M-#*`&Ij5sR3(lu>iG zq0MNDGz-)3I!k!wjv492Fwu?y<-)rABP2gmuom{|B!SeFMG0-FzD6I z@LRiGH(%?~l-_T`gfl4%yjWQ5U2v18zej@Y-oKDJ_`6u^Cd9#j*ZvvJw@sMH-K8gD z-yPZ|lZW5n^Rf^;56Ul&ou(t-nJ4>##oDop^g=ZyWtLQcIJ!3fu>1*)nh60e?Dp94 zPlBL9_q$~vy^!yh;C-HLqXezOlEOdxb?P}3cwK&Z)4UQ(%K{mF1&oAsPe+cL z!6RihAB@P6Wx>7Cn)tF-PY7-c)sbGvl3`@Lf#9~3IMCsrz)Kde{1H-(zk)~hKyzY_ z@pB{UpW%`D?h5}7g{xty2w!@k86+8Ag-51i;4Hgl9T;Hd<3z{p=(rBqxU*ABkb7M= zotz=WjY!SIBh$c}5EKOlSYx8-g&2sa&Otg{=!`O}b-xNfS45HN=*G`2PG83(c_T!d zN^$lWCKL+17GpVU!vHf^FpGMsuv%{o^K)V6D3|v&^x<)*kLe0u;RtYr%JUYkeb6le zn&IzTe_C;h1Ni%{qqXb}CD{zn0yDeu$6`;f&Qb!#{+$B4J$3dM|n*UqCX1gU6-x&gZoW^Ud$+)OWlQ z8MX7`8s)?BEuNn@Nd4s>o{*ML7C!lSKaP%Mtu%V7@Z1@%?K@r67Dd={I@@-gOQHy~ z$NVNB8wn;8{M`GPG6>G>zXiiR5UIGMkPQSj_gBz4mqO&3rE2XPyLG+J9B13UItR2( zTgMEtez^LCN{t#XOb}=7yg1Dh>)(1ct&RM@9(~%?JL9>Fal7HWB-f=-5{GML7m>bo zCF7z-^7qj!I82A1ohw3vDSmSoXqG|0Y8xAH8%j7t{`^Fh0_Si~%$qREx|Pob9JhHT zd~T_T<3K|#Qf6-elar_5pY&H5pZvV4r=DEgd*uH~hL4}dwd7M2h;!=PhmG53exTMz z>gNQzkKq>h9W$F++!({p8=u84Of8)1Uj6m%*Uw)Yzkc?d+cVhMJOpZ?3%XYxb)%d39pg z|0@^f4F)d};|HfWHpt(d+IsB&>eQl3Lv?MReP;WauKy#XM`_RVcR{9JSfHOiecE2x zbtBf-WNZ(>+d2F3lmFT|!;UlTtU5+UGZ`GZ8x}47LW&b!g?QDQR!mKRQ}qN5mt&{y$gt%`0N|2ROzwv zaR~kfSGZV@-e9XN-_|)f-j=y?FDRP=t%p`Q)?S>gUX{coI=6J|Z%lGq>qpmn|#41UBUS4e_g5>Y_t47k?xW@D}?rbqRyVVLQif?e;*qY zo+SzwlXaW(hrFW1Arefy_Sr`Lq%zec;=u9@JU@T|Edm;SyU9QxeDy*XAVQoUkU zRYpuAI1jyXANMb|7;};EjYAwT0>+^e{+sZ@E#87W4B%(`KWHx*6LstQf3|o>tAfw8A7kTsrgcR)O!e||63kzIm`qh^?Qbziwjig z;OjPjfdB>Q8Gm)Rj|+2uE2aF>cuFZZa&G@K``-O}>5nfRjk*R_^WVv`wUKin0}eEy z>Djr+UA0#;0$X${J!{4L-DGE*sCmVifjd1t6=)dC7m{v@iI*6~EA_SF{Z;2tQM!5z p*GmoROi$D0+QSAv41(0lMN&|qLDZevu60~w+`HyX_8A$z{{qQw#ex6; literal 0 HcmV?d00001 diff --git a/server/-product/production/OMAR/log/2022-05/mediacube-err-05-09-2022-1.log.gz b/server/-product/production/OMAR/log/2022-05/mediacube-err-05-09-2022-1.log.gz new file mode 100644 index 0000000000000000000000000000000000000000..605ced81cff3f786c180ed9d60ccd1a25998e48b GIT binary patch literal 308 zcmV-40n7d$iwFP!0000008Nn1Zo)7Sgzrea!`>RCVA&ypCK3ktGC5o1*ulW0j;T)}4W zZ+j&L9{@mOv;i$dZw%H@_eQD{=nl=OTkt*Z13~t`HgxU?20Smj~aqGn|b&dt>q^i zC|`84^`rX$0$HB#^X-0@t>wK G0RRBUtdZXU literal 0 HcmV?d00001 diff --git a/server/-product/production/OMAR/mediacube.bat b/server/-product/production/OMAR/mediacube.bat new file mode 100644 index 00000000..24a39d6c --- /dev/null +++ b/server/-product/production/OMAR/mediacube.bat @@ -0,0 +1,17 @@ +@echo off +if exist tmp rmdir tmp /s /q +mkdir tmp +java ^ +-Dorg.eclipse.epp.logging.aeri.skipReports=true ^ +-Declipse.ignoreApp=true ^ +-Dosgi.noShutdown=true ^ +-Dlog4j.configurationFile=settings/log4j2.xml ^ +-Djetty.home=settings ^ +-Djetty.etc.config.urls=jetty.xml ^ +-Djava.io.tmpdir=tmp ^ +-Dfile.encoding=UTF-8 ^ +-Dgosh.home=configuration ^ +-jar ../../target/products/MediaCube/linux/gtk/x86_64/plugins/org.eclipse.equinox.launcher_1.3.201.v20161025-1711.jar ^ +-Xms512m ^ +-Xmx1024m ^ +-console diff --git a/server/-product/production/OMAR/settings/application.yaml b/server/-product/production/OMAR/settings/application.yaml new file mode 100644 index 00000000..c870af39 --- /dev/null +++ b/server/-product/production/OMAR/settings/application.yaml @@ -0,0 +1,45 @@ +datasource: + mediacube: + url: jdbc:db2://localhost:50000/mc + user: db2admin + password: password + external-indexer: true + simple-search: true + login-timeout: 3 + pool-size: 10 + mediacube-nosql: + url: jdbc:db2://localhost:50000/mc + user: db2admin + password: password + schema: test + login-timeout: 3 + hsm: + url: jdbc:db2://10.11.1.89:51500/tsmdb1 + user: tsminst1 + password: tsminst1 + planair: + url: jdbc:sqlserver://10.11.254.86;databaseName=PA_Vivantis; + user: MAM + password: VDani +services: + ffmpeg: + executable-location: /opt/ffmpeg/ffmpeg + mediacube: + proxy-root: /data/mediacube + nexio: +# host: 10.10.1.55 +# collection-name: nexioclips +# use-mos-gateway: true + disabled: true +jobs: + validate-transfers: false + copy-buffer-size: 32768 + scheduled-execution-disabled: true +tsm: + randomize-archives: false + delimiter: / + node-name: JOBENGINE + fs-name: /JOBENGINE + alternate-fs-name: /JOBENGINE + hl-name: /JOBENGINE + \ No newline at end of file diff --git a/server/-product/production/OMAR/settings/dsm.opt b/server/-product/production/OMAR/settings/dsm.opt new file mode 100644 index 00000000..90fcbdd2 --- /dev/null +++ b/server/-product/production/OMAR/settings/dsm.opt @@ -0,0 +1 @@ +SErvername tsm.in.useribm.hu diff --git a/server/-product/production/OMAR/settings/dsmopt.lock b/server/-product/production/OMAR/settings/dsmopt.lock new file mode 100644 index 00000000..e69de29b diff --git a/server/-product/production/OMAR/settings/external-commands.yaml b/server/-product/production/OMAR/settings/external-commands.yaml new file mode 100644 index 00000000..5f3a3a63 --- /dev/null +++ b/server/-product/production/OMAR/settings/external-commands.yaml @@ -0,0 +1,27 @@ +profiles: +- + name: proxy + executable: /Programs/ffmpeg/bin/ffmpeg.exe + arguments: + - -y + - -v + - error + - -progress + - pipe:1 + - -i + - "%i" + - "%o" +- + name: length + executable: /Programs/ffmpeg/bin/ffprobe.exe + arguments: + - -v + - error + - -select_streams + - v:0 + - -show_entries + - stream=nb_frames + - -of + - default=noprint_wrappers=1:nokey=1 + - "%i" + \ No newline at end of file diff --git a/server/-product/production/OMAR/settings/jetty.xml b/server/-product/production/OMAR/settings/jetty.xml new file mode 100644 index 00000000..30bdeb08 --- /dev/null +++ b/server/-product/production/OMAR/settings/jetty.xml @@ -0,0 +1,45 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/server/-product/production/OMAR/settings/log4j2.xml b/server/-product/production/OMAR/settings/log4j2.xml new file mode 100644 index 00000000..5f514527 --- /dev/null +++ b/server/-product/production/OMAR/settings/log4j2.xml @@ -0,0 +1,105 @@ + + + + + + log + ${logPath}/mediacube.log + ${logPath}/movtest.log + ${logPath}/$${date:yyyy-MM}/movtest-%d{MM-dd-yyyy}-%i.log.gz + ${logPath}/delete-materials.log + ${logPath}/$${date:yyyy-MM}/delete-materials-%d{MM-dd-yyyy}-%i.log.gz + ${logPath}/$${date:yyyy-MM}/mediacube-%d{MM-dd-yyyy}-%i.log.gz + ${logPath}/markered-mediacube.log + ${logPath}/$${date:yyyy-MM}/markered-mediacube-%d{MM-dd-yyyy}-%i.log.gz + ${logPath}/mediacube-err.log + ${logPath}/$${date:yyyy-MM}/mediacube-err-%d{MM-dd-yyyy}-%i.log.gz + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/server/-product/production/OMAR/settings/maestro.yaml b/server/-product/production/OMAR/settings/maestro.yaml new file mode 100644 index 00000000..b8091a2f --- /dev/null +++ b/server/-product/production/OMAR/settings/maestro.yaml @@ -0,0 +1,38 @@ +sourceStoreUri: + name: Default + protocol: LOCAL + uri: "/mnt/NLE" + fileFilter: "*.mxf" + showDirectories: true +alternateSourceStoreUris: +- name: NLE1 + protocol: LOCAL + uri: "/mnt/NLE/NLE1" + fileFilter: "*.mxf" + showDirectories: true +- name: NLE2 + protocol: LOCAL + uri: "/mnt/NLE/NLE2" + fileFilter: "*.mxf" + showDirectories: true +- name: NLE3 + protocol: LOCAL + uri: "/mnt/NLE/NLE3" + fileFilter: "*.mxf" + showDirectories: true +- name: NLE4 + protocol: LOCAL + uri: "/mnt/NLE/NLE4" + fileFilter: "*.mxf" + showDirectories: true +- name: POLC + protocol: LOCAL + uri: "/mnt/POLC" + fileFilter: "*.mxf" + showDirectories: true +targets: +- name: FINISHED_SHOWS + killDateDays: 7 + storeUri: + protocol: LOCAL + uri: "/mnt/PROMISE/FINISHED_SHOWS" diff --git a/server/-product/production/OMAR/settings/mediacube.yaml b/server/-product/production/OMAR/settings/mediacube.yaml new file mode 100644 index 00000000..347ca896 --- /dev/null +++ b/server/-product/production/OMAR/settings/mediacube.yaml @@ -0,0 +1,53 @@ +jobQueuePollInterval: 1000 +disableHelp: true +maestroDisabled: false +alternateRetrieveSelector: false +disableStatistics: true +disableEditor: false +topTypeFilters: +- name: Hír bejátszó +- name: Hír nyers +- name: Visszarögzített +- name: Egyéb +bottomTypeFilters: +- name: Műsor +- name: Műsor nyers +- name: Promo +- name: Promo nyers +- name: Reklám +- name: Reklám nyers +authentication: + authEnabled: true + defaultUser: root + defaultPassword: password + adHost: intra.mediavivantis.hu + adNonSecurePort: 3268 + adBaseDn: DC=intra,DC=mediavivantis,DC=hu + adAdminMap: + - G_MV_U_MUSZAK + - G_MV_U_INGEST + adSubmitterMap: + - G_ECH_U_INFORMATIKUSOK + - G_ECH_U_MUSZAKVEZETOK + - ECH-ISILON-ADMINS + adEditorMap: + - G_ECH_U_INFORMATIKUSOK + - G_ECH_U_MUSZAKVEZETOK + - ECH-ISILON-ADMINS + localAccounts: + - user: user + password: 5F4DCC3B5AA765D61D8327DEB882CF99 + email: + - user: lebony + password: 4E25B117B14D86D7DCECB4E433CF932C + email: + - user: root + password: 5F4DCC3B5AA765D61D8327DEB882CF99 + email: vasary@elgekko.net + localAdmins: + - root + - admin + localSubmitters: + - lebony + localEditors: + - editor -- 2.54.0