From: elgekko Date: Mon, 7 Nov 2022 07:53:45 +0000 (+0100) Subject: Server changes X-Git-Url: http://git.useribm.hu/?a=commitdiff_plain;h=7a5a1865d60250e4359c2c742ab962ae9880fb36;p=mediacube.git Server changes --- diff --git a/server/-product/production/LOCAL/settings/mediacube.yaml b/server/-product/production/LOCAL/settings/mediacube.yaml index 347ca896..b86d5b7b 100644 --- a/server/-product/production/LOCAL/settings/mediacube.yaml +++ b/server/-product/production/LOCAL/settings/mediacube.yaml @@ -1,7 +1,12 @@ jobQueuePollInterval: 1000 disableHelp: true maestroDisabled: false -alternateRetrieveSelector: false +alternateRetrieveSelector: true +targetRestoreFilters: +- name: Alternate Low-res +- name: Alternate Low-res1 + allowedGroups: + - Pooler disableStatistics: true disableEditor: false topTypeFilters: @@ -18,8 +23,8 @@ bottomTypeFilters: - name: Reklám nyers authentication: authEnabled: true - defaultUser: root - defaultPassword: password +# defaultUser: root +# defaultPassword: password adHost: intra.mediavivantis.hu adNonSecurePort: 3268 adBaseDn: DC=intra,DC=mediavivantis,DC=hu @@ -44,6 +49,12 @@ authentication: - user: root password: 5F4DCC3B5AA765D61D8327DEB882CF99 email: vasary@elgekko.net + - user: library + password: 5F4DCC3B5AA765D61D8327DEB882CF99 + email: library@net + - user: engineer + password: 5F4DCC3B5AA765D61D8327DEB882CF99 + email: engineer@net localAdmins: - root - admin @@ -51,3 +62,6 @@ authentication: - lebony localEditors: - editor + localPoolers: + - library + - engineer diff --git a/server/-product/production/OMAR/configuration/etc/gosh_profile b/server/-product/production/OMAR/configuration/etc/gosh_profile deleted file mode 100644 index 1c659f49..00000000 --- a/server/-product/production/OMAR/configuration/etc/gosh_profile +++ /dev/null @@ -1 +0,0 @@ -prompt=mc> \ No newline at end of file diff --git a/server/-product/production/OMAR/jobs/executors.xml b/server/-product/production/OMAR/jobs/executors.xml deleted file mode 100644 index 0fd7a2f2..00000000 --- a/server/-product/production/OMAR/jobs/executors.xml +++ /dev/null @@ -1,43 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - \ No newline at end of file diff --git a/server/-product/production/OMAR/jobs/schedules.json b/server/-product/production/OMAR/jobs/schedules.json deleted file mode 100644 index a42bbeb4..00000000 --- a/server/-product/production/OMAR/jobs/schedules.json +++ /dev/null @@ -1,181 +0,0 @@ -{ -"joblist":[ - - { - "name":"n1", - "template":"cancelable.xml", - "active":true, - "executeimmediate":true, - "parameters":[{"name":"param","value":1000,"type":"java.lang.Integer"}] - }, - - { - "name":"n2", - "template":"cancelable.xml", - "active":true, - "executeimmediate":true, - "parameters":[{"name":"param","value":1000,"type":"java.lang.Integer"}] - }, - - { - "name":"Limitált archiválás", - "template":"archive-limited.xml", - "cronexpression":"0 */10 * * * ?", - "parameters":[ - - { - "name":"sourcePath", - "value":"/opt/mediacube/ARCHIVE", - "type":"java.lang.String" - }, - - { - "name":"globalSourcePath", - "value":"\\\\10.11.1.90\\data\\ARCHIVE", - "type":"java.lang.String" - }, - - { - "name":"transcoderTargetPath", - "value":"/mnt/PROMISE/TRANSCODER/FFASTRANSCODER/Out", - "type":"java.lang.String" - }, - - { - "name":"killDateDays", - "value":-1, - "type":"java.lang.Integer" - }, - - { - "name":"limit", - "value":1000, - "type":"java.lang.Integer" - } - ] - }, - - { - "name":"SYS: batch-retrieve-ondemand", - "template":"batch-retrieve-ondemand.xml" - }, - - { - "template":"calculatemd5.xml", - "name":"MD5 kiszámítása", - "parameters":[ - - { - "name":"fileName", - "value":"C:\\Users\\machine\\Downloads\\sample.mxf", - "type":"java.lang.String" - } - ] - }, - - { - "template":"dummy-test-job.xml", - "name":"dummy test job", - "parameters":[{"name":"param1","value":"Jozsi","type":"java.lang.String"}], - "cronexpression":"0/3 * * * * ?" - }, - - { - "template":"move-jpeg-to-isilon.xml", - "name":"JPEG másolása ISILON-ba", - "parameters":[ - - { - "name":"sourceUri", - "value":"localhost", - "type":"java.lang.String" - }, - - { - "name":"sourceProtocol", - "value":"LOCAL", - "type":"java.lang.String" - }, - - { - "name":"sourceFolder", - "value":"c:\\data\\video", - "type":"java.lang.String" - }, - - { - "name":"targetUri", - "value":"localhost", - "type":"java.lang.String" - }, - - { - "name":"targetProtocol", - "value":"LOCAL", - "type":"java.lang.String" - }, - - { - "name":"targetFolder", - "value":"c:\\data\\video2", - "type":"java.lang.String" - }, - - { - "name":"userName", - "value":"dani", - "type":"java.lang.String" - }, - - { - "name":"password", - "value":"dani", - "type":"java.lang.String" - }, - - { - "name":"port", - "value":"21", - "type":"java.lang.Integer" - } - ] - }, - - { - "template":"prores-archive.xml", - "name":"ProRes archiválás", - "parameters":[ - - { - "name":"sourceStoreFolder", - "value":"c:\\data", - "type":"java.lang.String" - }, - - { - "name":"sourceStoreName", - "value":"AMC_LOCAL", - "type":"java.lang.String" - }, - - { - "name":"filter", - "value":{"fileName":".*\\.(mov)$"}, - "type":"com.ibm.nosql.json.api.BasicDBObject" - }, - - { - "name":"targetStoreName", - "value":"AMC_LOCAL", - "type":"java.lang.String" - }, - - { - "name":"escortStoreFolder", - "value":"AMC_LOCAL", - "type":"java.lang.String" - } - ] - } - ] -} \ No newline at end of file diff --git a/server/-product/production/OMAR/jobs/steps/AnalyzeMediaFilesStep.java b/server/-product/production/OMAR/jobs/steps/AnalyzeMediaFilesStep.java deleted file mode 100644 index 98cf7fa9..00000000 --- a/server/-product/production/OMAR/jobs/steps/AnalyzeMediaFilesStep.java +++ /dev/null @@ -1,28 +0,0 @@ -package user.jobengine.server.steps; - -import java.nio.file.Paths; -import java.util.HashMap; -import java.util.List; -import java.util.Map; - -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; - -import user.commons.mediaarea.MediaArea; - -public class AnalyzeMediaFilesStep extends JobStep { - private static final Logger logger = LogManager.getLogger(); - - @StepEntry - public Object[] execute(List foundFiles) { - Map proResFiles = new HashMap(foundFiles.size()); - - for (int i = 0; i < foundFiles.size(); i++) { - MediaArea mediaArea = new MediaArea(Paths.get(foundFiles.get(i))); - mediaArea.process(); - proResFiles.put(Paths.get(foundFiles.get(i)).toAbsolutePath().toString(), mediaArea); - } - - return new Object[] { proResFiles }; - } -} diff --git a/server/-product/production/OMAR/jobs/steps/ArchiveListBuilderStep.java b/server/-product/production/OMAR/jobs/steps/ArchiveListBuilderStep.java deleted file mode 100644 index b2321829..00000000 --- a/server/-product/production/OMAR/jobs/steps/ArchiveListBuilderStep.java +++ /dev/null @@ -1,167 +0,0 @@ -package user.jobengine.server.steps; - -import java.io.File; -import java.io.IOException; -import java.nio.file.DirectoryStream; -import java.nio.file.Files; -import java.nio.file.Path; -import java.nio.file.Paths; -import java.util.LinkedList; -import java.util.List; - -import org.apache.commons.lang.StringUtils; -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; -import org.apache.logging.log4j.Marker; - -import user.jobengine.server.IJobEngine; -import user.jobengine.server.IJobRuntime; - -/** - * Az archivalhato mediak listazasa MediaFileWrapper objektumokban. A listazott media allomanyokat megjeloli .catched signal allomannyal, hogy a legkozelebbi - * listazas figyelmen kivul hagyja. - * - * @author robi - */ -public class ArchiveListBuilderStep extends JobStep { - private static final Logger logger = LogManager.getLogger(); - // private static final String UTF8 = "utf-8"; - private static final String STATUSFOLDER = ".STATUS"; - private static final String JSONEXT = ".json"; - private static final String CATCHEDEXT = ".catched"; - - public static final String ITEM_TITLE = "itemTitle"; - public static final String ITEM_HOUSEID = "itemHouseId"; - public static final String ITEM_DESCRIPTION = "itemDescription"; - public static final String MEDIA_HOUSEID = "mediaHouseId"; - public static final String MEDIA_TITLE = "mediaTitle"; - public static final String MEDIA_DESCRIPTION = "mediaDescription"; - public static final String MEDIA_TYPE = "mediaType"; - private static final String DURATION = "duration"; - private static final String EXISTING_MEDIAID = "existingMediaId"; - private static final String TAGS = "tags"; - - private Marker marker; - - private ArchiveItem createArchiveItem(Path jsonFilePath, Path mediaFilePath, Path catchedFilePath) { - ArchiveItem result = null; - try { - result = ArchiveItem.fromFile(jsonFilePath); - result.setMediaFile(mediaFilePath.toString()); - result.setCatchedFile(catchedFilePath.toString()); - } catch (Exception e) { - logger.catching(e); - } - - return result; - } - - private void createCatchedFile(Path catchedFilePath) { - try { - Files.createFile(catchedFilePath); - //Files.write(catchedFilePath, CATCHED.getBytes(UTF8), StandardOpenOption.CREATE); - } catch (Exception e) { - logger.catching(e); - } - } - - @StepEntry - public Object[] execute(String sourcePath, int limit, IJobEngine jobEngine, IJobRuntime jobRuntime) { - marker = jobRuntime.getSessionMarker(); - List archiveList = new LinkedList(); - DirectoryStream directoryStream = null; - try { - DirectoryStream stream = Files.newDirectoryStream(Paths.get(sourcePath)); - for (Path p : stream) { - processPathItem(p, archiveList); - } - } catch (Exception e) { - logger.catching(e); - logger.error(marker, "Az '{}' mappa elérése sikertelen. A rendszer hibaüzenete: {}", e.getMessage()); - } finally { - if (directoryStream != null) { - try { - directoryStream.close(); - } catch (IOException e) { - } - } - } - - if (limit > 0 && archiveList.size() > limit) { - archiveList = archiveList.subList(0, limit); - logger.info(marker, "A folyamat alkalmazza a beállított {} limitet.", limit); - } - - if (archiveList.size() == 0) - logger.info(marker, "Nincs archiválandó anyag."); - else - logger.info(marker, "Az archiváló folyamat {} új anyagot érzékelt.", archiveList == null ? 0 : archiveList.size()); - - for (ArchiveItem archiveItem : archiveList) { - createCatchedFile(Paths.get(archiveItem.getCatchedFile())); - } - - return new Object[] { archiveList }; - } - - private boolean processPathItem(Path mediaFilePath, final List archiveList) { - File mediaFile = mediaFilePath.toFile(); - - // if (mediaFile.length() > 0) - // return false; - - if (mediaFile.isDirectory()) { - return false; - } - - Path dotStorePath = Paths.get(mediaFilePath.getParent().toString(), STATUSFOLDER); - Path catchedFilePath = Paths.get(dotStorePath.toString(), mediaFile.getName() + CATCHEDEXT); - File catchedFile = catchedFilePath.toFile(); - if (catchedFile.exists()) { - logger.warn("{} file is already catched.", mediaFile.getName()); - return false; - } - - Path jsonFilePath = Paths.get(dotStorePath.toString(), mediaFile.getName() + JSONEXT); - File jsonFile = jsonFilePath.toFile(); - if (!jsonFile.exists()) { - logger.warn("{} has no json metadata.", mediaFile.getName()); - return false; - } - - ArchiveItem archiveItem = createArchiveItem(jsonFilePath, mediaFilePath, catchedFilePath); - - if (archiveItem == null) { - logger.warn("{} has no metadata specified.", mediaFile.getName()); - return false; - } - - if (StringUtils.isBlank(archiveItem.getItemHouseId())) { - logger.warn("{} has no Item HouseID specified in metadata.", mediaFile.getName()); - return false; - } - - if (StringUtils.isBlank(archiveItem.getItemTitle())) { - logger.warn("{} has no Item Title specified in metadata.", mediaFile.getName()); - return false; - } - - if (StringUtils.isBlank(archiveItem.getMediaHouseId())) { - logger.warn("{} has no Media HouseID specified in metadata.", mediaFile.getName()); - return false; - } - - if (StringUtils.isBlank(archiveItem.getMediaTitle())) { - logger.warn("{} has no Media Title specified in metadata.", mediaFile.getName()); - return false; - } - - //A tenyleges archivalast vesszuk elore - if (mediaFile.length() == 0) - archiveList.add(archiveItem); - else - archiveList.add(0, archiveItem); - //createCatchedFile(catchedFilePath); - return true; - } -} diff --git a/server/-product/production/OMAR/jobs/steps/ArchiveMaterialSubmitStep.java b/server/-product/production/OMAR/jobs/steps/ArchiveMaterialSubmitStep.java deleted file mode 100644 index 0aedbbe8..00000000 --- a/server/-product/production/OMAR/jobs/steps/ArchiveMaterialSubmitStep.java +++ /dev/null @@ -1,52 +0,0 @@ -package user.jobengine.server.steps; - -import java.io.File; -import java.util.List; - -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; -import org.apache.logging.log4j.Marker; - -import user.commons.ListUtils; -import user.jobengine.server.IJobEngine; -import user.jobengine.server.IJobRuntime; - -public class ArchiveMaterialSubmitStep extends JobStep { - private static final Logger logger = LogManager.getLogger(); - private static final String JOBTEMPLATE = "archive-material.xml"; - private static final String KILL_DATE_DAYS = "killDateDays"; - private static final String ARCHIVE = "Archiválás"; - private static final String ARCHIVE_ITEM = "archiveItem"; - private Marker marker; - - @StepEntry - public Object[] execute(List archiveList, int killDateDays, IJobEngine jobEngine, IJobRuntime jobRuntime) throws Exception { - marker = jobRuntime.getSessionMarker(); - if (archiveList == null || archiveList.size() == 0) - return null; - - if (jobRuntime.forkPrepare()) { - for (int i = 0; i < archiveList.size(); i++) { - ArchiveItem archiveItem = archiveList.get(i); - try { - IJobRuntime runtime = jobEngine.submit(jobRuntime, null, JOBTEMPLATE, ARCHIVE, - ListUtils.asMap(ARCHIVE_ITEM, archiveItem, KILL_DATE_DAYS, killDateDays)); - int progress = (i + 1) * 100 / archiveList.size(); - setProgress(progress); - //TODO kivezetni a submit hibaüzenetet - if (runtime == null) - throw new Exception("Submit returned null runtime"); - } catch (Exception e) { - logger.catching(e); - String fileName = new File(archiveItem.getMediaFile()).getName(); - logger.error(marker, "A(z) '{}' állomány archiválási kísérlete sikertelen. A rendszer üzenete: {}", fileName, e.getMessage()); - if (!archiveItem.removeCatchedFile()) - logger.error(marker, "A(z) '{}' állomány .catched jelző állománya nem törölhető.", fileName); - throw e; - } - } - } - jobRuntime.forkWaitComplete(); - return null; - } -} diff --git a/server/-product/production/OMAR/jobs/steps/ArchiveRecursive.java b/server/-product/production/OMAR/jobs/steps/ArchiveRecursive.java deleted file mode 100644 index 3f01f32b..00000000 --- a/server/-product/production/OMAR/jobs/steps/ArchiveRecursive.java +++ /dev/null @@ -1,324 +0,0 @@ -package user.jobengine.server.steps; - -import java.io.File; -import java.io.IOException; -import java.nio.file.FileVisitResult; -import java.nio.file.FileVisitor; -import java.nio.file.Files; -import java.nio.file.Path; -import java.nio.file.Paths; -import java.nio.file.attribute.BasicFileAttributes; -import java.text.SimpleDateFormat; -import java.util.Arrays; -import java.util.HashSet; -import java.util.List; -import java.util.Map; -import java.util.Set; - -import org.apache.commons.io.FilenameUtils; -import org.apache.commons.lang.StringUtils; -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; -import org.apache.logging.log4j.message.Message; -import org.apache.logging.log4j.message.ParameterizedMessage; - -import com.ibm.nosql.json.api.BasicDBObject; - -import user.commons.JobStatus; -import user.commons.ListUtils; -import user.commons.log4j2.marker.MediaCubeMarker; -import user.commons.mediatool.MediaInfo; -import user.jobengine.server.IJobEngine; -import user.jobengine.server.IJobRuntime; -import user.jobengine.server.steps.shared.EscortFiles; -import user.mediacube.metadata.interfaces.IMetadata; -import user.mediacube.metadata.interfaces.IMetadataProvider; -import user.mediacube.metadata.interfaces.IMetadataProviderFactory; -import user.mediacube.metadata.interfaces.MetadataProviderType; -import user.mediacube.metadata.interfaces.MetadataType; -import user.mediacube.metadata.interfaces.PlanAirMetadataListOptions; - -public class ArchiveRecursive extends JobStep implements FileVisitor { - private static final Logger logger = LogManager.getLogger(); - private static final String JOBTEMPLATE = "archive-material.xml"; - private static final String ITEM_TITLE = "itemTitle"; - private static final String ITEM_HOUSEID = "itemHouseId"; - private static final String MEDIA_HOUSEID = "mediaHouseId"; - private static final String MEDIA_TITLE = "mediaTitle"; - private static final String MEDIA_DESCRIPTION = "mediaDescription"; - private static final String MEDIA_TYPE = "mediaType"; - private static final String ARCHIVE = "Archiválás"; - private static final String ARCHIVE_ITEM = "archiveItem"; - private static final String KILL_DATE_DAYS = "killDateDays"; - private SimpleDateFormat df = new SimpleDateFormat("yyyy.MM.dd"); - - private List skipPathNames = Arrays.asList("!ARCHIVALAS_ALATT", EscortFiles.STATUSFOLDER, - EscortFiles.CONFLICTFOLDER); - private int limit; - private int submitted; - private int killDateDays; - private boolean disableProxy; - - private boolean canReadMediaInfo(Path mediaFilePath) { - boolean result = false; - try { - MediaInfo mi = new MediaInfo(mediaFilePath); - mi.process(); - result = true; - } catch (Exception e) { - logger.warn(getSessionMarker(), e.getMessage()); - } - return result; - } - - private void checkArchiveItem(ArchiveItem archiveItem) throws Exception { - if (archiveItem == null) - throw new Exception("No metadata specified."); - - if (StringUtils.isBlank(archiveItem.getItemHouseId())) - throw new Exception("No Item HouseID specified in metadata."); - - if (StringUtils.isBlank(archiveItem.getItemTitle())) - throw new Exception("No Item Title specified in metadata."); - - if (StringUtils.isBlank(archiveItem.getMediaHouseId())) - throw new Exception("No Media HouseID specified in metadata."); - - if (StringUtils.isBlank(archiveItem.getMediaTitle())) - throw new Exception("No Media Title specified in metadata."); - } - - private String archiveItemJSON(ArchiveItem result) { - BasicDBObject obj = new BasicDBObject(); - obj.put("itemHouseId", result.getItemHouseId()); - obj.put("itemTitle", result.getItemTitle()); - obj.put("mediaHouseId", result.getMediaHouseId()); - obj.put("mediaTitle", result.getMediaTitle()); - obj.put("mediaDescription", result.getMediaDescription()); - obj.put("mediaType", result.getMediaType()); - return obj.toPrettyString(""); - } - - private ArchiveItem createArchiveItem(Path filePath) throws Exception { - ArchiveItem result = null; - - String fileName = filePath.getFileName().toString(); - String mediaHouseId = FilenameUtils.removeExtension(fileName); - try { - result = getPlanAirMetadata(mediaHouseId); - - if (result != null) { - result.setMediaFile(filePath.toString()); - // 210617 proxy keszites tiltasa - result.setDisableProxy(disableProxy); - } - - logger.info(getSessionMarker(), "PlanAir query done for {}", filePath); - } catch (Exception e) { - logger.error(getSessionMarker(), "PlanAir metadata error", e); - // nem latja a drivert pl. - throw e; - } - -//210616 Ha nincs metaadat, nem archivalunk - if (result == null) { - result = new ArchiveItem(); - BasicFileAttributes attr = Files.readAttributes(filePath, BasicFileAttributes.class); - result.setItemHouseId(df.format(attr.lastModifiedTime().toMillis())); - result.setItemTitle(filePath.getParent().toString()); - result.setMediaHouseId(mediaHouseId); - result.setMediaTitle(fileName); - result.setMediaDescription("/ARCHIVE-TEST"); - result.setMediaType("Generic"); - result.setMediaFile(filePath.toString()); - } - - return result; - } - - @StepEntry - public Object[] execute(String sourcePath, int killDateDays, int limit, boolean disableProxy) throws Exception { - this.killDateDays = killDateDays; - this.limit = limit; - this.disableProxy = disableProxy; - logger.info(getSessionMarker(), "Starting in {}", sourcePath); - try { - if (getJobRuntime().forkPrepare()) { - Files.walkFileTree(Paths.get(sourcePath), this); - } - } catch (Exception e) { - logger.error(getSessionMarker(), "Az '{}' mappa elérése sikertelen. A rendszer hibaüzenete: {}", sourcePath, - e.getMessage()); - } finally { - if (submitted > 0) - getJobRuntime().forkWaitComplete(); - else - getJobRuntime().cancelForkPrepare(); - } - return null; - } - - private ArchiveItem getPlanAirMetadata(String mediaHouseId) throws Exception { - PlanAirMetadataListOptions opt = new PlanAirMetadataListOptions(); - opt.setSearch(mediaHouseId); - opt.setType(MetadataType.Material); - - BasicDBObject json = null; - List data = null; - - IMetadataProviderFactory factory = getService(IMetadataProviderFactory.class); - if (factory == null) - logger.info(getSessionMarker(), "IMetadataProviderFactory is null"); - - IMetadataProvider planairProvider = factory.getProvider(MetadataProviderType.PLANAIR); - if (planairProvider == null) - logger.info(getSessionMarker(), "IMetadataProvider is null"); - - ArchiveItem result = null; - data = planairProvider.list(opt); - if (data.size() != 0) - json = data.get(0).asJSON(); - else { - opt.setType(MetadataType.Promo); - data = planairProvider.list(opt); - if (data.size() != 0) - json = data.get(0).asJSON(); - else { - opt.setType(MetadataType.AD); - data = planairProvider.list(opt); - if (data.size() != 0) - json = data.get(0).asJSON(); - } - } - if (json != null) { - result = new ArchiveItem(); - result.setItemHouseId(json.getString(ITEM_HOUSEID)); - result.setItemTitle(json.getString(ITEM_TITLE)); - result.setMediaHouseId(json.getString(MEDIA_HOUSEID)); - result.setMediaTitle(json.getString(MEDIA_TITLE)); - result.setMediaDescription(json.getString(MEDIA_DESCRIPTION)); - result.setMediaType(json.getString(MEDIA_TYPE)); - } - return result; - } - - private boolean handleArchiveConflict(Path mediaPath) throws Exception { - boolean result = false; - String sourceFileName = mediaPath.getFileName().toString(); - if (getManager().isMediaFileExists(sourceFileName)) { - EscortFiles.createMediaCatch(mediaPath); - result = true; - } - return result; - } - - @Override - public FileVisitResult postVisitDirectory(Path paramT, IOException paramIOException) throws IOException { - return FileVisitResult.CONTINUE; - } - - @Override - public FileVisitResult preVisitDirectory(Path dir, BasicFileAttributes paramBasicFileAttributes) - throws IOException { - Path dirName = dir.getFileName(); - - if (skipPathNames.contains(dirName.toString())) { - logger.info(getSessionMarker(), "PreVisit skip {}", dir); - return FileVisitResult.SKIP_SUBTREE; - } else - logger.info(getSessionMarker(), "PreVisit {}", dir); - - return FileVisitResult.CONTINUE; - } - - private boolean processPathItem(Path mediaPath) throws Exception { - if (limit != 0 && submitted == limit) { - logger.info(getSessionMarker(), "Limit reached {}, canceling", limit); - return false; - } - - String fileName = mediaPath.getFileName().toString(); - if (fileName.startsWith(".") || fileName.endsWith(".nomd")) - return false; - - logger.info(getSessionMarker(), "Processing {}", mediaPath); - - File mediaFile = mediaPath.toFile(); - if (mediaFile.isDirectory()) { - //logger.info(getSessionMarker(), "Skipping directory {}", mediaPath); - return false; - } - - if (EscortFiles.isMediaCatched(mediaPath)) { - //logger.info(getSessionMarker(), "Skipping already catched {}", mediaPath); - return false; - } - - Path nomdFile = Paths.get(mediaPath.toString() + ".nomd"); - - if (Files.exists(nomdFile)) { - //logger.info(getSessionMarker(), "Skipping nomd file exists {}", nomdFile); - return false; - } - - if (handleArchiveConflict(mediaPath)) { - logger.info(getSessionMarker(), "Skipping archive db already contains {}", mediaPath); - return false; - } - - if (!canReadMediaInfo(mediaPath)) { - logger.info(getSessionMarker(), "Skipping cant read mediainfo {}", mediaPath); - return false; - } - - ArchiveItem archiveItem = createArchiveItem(mediaPath); - - if (archiveItem == null) { - Message msg = new ParameterizedMessage("Nincs metaadat!"); - logger.info(new MediaCubeMarker("vasary@elgekko.net,muszak@mediavivantis.hu", - "Értesítés problémás " + mediaPath.getFileName().toString() + " archiválásról"), msg); - Files.createFile(nomdFile); - return false; - } - - try { - checkArchiveItem(archiveItem); - - Map parameters = ListUtils.asMap(ARCHIVE_ITEM, archiveItem, KILL_DATE_DAYS, killDateDays); - IJobRuntime runtime = getEngine().submit(getJobRuntime(), e -> { - if (e.getStatus().equals(JobStatus.CANCELED) || e.getStatus().equals(JobStatus.SUSPENDED)) - EscortFiles.removeMediaCatch(mediaPath); - }, JOBTEMPLATE, ARCHIVE, 1, IJobEngine.DEFAULT_OWNER, parameters); - if (runtime == null) - throw new Exception("Submit returned null runtime"); - runtime.setRelated(mediaPath.toString()); - EscortFiles.createMediaCatch(mediaPath); - String metadata = archiveItemJSON(archiveItem); - EscortFiles.createMetadata(mediaPath.getParent().toString(), mediaPath.getFileName().toString(), metadata); - submitted++; - } catch (Exception e) { - logger.error(getSessionMarker(), - "A(z) '{}' állomány archiválási kísérlete sikertelen. A rendszer üzenete: {}", mediaPath, - e.getMessage()); - } - - return true; - } - - @Override - public FileVisitResult visitFile(Path filePath, BasicFileAttributes paramBasicFileAttributes) throws IOException { - logger.info(getSessionMarker(), "Will checked {}", filePath); - try { - processPathItem(filePath); - } catch (Exception e) { - logger.catching(e); - } - return FileVisitResult.CONTINUE; - } - - @Override - public FileVisitResult visitFileFailed(Path filePath, IOException paramIOException) throws IOException { - logger.info("Error archive {}", filePath); - return FileVisitResult.CONTINUE; - } -} diff --git a/server/-product/production/OMAR/jobs/steps/BatchRetrieveForkStep.java b/server/-product/production/OMAR/jobs/steps/BatchRetrieveForkStep.java deleted file mode 100644 index 4f6b130b..00000000 --- a/server/-product/production/OMAR/jobs/steps/BatchRetrieveForkStep.java +++ /dev/null @@ -1,76 +0,0 @@ -package user.jobengine.server.steps; - -import java.util.List; -import java.util.Map; - -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; - -import user.commons.log4j2.marker.MediaCubeFinishMarker; -import user.commons.log4j2.marker.MediaCubeMarker; -import user.jobengine.db.ArchivedMedia; -import user.jobengine.server.IJobEngine; -import user.jobengine.server.IJobRuntime; -import user.jobengine.server.scheduler.ScheduledJob; - -public class BatchRetrieveForkStep extends JobStep { - private static final String TARGET_PATH_TYPE = "targetPathType"; - private static final Logger logger = LogManager.getLogger(); - private static final String CHILD_TEMPLATE = "retrieve-ondemand.xml"; - private static final String ARCHIVEDMEDIA = "archivedMedia"; - private static final String RECIPIENT = "successRecipient"; - private static final String HOUSEID = "houseId"; - private MediaCubeMarker marker; - - @StepEntry - public Object[] execute(List basket, String houseId, String recipient, String targetPathType, IJobEngine jobEngine, IJobRuntime jobRuntime) - throws Exception { - marker = (MediaCubeMarker) jobRuntime.getSessionMarker(); - - //session szinten csak a finishMarker cimzettje az erdekes, es ezt a cimet pluszban hasznalja a konfigban megadott cimmel - //a finishMarker orokli a cim bellitast a sessionMarkertol - marker.setTo(recipient); - - ((MediaCubeMarker) jobRuntime.getFinishMarker()).setTo(recipient); - - if (basket == null || basket.size() == 0) - return null; - setProgress(10); - - MediaCubeMarker mailMarker = new MediaCubeMarker(recipient); - mailMarker.setSessionName("Archívum viszatöltés"); - mailMarker.setSessionID(houseId); - logger.info(mailMarker, "A visszatöltések elindultak az alábbi állományokra:"); - - if (jobRuntime.forkPrepare()) { - for (ArchivedMedia archivedMedia : basket) { - logger.info(mailMarker, archivedMedia.getMedia().getMediaFilesName()); - submit(archivedMedia, recipient, houseId, targetPathType, jobEngine, jobRuntime); - } - } - setProgress(50); - logger.info(new MediaCubeFinishMarker(mailMarker), "A visszatöltések végeztével megerősítő üzenetet küldünk."); - jobRuntime.forkWaitComplete(); - setProgress(100); - return null; - } - - public void submit(ArchivedMedia archivedMedia, String recipient, String houseId, String targetPathType, IJobEngine jobEngine, IJobRuntime jobRuntime) - throws Exception { - try { - ScheduledJob scheduledJob = jobEngine.getScheduledJob(CHILD_TEMPLATE); - Map parameters = scheduledJob.getJobParameters(); - parameters.put(ARCHIVEDMEDIA, archivedMedia); - parameters.put(HOUSEID, houseId); - parameters.put(RECIPIENT, recipient); - parameters.put(TARGET_PATH_TYPE, targetPathType); - IJobRuntime child = jobEngine.submit(jobRuntime, null, CHILD_TEMPLATE, String.format("Visszatöltés %s részére", recipient), parameters); - ((MediaCubeMarker) child.getSessionMarker()).setTo(recipient); - } catch (Exception e) { - logger.catching(e); - logger.error(marker, "Hiba a kötegelt visszatöltésben. A rendszer üzenete: {}", e.getMessage()); - } - - } - -} diff --git a/server/-product/production/OMAR/jobs/steps/CalculateMD5Step.java b/server/-product/production/OMAR/jobs/steps/CalculateMD5Step.java deleted file mode 100644 index 75af5964..00000000 --- a/server/-product/production/OMAR/jobs/steps/CalculateMD5Step.java +++ /dev/null @@ -1,36 +0,0 @@ -package user.jobengine.server.steps; - -import java.io.InputStream; -import java.nio.file.Files; -import java.nio.file.Paths; -import java.security.DigestInputStream; -import java.security.MessageDigest; - -import javax.xml.bind.DatatypeConverter; - -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; - -public class CalculateMD5Step extends JobStep { - private static final Logger logger = LogManager.getLogger(CalculateMD5Step.class); - - @StepEntry - public Object[] execute(String fileName) throws Exception { - logger.info(getMarker(), "Executing"); - - MessageDigest messageDigest = MessageDigest.getInstance("MD5"); - InputStream is = Files.newInputStream(Paths.get(fileName)); - DigestInputStream dis = new DigestInputStream(is, messageDigest); - byte[] digest = new byte[32768]; //32k buffer - - while ((dis.read(digest)) != -1) { - messageDigest.update(digest); - } - - String md5String = DatatypeConverter.printHexBinary(digest).toUpperCase(); - logger.info("calculated MD5 hash= {}", md5String); - dis.close(); - is.close(); - return new Object[] { md5String }; - } -} \ No newline at end of file diff --git a/server/-product/production/OMAR/jobs/steps/CancelableStep.java b/server/-product/production/OMAR/jobs/steps/CancelableStep.java deleted file mode 100644 index 9293427a..00000000 --- a/server/-product/production/OMAR/jobs/steps/CancelableStep.java +++ /dev/null @@ -1,42 +0,0 @@ -package user.jobengine.server.steps; - -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; - -public class CancelableStep extends JobStep { - private static final Logger logger = LogManager.getLogger(); - int count = 10; - - @StepEntry - public Object[] execute(int param) throws Exception { - - getJobRuntime().setRelated("TESZT" + param); - for (int i = 0; i < count; i++) { - if (getJobRuntime().isWaitingCancel()) - break; -// if (i == 1) -// throw new Exception("AAAAAAAAAA"); - - Thread.sleep(1000); - int progress = (i + 1) * 100 / count; - setProgress(progress); - } - -// try { -// -// logger.warn("Ez a fo logba megy"); -// -// //marker="MEDIACUBE" -// logger.warn(getMarker(), "Ez a markered logba megy"); -// -// //marker="MEDIACUBE | folyamat_nev" -// logger.warn(getJobRuntime().getSessionMarker(), "Ez a markered logba megy es a nevesitett logba"); -// -// } catch (Exception e) { -// e.printStackTrace(); -// throw e; -// } - return new Object[] { param }; - } - -} diff --git a/server/-product/production/OMAR/jobs/steps/CleanupMountedLocationStep.java b/server/-product/production/OMAR/jobs/steps/CleanupMountedLocationStep.java deleted file mode 100644 index 95fff73f..00000000 --- a/server/-product/production/OMAR/jobs/steps/CleanupMountedLocationStep.java +++ /dev/null @@ -1,283 +0,0 @@ -package user.jobengine.server.steps; - -import java.io.File; -import java.io.IOException; -import java.nio.file.DirectoryStream; -import java.nio.file.FileVisitResult; -import java.nio.file.FileVisitor; -import java.nio.file.Files; -import java.nio.file.Path; -import java.nio.file.Paths; -import java.nio.file.SimpleFileVisitor; -import java.nio.file.attribute.BasicFileAttributes; -import java.text.ParseException; -import java.text.SimpleDateFormat; -import java.util.ArrayList; -import java.util.Collections; -import java.util.Date; -import java.util.List; - -import org.apache.commons.io.FileUtils; -import org.apache.commons.lang.StringUtils; -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; -import org.apache.logging.log4j.Marker; - -import user.jobengine.server.steps.shared.ItemManagerExtensions; - -public class CleanupMountedLocationStep extends JobStep implements FileVisitor { - - private static final Logger logger = LogManager.getLogger(); - private static final String PROJECTFOLDER = "PROJECT"; - private static final String DATEFORMAT = "yyyyMMdd"; - private static final String DOT = "."; - private static final String STATUSFOLDER = ".STATUS"; - private static final String EWC2EXT = ".ewc2"; - private static final String XMPEXT = ".xmp"; - private static final String CATCHEDEXT = ".catched"; - private static final String KILLDATEEXT = ".killdate"; - private static final String JSONEXT = ".json"; - - private static boolean isEmpty(final Path directory) throws IOException { - try (DirectoryStream dirStream = Files.newDirectoryStream(directory)) { - final int[] count = new int[] { 0 }; - final int[] specialCount = new int[] { 0 }; - dirStream.forEach(p -> { - count[0]++; - // if (p.getFileName().toString().toLowerCase().equals(PROJECTFOLDER.toLowerCase())) - // specialCount[0]++; - if (p.getFileName().toString().toLowerCase().equals(STATUSFOLDER.toLowerCase())) - specialCount[0]++; - - }); - if (specialCount[0] == count[0]) - return true; - } - return false; - } - - private Marker marker; - - final int[] allCount = new int[] { 0 }; - final int[] currentCount = new int[] { 0 }; - - private Path sourcePath; - private SimpleDateFormat dateFormat; - private boolean skipArchiveCheck; - - private Date checkExpiration(List killDateFiles) { - Date killDate = null; - for (Path killDateFile : killDateFiles) { - Date currentKillDate = getKillDate(killDateFile); - if (currentKillDate == null) - continue; - if ((killDate != null && currentKillDate.after(killDate)) || killDate == null) - killDate = currentKillDate; - } - return new Date().after(killDate) ? killDate : null; - } - - @StepEntry - public Object[] execute(String sourceFolder, boolean skipArchiveCheck) throws Exception { - this.skipArchiveCheck = skipArchiveCheck; - marker = getSessionMarker(); - sourcePath = Paths.get(sourceFolder); - DirectoryStream directoryStream = null; - if (StringUtils.isBlank(sourcePath.toString())) { - logger.error(marker, "A folyamat 'sourcePath' bemeneti paramétere üres."); - throw new NullPointerException("System is not configured properly, 'sourceFolder' input parameter missing."); - } - - if (!Files.exists(sourcePath) || !Files.isDirectory(sourcePath)) { - logger.error(marker, "A {} mappa nem létezik.", sourceFolder); - throw new NullPointerException(String.format("Directory %s does not exist.", sourceFolder)); - } - - try { - setProgress(1); - dateFormat = new SimpleDateFormat(DATEFORMAT); - - Files.walkFileTree(sourcePath, new SimpleFileVisitor() { - @Override - public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) throws IOException { - allCount[0]++; - return super.visitFile(file, attrs); - } - }); - Files.walkFileTree(sourcePath, this); - } catch (Exception e) { - logger.catching(e); - logger.error(marker, "Hiba a '{}' mappa feldolgozásában. A rendszer hibaüzenete: {}", sourcePath, e.getMessage()); - throw e; - } finally { - if (directoryStream != null) { - try { - directoryStream.close(); - } catch (IOException e) { - } - } - } - return null; - } - - private Date getKillDate(Path killDateFile) { - String fileName = killDateFile.getFileName().toString(); - int end = fileName.lastIndexOf(DOT); - if (end < 1) - return null; - int start = fileName.lastIndexOf(DOT, end - 1); - if (start < 0) - return null; - String strKillDate = fileName.substring(start + 1, end); - Date result = null; - if (StringUtils.isNumeric(strKillDate)) { - try { - result = dateFormat.parse(strKillDate); - } catch (ParseException e) { - logger.error(marker, "A {} fájl 'killdate' állománya hibás formátumú, a {} karaktersorozat nem konvertálható dátummá.", fileName, strKillDate); - return null; - } - } else - logger.error(marker, "A {} fájl 'killdate' állománya hibás formátumú, az dátum helyett ez áll: '{}'.", fileName, strKillDate); - return result; - } - - private List getKillDateFiles(Path filePath) { - String killDateFilePattern = String.format("%s.*%s", filePath.getFileName().toString(), KILLDATEEXT); - List result = new ArrayList<>(); - Path statusPath = null; - try { - statusPath = Paths.get(filePath.getParent().toString(), STATUSFOLDER); - } catch (Exception e) { - logger.catching(e); - return null; - } - File statusPathFile = statusPath.toFile(); - if (statusPathFile.exists() && statusPathFile.isDirectory()) { - try (DirectoryStream stream = Files.newDirectoryStream(statusPath, killDateFilePattern)) { - stream.forEach(p -> result.add(p)); - } catch (Exception e) { - logger.catching(e); - } - } - Collections.sort(result); - return result; - } - - @Override - public FileVisitResult postVisitDirectory(Path dir, IOException exc) throws IOException { - if (!dir.equals(sourcePath) && isEmpty(dir)) { - if (!removeExistingSpecialDirectory(dir, PROJECTFOLDER)) - return FileVisitResult.CONTINUE; - if (!removeExistingSpecialDirectory(dir, STATUSFOLDER)) - return FileVisitResult.CONTINUE; - if (removeFile(dir)) - logger.info(marker, "A {} üres mappa törlése sikeres.", dir); - - } - return FileVisitResult.CONTINUE; - } - - @Override - public FileVisitResult preVisitDirectory(Path dir, BasicFileAttributes attrs) throws IOException { - - //A .-al kezdodo mappakat kihagyjuk - if (dir.getFileName().toString().startsWith(".")) - return FileVisitResult.SKIP_SUBTREE; - - return FileVisitResult.CONTINUE; - } - - private void processPathItem(Path filePath) { - currentCount[0]++; - - if (filePath.getFileName().toString().startsWith(".")) - return; - - int progress = currentCount[0] * 100 / allCount[0]; - setProgress(progress); - - logger.info("Checking {}", filePath); - List killDateFiles = getKillDateFiles(filePath); - if (killDateFiles == null || killDateFiles.size() == 0) { - logger.warn(marker, "A {} fájlhoz nem található 'killdate' állomány.", filePath); - return; - } - - if (killDateFiles.size() != 1) - logger.warn(marker, "A {} fájlhoz több 'killdate' állomány található, a legújabb dátum határozza meg a törlés időpontját.", filePath); - - Date killDate = checkExpiration(killDateFiles); - if (killDate == null) - return; - - if (!skipArchiveCheck && filePath.toFile().length() > 0) { - if (!ItemManagerExtensions.isArchived(getManager(), filePath)) { - logger.error(marker, "A(z) {} anyag törlésre van kijelölve, de nem található az archívumban.", filePath); - return; - } - } - - if (removeFiles(filePath, killDateFiles)) - logger.info(marker, "A {} fájl és kapcsolódó állományai a {} killdate bejegyzés alapján sikeresen törlődtek.", filePath.getFileName(), - dateFormat.format(killDate)); - else - logger.warn(marker, "A {} fájl és kapcsolódó állományai a {} killdate bejegyzés alapján csak részlegesen vagy egyáltalán nem törlődtek.", - filePath.getFileName(), dateFormat.format(killDate)); - } - - private boolean removeExistingSpecialDirectory(Path dir, String folderName) throws IOException { - File projectPath = Paths.get(dir.toString(), folderName).toFile(); - if (projectPath.exists() && projectPath.isDirectory()) { - FileUtils.deleteDirectory(projectPath); - if (projectPath.exists()) { - logger.warn(marker, "A {} alatti {} mappa törlése nem sikerült.", dir, folderName); - return false; - } - } - return true; - } - - private boolean removeFile(Path filePath) { - boolean result = false; - try { - //logger.error("REMOVE {}", filePath); - File file = filePath.toFile(); - if (file.exists()) - result = file.delete(); - } catch (Exception e) { - logger.error(marker, "A {} fájl nem törölhető. A rendszer hibaüzenete: {}", filePath, e.getMessage()); - } - return result; - } - - private boolean removeFiles(Path filePath, List killDateFiles) { - if (!removeFile(filePath)) - return false; - - removeFile(Paths.get(filePath.toString() + EWC2EXT)); - removeFile(Paths.get(filePath.toString() + XMPEXT)); - removeFile(Paths.get(filePath.getParent().toString(), STATUSFOLDER, filePath.getFileName().toString() + CATCHEDEXT)); - removeFile(Paths.get(filePath.getParent().toString(), STATUSFOLDER, filePath.getFileName().toString() + JSONEXT)); - - boolean result = true; - for (Path killDateFile : killDateFiles) { - if (!removeFile(killDateFile)) - result = false; - } - - return result; - } - - @Override - public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) throws IOException { - processPathItem(file); - return FileVisitResult.CONTINUE; - } - - @Override - public FileVisitResult visitFileFailed(Path file, IOException e) throws IOException { - logger.error(marker, "A {} fájl nem érhető el. A rendszer hibaüzenete: {}", file.toString(), e.getMessage()); - return FileVisitResult.CONTINUE; - } -} diff --git a/server/-product/production/OMAR/jobs/steps/CreateArchiveItemStep.java b/server/-product/production/OMAR/jobs/steps/CreateArchiveItemStep.java deleted file mode 100644 index ec165e87..00000000 --- a/server/-product/production/OMAR/jobs/steps/CreateArchiveItemStep.java +++ /dev/null @@ -1,47 +0,0 @@ -package user.jobengine.server.steps; - -import java.nio.file.Paths; - -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; - -import com.ibm.nosql.json.api.BasicDBObject; -import com.ibm.nosql.json.api.DB; -import com.ibm.nosql.json.api.DBCollection; - -import user.commons.nosql.NoSQLUtils; -import user.jobengine.db.Media; -import user.jobengine.db.Store; - -public class CreateArchiveItemStep extends JobStep { - private static final Logger logger = LogManager.getLogger(); - - @StepEntry - public Object[] execute(Media mediaCubeMedia, String localHiresPath) throws Exception { - DB db = NoSQLUtils.getNoSQLDB(); - DBCollection collection = db.getCollection("missing_lowres"); - Store highResStore = getManager().getSystemStore(false); - - ArchiveItem archiveItem = null; - try { - if (mediaCubeMedia.getMediaFilesCount() != 1) { - throw new Exception("Expected media count is 1, found " + mediaCubeMedia.getMediaFilesCount()); - } - if (mediaCubeMedia.getMediaFiles().get(0).getStoreId() != highResStore.getId()) { - throw new Exception("Expected media store is a high-res store"); - } - - String name = mediaCubeMedia.getMediaFileRealName(); - archiveItem = new ArchiveItem(); - archiveItem.setMediaFile(Paths.get(localHiresPath, name).toString()); - collection.save(new BasicDBObject("name", name)); - } catch (Exception e) { - logger.catching(e); - logger.info(getMarker(), e.getMessage()); - throw e; - } finally { - setProgress(100); - } - return new Object[] { archiveItem }; - } -} diff --git a/server/-product/production/OMAR/jobs/steps/CreateMissingLowresStep.java b/server/-product/production/OMAR/jobs/steps/CreateMissingLowresStep.java deleted file mode 100644 index 72b4cfb1..00000000 --- a/server/-product/production/OMAR/jobs/steps/CreateMissingLowresStep.java +++ /dev/null @@ -1,95 +0,0 @@ -package user.jobengine.server.steps; - -import java.nio.file.Paths; - -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; - -import com.ibm.nosql.json.api.BasicDBObject; -import com.ibm.nosql.json.api.DB; -import com.ibm.nosql.json.api.DBCollection; - -import user.commons.log4j2.marker.MediaCubeUndoMarker; -import user.commons.nosql.NoSQLUtils; -import user.jobengine.db.IItemManager; -import user.jobengine.db.Media; -import user.jobengine.db.MediaFile; -import user.jobengine.server.IJobEngine; -import user.jobengine.server.IJobRuntime; - -public class CreateMissingLowresStep extends JobStep { - private static final Logger logger = LogManager.getLogger(); - - @StepEntry - public Object[] execute(String localHiresPath, IJobEngine jobEngine, IJobRuntime jobRuntime) throws Exception { - Object[] result = new Object[] { null, null, "%s", null, 0, true }; - - DB db = NoSQLUtils.getNoSQLDB(); - DBCollection collection = db.getCollection("missing_lowres"); - IItemManager manager = jobEngine.getItemManager(); - Media media = getFirstUntranscodedMedia(manager, collection); - - try { - if (media == null) { - logger.info(new MediaCubeUndoMarker(getSessionMarker().getSessionID()), "Nincs feldolgozandó hiány."); - // throw new Exception("Nincs feldolgozandó hiány."); - cancel(); - return null; - } - - String name = media.getMediaFileRealName(); - result[0] = media; - ArchiveItem archiveItem = new ArchiveItem(); - archiveItem.setMediaFile(Paths.get(localHiresPath, name).toString()); - result[1] = archiveItem; - collection.save(new BasicDBObject("name", name)); - logger.info(getSessionMarker(), "Processing mediaID: {}", media.getId()); - - } catch (Exception e) { - logger.catching(e); - logger.error(getSessionMarker(), e.getMessage()); - throw e; - } finally { - setProgress(100); - } - return result; - } - - private Media getFirstUntranscodedMedia(IItemManager manager, DBCollection collection) { - Media[] result = new Media[] { null }; - // MV - String query = "SELECT mediaid FROM VW_MISSING_PROXY_IDS WHERE HOUSEID like 'M%' or HOUSEID like 'P%' or HOUSEID like 'R%' ORDER BY modified DESC"; - - // HTV - // String query = "SELECT mediaid FROM VW_MISSING_PROXY_IDS"; - manager.executeQuery(query, rs -> { - try { - long mediaId = rs.getLong(1); - Media media = manager.getMedia(mediaId); - // a nevgeneralas miatt az eredeti MediaFilesName nem jo, a pontos nev kell - // nekunk - String name = media.getMediaFileRealName(); - //logger.info(getSessionMarker(), "Checking {}", name); - long existing = collection.find(new BasicDBObject("name", name)).count(); - if (existing > 0) { - // logger.info(getSessionMarker(), "{} is on missing_lowres list", name); - return true; - } - - // 210617 proxy keszites tiltasa - MediaFile mf = manager.getSystemMediaFile(media); - if (mf.isDisableProxy()) { - logger.info(getSessionMarker(), "Proxy disabled {}", name); - - return true; - } - - result[0] = media; - } catch (Exception e) { - logger.error(e); - } - return false; - }, null); - return result[0]; - } -} diff --git a/server/-product/production/OMAR/jobs/steps/DeleteFile.java b/server/-product/production/OMAR/jobs/steps/DeleteFile.java deleted file mode 100644 index 539faf74..00000000 --- a/server/-product/production/OMAR/jobs/steps/DeleteFile.java +++ /dev/null @@ -1,22 +0,0 @@ -package user.jobengine.server.steps; - -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; - -import user.commons.RemoteFile; -import user.commons.StoreUri; - -public class DeleteFile extends JobStep { - private static final Logger logger = LogManager.getLogger(); - - @StepEntry - public Object[] execute(String fileName, StoreUri tempStoreUri) throws Exception { - try { - RemoteFile remoteFile = tempStoreUri.getRemoteFile(fileName); - tempStoreUri.delete(remoteFile); - } catch (Exception e) { - logger.warn(getJobRuntime().getSessionMarker(), e.getMessage()); - } - return null; - } -} diff --git a/server/-product/production/OMAR/jobs/steps/DummyTestStep1.java b/server/-product/production/OMAR/jobs/steps/DummyTestStep1.java deleted file mode 100644 index 22b3ab97..00000000 --- a/server/-product/production/OMAR/jobs/steps/DummyTestStep1.java +++ /dev/null @@ -1,23 +0,0 @@ -package user.jobengine.server.steps; - -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; - -public class DummyTestStep1 extends JobStep { - private static final Logger logger = LogManager.getLogger(DummyTestStep1.class); - - @StepEntry - public Object[] execute(String param1) { - logger.info("Executing DummyTestStep1"); - int var1 = 0; - if (param1.equals("Jozsi")) { - var1 = 1; - Thread.sleep(1000); - setProgress(50); - Thread.sleep(1000); - setProgress(50); - } - - return new Object[] {var1}; - } -} diff --git a/server/-product/production/OMAR/jobs/steps/DummyTestStep2.java b/server/-product/production/OMAR/jobs/steps/DummyTestStep2.java deleted file mode 100644 index e7579f12..00000000 --- a/server/-product/production/OMAR/jobs/steps/DummyTestStep2.java +++ /dev/null @@ -1,20 +0,0 @@ -package user.jobengine.server.steps; - -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; - -public class DummyTestStep2 extends JobStep { - private static final Logger logger = LogManager.getLogger(DummyTestStep2.class); - - @StepEntry - public Object[] execute(int var2) { - logger.info("Executing DummyTestStep2"); - long var3 = 0L; - Thread.sleep(1000); - setProgress(50); - Thread.sleep(1000); - setProgress(50); - - return new Object[] { var3 }; - } -} diff --git a/server/-product/production/OMAR/jobs/steps/DummyTestStep3.java b/server/-product/production/OMAR/jobs/steps/DummyTestStep3.java deleted file mode 100644 index 93d76a75..00000000 --- a/server/-product/production/OMAR/jobs/steps/DummyTestStep3.java +++ /dev/null @@ -1,17 +0,0 @@ -package user.jobengine.server.steps; - -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; - -public class DummyTestStep3 extends JobStep { - private static final Logger logger = LogManager.getLogger(DummyTestStep3.class); - - @StepEntry - public void execute(long var3) { - Thread.sleep(1000); - setProgress(50); - Thread.sleep(1000); - setProgress(50); - logger.info("Executing DummyTestStep3"); - } -} diff --git a/server/-product/production/OMAR/jobs/steps/FileCopyStep.java b/server/-product/production/OMAR/jobs/steps/FileCopyStep.java deleted file mode 100644 index c4172550..00000000 --- a/server/-product/production/OMAR/jobs/steps/FileCopyStep.java +++ /dev/null @@ -1,95 +0,0 @@ -package user.jobengine.server.steps; - -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; -import org.apache.logging.log4j.Marker; - -import user.commons.StoreUri; -import user.commons.remotestore.IProgressEventListener; -import user.commons.remotestore.ProgressEvent; -import user.commons.remotestore.RemoteStoreProtocol; -import user.jobengine.db.IItemManager; -import user.jobengine.server.IJobEngine; -import user.jobengine.server.IJobRuntime; - -public class FileCopyStep extends JobStep { - private static final Logger logger = LogManager.getLogger(); - private IItemManager manager; - private Marker marker; - - private void check(String sourceProtocol, String sourcePath, String sourceFileName, String targetProtocol, String targetPath, String targetFileName, - IJobEngine jobEngine, IJobRuntime jobRuntime) { - if (jobEngine == null) { - logger.error(marker, "A folyamatkezelő réteg nem elérhető."); - throw new NullPointerException("Internal error, missing JobEngine reference."); - } - manager = jobEngine.getItemManager(); - if (manager == null) { - logger.error(marker, "Az adatbáziskezelő réteg nem elérhető."); - throw new NullPointerException("Internal error, missing ItemManager reference."); - } - if (sourceProtocol == null) { - logger.error(marker, "A forrás protokol bemeneti paramétere üres."); - throw new NullPointerException("System is not configured properly, 'sourceProtocol' input parameter missing."); - } - if (sourcePath == null) { - logger.error(marker, "A forrás fájl elérés bemeneti paramétere üres."); - throw new NullPointerException("System is not configured properly, 'sourcePath' input parameter missing."); - } - if (sourceFileName == null) { - logger.error(marker, "A forrás fájlnév bemeneti paramétere üres."); - throw new NullPointerException("System is not configured properly, 'sourceFileName' input parameter missing."); - } - if (targetProtocol == null) { - logger.error(marker, "A cél protokol bemeneti paramétere üres."); - throw new NullPointerException("System is not configured properly, 'targetProtocol' input parameter missing."); - } - if (targetPath == null) { - logger.error(marker, "A cél fájl elérés bemeneti paramétere üres."); - throw new NullPointerException("System is not configured properly, 'targetPath' input parameter missing."); - } - if (targetFileName == null) { - logger.error(marker, "A cél fájlnév bemeneti paramétere üres."); - throw new NullPointerException("System is not configured properly, 'targetFileName' input parameter missing."); - } - } - - @StepEntry - public Object[] execute(String sourceProtocol, String sourcePath, String sourceFileName, String targetProtocol, String targetPath, String targetFileName, - int killDateDays, IJobEngine jobEngine, IJobRuntime jobRuntime) throws Exception { - marker = jobRuntime.getSessionMarker(); - - check(sourceProtocol, sourcePath, sourceFileName, targetProtocol, targetPath, targetFileName, jobEngine, jobRuntime); - - StoreUri source = null; - StoreUri target = null; - - try { - source = manager.createStoreUri(Enum.valueOf(RemoteStoreProtocol.class, sourceProtocol), sourcePath); - target = manager.createStoreUri(Enum.valueOf(RemoteStoreProtocol.class, targetProtocol), targetPath); - source.addProgressListener(new IProgressEventListener() { - @Override - public void progressChanged(ProgressEvent evt) { - jobRuntime.incrementProgress(evt.getProgress()); - } - }); - - source.transferFrom(target, sourceFileName, targetFileName); - if (killDateDays > -1) - EscortFiles.createUNCKillDate(targetPath, targetFileName, killDateDays, marker); - - } catch (Exception e) { - logger.catching(e); - throw e; - } finally { - if (source != null) { - source.cleanUp(); - } - if (target != null) { - target.cleanUp(); - } - } - - return null; - } -} diff --git a/server/-product/production/OMAR/jobs/steps/GenerateJSONMetadataStep.java b/server/-product/production/OMAR/jobs/steps/GenerateJSONMetadataStep.java deleted file mode 100644 index 210d7354..00000000 --- a/server/-product/production/OMAR/jobs/steps/GenerateJSONMetadataStep.java +++ /dev/null @@ -1,65 +0,0 @@ -package user.jobengine.server.steps; - -import java.io.IOException; -import java.nio.file.Paths; -import java.sql.Timestamp; -import java.util.Map; - -import org.apache.commons.io.FilenameUtils; -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; - -import user.commons.DownloadableMedia; -import user.commons.StoreUri; -import user.commons.mediaarea.MediaArea; -import user.commons.remotestore.RemoteStoreProtocol; -import user.jobengine.server.steps.shared.EscortFiles; - -public class GenerateJSONMetadataStep extends JobStep { - private static final Logger logger = LogManager.getLogger(); - - @StepEntry - public Object[] execute(Map files, String sourceStoreName, String targetStoreName, - String escortStoreFolder) { - if (!files.isEmpty()) { - for (String fullPath : files.keySet()) { - MediaArea mediaArea = files.get(fullPath); - String title = FilenameUtils.getBaseName(fullPath).replace(FilenameUtils.getExtension(fullPath), ""); - String pathOnly = fullPath.substring(0, fullPath.lastIndexOf("\\") + 1); - String filenameOnly = FilenameUtils.getBaseName(fullPath); - Timestamp created = null; - Timestamp modified = null; - long frameCount = mediaArea.getFrameCount(); - long mediaId = 0; - StoreUri sourceStoreUri = getManager().getStoreUri(sourceStoreName, RemoteStoreProtocol.LOCAL); - StoreUri targetStoreUri = getManager().getStoreUri(targetStoreName, RemoteStoreProtocol.LOCAL); - StoreUri escortStoreUri = getManager().getStoreUri(escortStoreFolder, RemoteStoreProtocol.LOCAL); - - try { - String outputPath = Paths.get(escortStoreUri.toString(true)).toString(); - } catch (Exception e1) { - e1.printStackTrace(); - } - - DownloadableMedia downloadable = DownloadableMedia.create(title, filenameOnly, modified, created, - frameCount, 0, sourceStoreUri.getId(), targetStoreUri.getId(), mediaId); - String escortFileName = targetStoreName + "." + downloadable.getString("fileName"); // needed without - // extension - try { - if (EscortFiles.createMetadataIfNotExists(pathOnly, escortFileName, - downloadable.toPrettyString(""))) { - logger.info(getMarker(), "Archive status file created for {}", fullPath); - } else { - logger.info(getMarker(), "Archive status file already exists for {}", fullPath); - } - } catch (IOException e) { - logger.error("{}", e.getCause()); - e.printStackTrace(); - } - } - } else { - logger.info("files is empty!"); - } - return null; - } -} diff --git a/server/-product/production/OMAR/jobs/steps/HSMMigrateStep.java b/server/-product/production/OMAR/jobs/steps/HSMMigrateStep.java deleted file mode 100644 index 895b1411..00000000 --- a/server/-product/production/OMAR/jobs/steps/HSMMigrateStep.java +++ /dev/null @@ -1,401 +0,0 @@ -package user.jobengine.server.steps; - -import java.io.BufferedInputStream; -import java.io.BufferedOutputStream; -import java.io.File; -import java.io.FileInputStream; -import java.io.FileOutputStream; -import java.io.IOException; -import java.io.InputStream; -import java.io.OutputStream; -import java.nio.file.Files; -import java.nio.file.Path; -import java.nio.file.Paths; -import java.nio.file.attribute.BasicFileAttributes; -import java.text.SimpleDateFormat; -import java.util.Arrays; -import java.util.LinkedHashMap; -import java.util.List; -import java.util.Map; - -import org.apache.commons.io.FilenameUtils; -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; -import org.apache.logging.log4j.Marker; - -import com.ibm.nosql.json.api.BasicDBObject; -import com.ibm.nosql.json.api.DB; -import com.ibm.nosql.json.api.DBCollection; -import com.ibm.nosql.json.api.QueryBuilder; - -import user.commons.IEntityBase; -import user.commons.log4j2.marker.MediaCubeMarker; -import user.commons.nosql.NoSQLUtils; -import user.jobengine.db.IItemManager; -import user.jobengine.db.MediaFile; -import user.jobengine.db.MediaFileDAO; -import user.mediacube.metadata.interfaces.IMetadata; -import user.mediacube.metadata.interfaces.IMetadataProvider; -import user.mediacube.metadata.interfaces.IMetadataProviderFactory; -import user.mediacube.metadata.interfaces.MetadataProviderType; -import user.mediacube.metadata.interfaces.MetadataType; -import user.mediacube.metadata.interfaces.PlanAirMetadataListOptions; - -public class HSMMigrateStep extends JobStep { - private static final Logger logger = LogManager.getLogger(); - private static final String MXFEXT = ".mxf"; - private static final String MOVEXT = ".mov"; - private Marker marker = null; - private IMetadataProvider hsmProvider; - private Map tapeContents = new LinkedHashMap<>(); - private DBCollection excludes; - private DBCollection fileHistory; - private DBCollection volumeHistory; - private DB db; - private IMetadataProvider planairProvider; - private SimpleDateFormat df = new SimpleDateFormat("yyyy.MM.dd"); - - private void cleanupHistory() { - if (db == null) - db = NoSQLUtils.getNoSQLDB(); - if (fileHistory == null) - fileHistory = db.getCollection("hsm_migrate_file_history"); - if (volumeHistory == null) - volumeHistory = db.getCollection("hsm_migrate_volume_history"); - fileHistory.drop(); - volumeHistory.drop(); - } - - private BasicDBObject createMetadata(String volumeName, String fileName) throws Exception { - - Path filePath = Paths.get(fileName); - String mediaHouseId = FilenameUtils.removeExtension(filePath.getFileName().toString()); - BasicDBObject result = null; - try { - result = getPlanAirMetadata(mediaHouseId); - } catch (Exception e) { - - logger.error("PlanAir metadata error", e); - //nem latja a drivert pl. - //throw e; - } - - if (result == null) { - result = new BasicDBObject(); - BasicFileAttributes attr = Files.readAttributes(filePath, BasicFileAttributes.class); - result.put("itemHouseId", df.format(attr.lastModifiedTime().toMillis())); - result.put("itemTitle", filePath.getParent().toString()); - result.put("mediaHouseId", mediaHouseId); - result.put("mediaTitle", fileName); - result.put("mediaDescription", volumeName); - result.put("mediaType", "Generic"); - } - result.put("userName", "mediacube"); - return result; - } - - @SuppressWarnings("serial") - @StepEntry - public Object[] execute(String sourceLocation, String targetLocation) throws Exception { - marker = getJobRuntime().getSessionMarker(); - //remove from prod - //cleanupHistory(); - hsmProvider = getMetadataProvider(MetadataProviderType.HSM); - if (hsmProvider == null) - throw new NullPointerException("No HSM metadata provider available"); - planairProvider = getMetadataProvider(MetadataProviderType.PLANAIR); - if (planairProvider == null) - throw new NullPointerException("No PLANAIR metadata provider available"); - Path targetPath = Paths.get(targetLocation); - try { - List volumes = hsmProvider.list(new BasicDBObject()); - for (IMetadata volume : volumes) { - String volumeName = volume.getTitle(); - - BasicDBObject historyResult = queryVolumeHistory(volumeName); - if (historyResult != null) { - logger.info(marker, "A kazetta már feldolgozásra került: {}", volumeName); - continue; - } - logger.info(marker, "A kazetta feldolgozása elindul: {}", volumeName); - List contents = getContents(volumeName); - int p = 0; - boolean oneSuccess = false; - boolean hasError = false; - for (IMetadata content : contents) { - BasicDBObject c = content.asJSON(); - String hsmFileName = c.getString("fileName"); - long contentFileSize = NoSQLUtils.asLong(c, "fileSize"); - if (!tapeContents.containsKey(hsmFileName)) { - tapeContents.put(hsmFileName, c); - Path sourceFilePath = Paths.get(sourceLocation, hsmFileName); - if (contentFileSize < Files.getFileStore(targetPath).getUsableSpace()) { - Path targetFilePath = Paths.get(targetLocation, sourceFilePath.getFileName().toString()); - - try { - if (processHSMFile(volumeName, hsmFileName, sourceFilePath, targetFilePath)) { - oneSuccess = true; - } - } catch (Exception e) { - hasError = true; - } - } - } - setProgress(p++ * 100 / contents.size()); - - if (getJobRuntime().isWaitingCancel()) { - logger.info("Job canceled by user"); - //ne mentsuk a szalagot meg - return null; - } - - } - - if (oneSuccess && !hasError) { - saveVolumeHistory(volumeName); - String subject = "A kazetta eltávolítható a HSM rendszerből: " + volumeName; - logger.info(new MediaCubeMarker() { - { - setSubject(subject); - } - }, subject); - } - - //ha mar sikerult valamit archivalni kilepunk - if (oneSuccess) - break; - } - } catch (Exception e) { - logger.error(marker, "Hiba a migráció során. A rendszer hibaüzenete: {}", e.getMessage()); - throw e; - } - - return null; - } - - private List getContents(String volumeName) throws Exception { - List contents = null; - contents = hsmProvider.list(new BasicDBObject("volumeName", volumeName)); - return contents; - } - - protected IMetadataProvider getMetadataProvider(MetadataProviderType type) { - IMetadataProviderFactory factory = getService(IMetadataProviderFactory.class); - if (factory == null) - return null; - return factory.getProvider(type); - } - - private BasicDBObject getPlanAirMetadata(String mediaHouseId) throws Exception { - PlanAirMetadataListOptions opt = new PlanAirMetadataListOptions(); - opt.setSearch(mediaHouseId); - - List result = null; - opt.setType(MetadataType.Material); - result = planairProvider.list(opt); - if (result.size() != 0) - return result.get(0).asJSON(); - - opt.setType(MetadataType.Promo); - result = planairProvider.list(opt); - if (result.size() != 0) - return result.get(0).asJSON(); - - opt.setType(MetadataType.AD); - result = planairProvider.list(opt); - if (result.size() != 0) - return result.get(0).asJSON(); - - return null; - } - - //true if need copy - public boolean prepareCopy(String hsmFileName, Path source, Path target) throws IOException { - boolean result = true; - BasicDBObject excludeResult = queryExclude(hsmFileName); - if (excludeResult != null) { - logger.warn(marker, "Kivételként megjelölt: {}", hsmFileName); - return false; - } - - File sourceFile = source.toFile(); - File targetFile = target.toFile(); - - if (!sourceFile.exists()) { - logger.warn(marker, "A forrás nem elérhető: {}", source); - return false; - } - - // BasicDBObject historyResult = queryFileHistory(contentFileName); - // if (historyResult != null) - // return false; - - // if (!mediaFile.getName().toLowerCase().endsWith(MOVEXT.toLowerCase()) && !mediaFile.getName().toLowerCase().endsWith(MXFEXT.toLowerCase())) - // return; - // logger.info("Start copy from {} to {}", sourceFilePath, targetFilePath); - - boolean targetExists = targetFile.exists(); - - long targetLength = targetFile.length(); - long sourceLength = sourceFile.length(); - - if (targetLength == sourceLength) { - logger.warn(marker, "A fájl már fel van dolgozva: {}, {} -> {}", source, sourceLength, targetLength); - return false; - } - - if (targetLength > sourceLength) { - logger.warn(marker, "A célfájl nagyobb, törlöm: {}", target); - Files.delete(target); - targetLength = 0; - targetExists = false; - } - - if (targetExists) { - logger.warn(marker, "A fájl már létezik, a másolás folytatódik: {}, {} -> {}", target, sourceLength, targetLength); - } else - logger.warn(marker, "Migrálás: {}, {} -> {}", source, sourceLength, targetLength); - - return result; - } - - private boolean processHSMFile(String volumeName, String hsmFileName, Path sourceFilePath, Path targetFilePath) throws Exception { - int repeat = 4; - boolean successCopy = false; - - IItemManager manager = getManager(); - MediaFileDAO mfDAO = (MediaFileDAO) manager.getBaseDAO(MediaFile.class); - List mediaFiles = mfDAO.getByHouseId(sourceFilePath.getFileName().toString()); - if (mediaFiles != null && mediaFiles.size() > 0) { - logger.warn(marker, "Már archivált: {}", hsmFileName); - return false; - } - - if (prepareCopy(hsmFileName, sourceFilePath, targetFilePath)) { - while (repeat > 0) { - try { - resumeableCopy(sourceFilePath, targetFilePath); - repeat = 0; - successCopy = true; - } catch (Exception e) { - if (Files.exists(targetFilePath) && targetFilePath.toFile().length() == 0) - Files.delete(targetFilePath); - //logger.warn(marker, "Hiba a másolás során: {} ({})", sourceFilePath, e.getMessage()); - repeat--; - } - } - } - String metadataFileName = sourceFilePath.getFileName() + EscortFiles.DOT_JSON; - Path metadataPath = Paths.get(targetFilePath.getParent().toString(), EscortFiles.STATUSFOLDER, metadataFileName); - boolean createMetadata = Files.exists(targetFilePath) && !Files.exists(metadataPath); - - if (successCopy || createMetadata) { - String metadata = null; - try { - metadata = createMetadata(volumeName, hsmFileName).toPrettyString(""); - EscortFiles.createMetadata(targetFilePath.getParent().toString(), targetFilePath.getFileName().toString(), metadata); - //saveFileHistory(contentFileName); - - } catch (Exception e) { - logger.error(marker, "Metadata error", e); - return false; - } - return true; - } - - logger.error(marker, "A fájl másolása nem lehetséges: {}", sourceFilePath); - return false; - } - - public BasicDBObject queryExclude(String fileName) { - if (db == null) - db = NoSQLUtils.getNoSQLDB(); - if (excludes == null) - excludes = db.getCollection("hsm_migrate_exclude"); - Path filePath = Paths.get(fileName); - String pureFileName = FilenameUtils.removeExtension(filePath.getFileName().toString()); - QueryBuilder qb = QueryBuilder.start("name").in(Arrays.asList(fileName, pureFileName)); - BasicDBObject exceptionResult = NoSQLUtils.asSingle(excludes.find(qb.get())); - return exceptionResult; - } - - public BasicDBObject queryFileHistory(String fileName) { - if (db == null) - db = NoSQLUtils.getNoSQLDB(); - if (fileHistory == null) - fileHistory = db.getCollection("hsm_migrate_file_history"); - QueryBuilder qb = QueryBuilder.start("name").in(Arrays.asList(fileName)); - BasicDBObject historyResult = NoSQLUtils.asSingle(fileHistory.find(qb.get())); - return historyResult; - } - - public BasicDBObject queryVolumeHistory(String volumeName) { - if (db == null) - db = NoSQLUtils.getNoSQLDB(); - if (volumeHistory == null) - volumeHistory = db.getCollection("hsm_migrate_volume_history"); - QueryBuilder qb = QueryBuilder.start("name").in(Arrays.asList(volumeName)); - BasicDBObject historyResult = NoSQLUtils.asSingle(volumeHistory.find(qb.get())); - return historyResult; - } - - public void resumeableCopy(Path source, Path target) throws Exception { - File sourceFile = source.toFile(); - File targetFile = target.toFile(); - boolean targetExists = targetFile.exists(); - - long targetLength = targetFile.length(); - long sourceLength = sourceFile.length(); - - try (InputStream in = new BufferedInputStream(new FileInputStream(sourceFile)); - OutputStream out = new BufferedOutputStream(new FileOutputStream(targetFile, targetExists))) { - - byte[] buffer = new byte[128 * 1024]; - int lengthRead; - - if (targetExists) - in.skip(targetLength); - - while ((lengthRead = in.read(buffer)) > 0) { - out.write(buffer, 0, lengthRead); - out.flush(); - targetLength = targetFile.length(); - if (targetLength > sourceLength) { - throw new Exception("Hiba! A fájl túl nagy lett."); - } - - if (getJobRuntime().isWaitingCancel()) { - break; - } - } - - targetLength = targetFile.length(); - sourceLength = sourceFile.length(); - if (targetLength != sourceLength) { - throw new Exception("Hiba! A fájl mérete nem egyezik."); - } - } - } - - private void saveFileHistory(String fileName) { - if (db == null) - db = NoSQLUtils.getNoSQLDB(); - if (fileHistory == null) - fileHistory = db.getCollection("hsm_migrate_file_history"); - BasicDBObject item = new BasicDBObject(); - item.put("name", fileName); - fileHistory.save(item); - - } - - private void saveVolumeHistory(String volumeName) { - if (db == null) - db = NoSQLUtils.getNoSQLDB(); - if (volumeHistory == null) - volumeHistory = db.getCollection("hsm_migrate_volume_history"); - BasicDBObject item = new BasicDBObject(); - item.put("name", volumeName); - volumeHistory.save(item); - - } -} diff --git a/server/-product/production/OMAR/jobs/steps/IntegrationTestStep.java b/server/-product/production/OMAR/jobs/steps/IntegrationTestStep.java deleted file mode 100644 index 081d02d8..00000000 --- a/server/-product/production/OMAR/jobs/steps/IntegrationTestStep.java +++ /dev/null @@ -1,128 +0,0 @@ -package user.jobengine.server.steps; - -import java.nio.file.Files; -import java.nio.file.Path; -import java.nio.file.Paths; -import java.util.concurrent.CountDownLatch; - -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; - -import user.commons.ListUtils; -import user.jobengine.db.ArchivedMedia; -import user.jobengine.db.Item; -import user.jobengine.db.Media; -import user.jobengine.server.steps.shared.ItemManagerExtensions; - -public class IntegrationTestStep extends JobStep { - private static final Logger logger = LogManager.getLogger(); - private ArchivedMedia archivedMedia; - - private void _00_test_cancelable() throws Exception { - CountDownLatch finishLatch = new CountDownLatch(1); - getEngine().submit(null, e -> { - if (e.isRuntimeTerminated()) - finishLatch.countDown(); - }, "cancelable.xml", "Test cancelable", ListUtils.asMap("param", 1)); - finishLatch.await(); - logger.info("_00_test_cancelable SUCCESS"); - } - - private void _01_test_retrieve_ondemand() throws Exception { - archivedMedia = new ArchivedMedia(); - Media media = getManager().getMedia(15285); - Item item = getManager().getItem(media.getItemId()); - archivedMedia.setItem(item); - archivedMedia.setMedia(media); - - CountDownLatch finishLatch = new CountDownLatch(1); - getEngine().submit(null, e -> { - if (e.isRuntimeTerminated()) - finishLatch.countDown(); - - }, "retrieve-ondemand.xml", "Test retrieve-ondemand", - ListUtils.asMap("globalRetrievePath", "file://10.11.1.100", "localRetrievePath", "/mediacube/data", "materialOutputFolder", "/", - "promoOutputFolder", "/", "advertisementOutputFolder", "/", "octopusOutputFolder", "/", "genericOutputFolder", "/", - "onlineOutputFolder", "/", "killDateDays", -1, "nexioAgency", "ARCHIVE_RESTORE", "nexioPort", 2098, "nexioUserName", "administrator", - "nexioPassword", "system", "archivedMedia", archivedMedia, "successRecipient", "vasary@elgekko.net", "houseId", - archivedMedia.getMedia().getHouseId(), "targetPathType", "0")); - - finishLatch.await(); - Path output = Paths.get("/mediacube/data", archivedMedia.getMedia().getHouseId(), - archivedMedia.getMedia().getHouseId() + "-ARCH-" + archivedMedia.getMedia().getMediaFileRealName()); - if (!Files.exists(output)) - throw new Exception("File does not exist: " + output); - - logger.info("_01_test_retrieve_ondemand SUCCESS"); - } - - private void _02_test_archive_material() throws Exception { - Path input = Paths.get("/mediacube/data", archivedMedia.getMedia().getHouseId(), - archivedMedia.getMedia().getHouseId() + "-ARCH-" + archivedMedia.getMedia().getMediaFileRealName()); - - String outputName = "IntegrationTest_" + System.currentTimeMillis() + ".mxf"; - Path output = Paths.get(input.getParent().toString(), outputName); - Files.copy(input, output); - if (!Files.exists(output)) - throw new Exception("File does not exist: " + output); - - ArchiveItem archiveItem = new ArchiveItem(); - archiveItem.setMediaFile(output.toString()); - archiveItem.setItemHouseId(outputName); - archiveItem.setItemTitle(outputName); - archiveItem.setMediaHouseId(outputName); - archiveItem.setMediaTitle(outputName); - archiveItem.setMediaType("Generic"); - - CountDownLatch finishLatch = new CountDownLatch(1); - getEngine().submit(null, e -> { - if (e.isRuntimeTerminated()) - finishLatch.countDown(); - }, "archive-material.xml", "Test archive-material", ListUtils.asMap("archiveItem", archiveItem, "killDateDays", 0)); - finishLatch.await(); - - if (!ItemManagerExtensions.isArchived(getManager(), output)) - throw new Exception("File not archived: " + output); - - Files.delete(output); - - logger.info("_02_test_archive_material SUCCESS"); - } - - private void _03_test_delete_materials() throws Exception { - Path output = Paths.get("/mediacube/data", archivedMedia.getMedia().getHouseId(), - archivedMedia.getMedia().getHouseId() + "-ARCH-" + archivedMedia.getMedia().getMediaFileRealName()); - String source = output.getParent().toString(); - - CountDownLatch finishLatch = new CountDownLatch(1); - getEngine().submit(null, e -> { - if (e.isRuntimeTerminated()) - finishLatch.countDown(); - }, "delete-materials.xml", "Test delete-materials", ListUtils.asMap("sourcePath", source, "skipArchiveCheck", true)); - finishLatch.await(); - - if (Files.exists(output)) - throw new Exception("File exists: " + output); - } - - @StepEntry - public Object[] execute() throws Exception { - getJobRuntime().setDescription("_00_test_cancelable"); - _00_test_cancelable(); - setProgress(25); - - getJobRuntime().setDescription("_01_test_retrieve_ondemand"); - _01_test_retrieve_ondemand(); - setProgress(50); - - getJobRuntime().setDescription("_02_test_archive_material"); - _02_test_archive_material(); - setProgress(75); - - getJobRuntime().setDescription("_03_test_delete_materials"); - _03_test_delete_materials(); - setProgress(100); - return null; - } - -} diff --git a/server/-product/production/OMAR/jobs/steps/MXFCutterStep.java b/server/-product/production/OMAR/jobs/steps/MXFCutterStep.java deleted file mode 100644 index 58a5944d..00000000 --- a/server/-product/production/OMAR/jobs/steps/MXFCutterStep.java +++ /dev/null @@ -1,129 +0,0 @@ -package user.jobengine.server.steps; - -import org.apache.commons.lang.StringUtils; -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; -import org.apache.logging.log4j.Marker; - -import user.commons.RemoteFile; -import user.commons.StoreUri; -import user.commons.remotestore.IProgressEventListener; -import user.commons.remotestore.ProgressEvent; -import user.commons.remotestore.RemoteStoreProtocol; -//import user.jobengine.db.Media; -import user.jobengine.db.ArchivedMedia; -import user.jobengine.db.IItemManager; -import user.jobengine.db.Store; -import user.jobengine.server.IJobEngine; -import user.jobengine.server.IJobRuntime; - -public class MXFCutterStep extends JobStep { - private static final String TARGETNAMEPATTERN = "-ARCH-%s"; - private static final Logger logger = LogManager.getLogger(); - private IItemManager manager; - private StoreUri tempTargetUri; - private StoreUri tempSourceUri; - private String sourceFileName; - private Marker marker; - private int nexioPort; - private String nexioUserName, nexioPassword; - private String nexioHost; - - protected void checkTargetPath(String targetPath) { - if (StringUtils.isBlank(targetPath)) { - logger.error(marker, "A folyamat 'targetPath' bemeneti paramétere üres."); - throw new NullPointerException("System is not configured properly, 'targetPath' input parameter missing."); - } - } - - protected StoreUri createTargetUri(IItemManager manager, String targetPath) throws NullPointerException { - StoreUri result = null; - result = manager.createStoreUri(RemoteStoreProtocol.FTP, nexioHost); - result.setPortNumber(nexioPort); - result.setUserName(nexioUserName); - result.setPassword(nexioPassword); - - return result; - } - - @StepEntry - public Object[] execute(ArchivedMedia archivedMedia, String targetPath, String houseId, String successRecipient, int killDateDays, boolean useNexioTarget, - String nexioAgency, int nexioPort, String nexioUserName, String nexioPassword, IJobEngine jobEngine, IJobRuntime jobRuntime) throws Exception { - this.nexioPort = nexioPort; - this.nexioUserName = nexioUserName; - this.nexioPassword = nexioPassword; - nexioHost = System.getProperty("nexio.host"); - marker = jobRuntime.getSessionMarker(); - - if (useNexioTarget && archivedMedia.getTcIn() != null && archivedMedia.getTcOut() != null) { - setAndCheck(archivedMedia, houseId, targetPath, useNexioTarget, jobEngine); - - final IJobRuntime runtime = jobRuntime; - sourceFileName = houseId + TARGETNAMEPATTERN; - tempSourceUri.addProgressListener(new IProgressEventListener() { - @Override - public void progressChanged(ProgressEvent evt) { - runtime.incrementProgress(evt.getProgress()); - } - }); - - RemoteFile result = tempSourceUri.transferFrom(tempTargetUri, sourceFileName, sourceFileName); - - EscortFiles.setNEXIOKillDate(killDateDays, houseId, nexioAgency, tempTargetUri); - - logger.info("A {} videó kivágva {}s - {}s", sourceFileName, archivedMedia.getTcIn(), archivedMedia.getTcOut()); - } - - return null; - } - - // private String getSourceFileName(ArchivedMedia archivedMedia, Store - // store) { - // List mediaFiles = archivedMedia.getMedia().getMediaFiles(); - // if (mediaFiles == null) - // return null; - // for (MediaFile mediaFile : mediaFiles) { - // if (mediaFile.getStore().getId() == store.getId()) - // return mediaFile.getRelativePath(); - // } - // return null; - // } - - private void setAndCheck(ArchivedMedia archivedMedia, String houseId, String targetPath, boolean useNexioTarget, IJobEngine jobEngine) { - if (jobEngine == null) { - logger.error(marker, "A folyamatkezelő réteg nem elérhető."); - throw new NullPointerException("Internal error, missing JobEngine reference."); - } - manager = jobEngine.getItemManager(); - if (manager == null) { - logger.error(marker, "Az adatbáziskezelő réteg nem elérhető."); - throw new NullPointerException("Internal error, missing ItemManager reference."); - } - if (archivedMedia == null) { - logger.error(marker, "A folyamat 'mediaCubeMedia' bemeneti paramétere üres."); - throw new NullPointerException("System is not configured properly, 'mediaCubeMedia' input parameter missing."); - } - checkTargetPath(targetPath); - if (StringUtils.isBlank(houseId)) { - logger.error(marker, "A folyamat 'houseId' bemeneti paramétere üres."); - throw new NullPointerException("System is not configured properly, 'houseId' input parameter missing."); - } - Store tsmStore = manager.getSystemStore(false); - if (tsmStore == null) { - logger.error(marker, "A TSM rendszer beállítás nem elérhető."); - throw new NullPointerException("System is not configured properly, missing TSM Store."); - } - - tempSourceUri = manager.createStoreUri(RemoteStoreProtocol.LOCAL, targetPath); - if (tempSourceUri == null) { - logger.error(marker, "A TSM rendszer beállítás paraméterei nem elérhetőek."); - throw new NullPointerException("System is not configured properly, missing TSM StoreUri."); - } - tempTargetUri = createTargetUri(manager, targetPath); - // sourceFileName = getSourceFileName(archivedMedia, tsmStore); - if (sourceFileName == null) { - logger.error(marker, "Adatbázis bejegyzés hiba, a visszatöltendő fájl neve nem található."); - throw new NullPointerException("Database error, missing MediaFile 'relativePath'."); - } - } -} diff --git a/server/-product/production/OMAR/jobs/steps/MediaToolStep.java b/server/-product/production/OMAR/jobs/steps/MediaToolStep.java deleted file mode 100644 index ecc0ba2d..00000000 --- a/server/-product/production/OMAR/jobs/steps/MediaToolStep.java +++ /dev/null @@ -1,29 +0,0 @@ -package user.jobengine.server.steps; - -import java.nio.file.Path; -import java.nio.file.Paths; - -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; - -import user.commons.mediaarea.MediaArea; -import user.jobengine.db.Media; - -public class MediaToolStep extends JobStep { - private static final Logger logger = LogManager.getLogger(); - - @StepEntry - public Object[] execute(ArchiveItem archiveItem, Media mediaCubeMedia) throws Exception { - Path filePath = Paths.get(archiveItem.getMediaFile()); - MediaArea ma = new MediaArea(filePath); - ma.process(); - long frames = ma.getFrameCount(); - if (frames > 0) { - logger.info("Media {} length is {}", filePath, frames); - mediaCubeMedia.setLength(frames); - getManager().modify(mediaCubeMedia); - } - return null; - } - -} diff --git a/server/-product/production/OMAR/jobs/steps/MetadataTransformStep.java b/server/-product/production/OMAR/jobs/steps/MetadataTransformStep.java deleted file mode 100644 index 9bf16e96..00000000 --- a/server/-product/production/OMAR/jobs/steps/MetadataTransformStep.java +++ /dev/null @@ -1,168 +0,0 @@ -package user.jobengine.server.steps; - -import java.io.File; -import java.nio.file.Files; -import java.nio.file.Path; -import java.nio.file.Paths; -import java.nio.file.attribute.FileAttribute; -import java.nio.file.attribute.PosixFilePermission; -import java.nio.file.attribute.PosixFilePermissions; -import java.util.Date; -import java.util.Set; - -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; -import org.apache.logging.log4j.Marker; - -import com.ibm.nosql.json.api.BasicDBList; - -import user.jobengine.db.Item; -import user.jobengine.db.ItemManager; -import user.jobengine.db.ItemType; -import user.jobengine.db.Media; -import user.jobengine.server.IJobEngine; -import user.jobengine.server.IJobRuntime; - -/** - * Itemek es mediak krealasa az ArchiveItem objektum alapjan. - * - * @author robi - */ -public class MetadataTransformStep extends JobStep { - private static final String CONFLICT = ".CONFLICT"; - private static final Logger logger = LogManager.getLogger(); - private static final String ITEM_MANAGER_IS_NULL = "ItemManager is null"; - public static final String DEFAULT_MEDIATYPE = "Generic"; - private ItemManager itemManager; - - private Marker marker; - - private void addTags(ArchiveItem archiveItem, Media mediaCubeMedia) { - BasicDBList tags = archiveItem.getTags(); - if (tags != null) { - for (Object tag : tags) { - - try { - String tagText = String.valueOf(tag); - itemManager.addMediaTag(tagText, mediaCubeMedia.getId()); - System.out.println(); - - } catch (Exception e) { - logger.catching(e); - } - } - } - } - - private void checkDuplicates(ArchiveItem archiveItem, String sourceFileName) throws Exception { - if (itemManager.isMediaFileExists(sourceFileName)) { - try { - Path sourcePath = Paths.get(archiveItem.getMediaFile()); - Path parent = sourcePath.getParent(); - Path conflictPath = Paths.get(parent.toString(), CONFLICT); - File folder = conflictPath.toFile(); - if (!folder.exists() || !folder.isDirectory()) { - Set perms = PosixFilePermissions.fromString("rwxrwxrwx"); - FileAttribute> attr = PosixFilePermissions.asFileAttribute(perms); - try { - Files.createDirectories(conflictPath, attr); - } catch (Exception e) { - try { - Files.createDirectory(conflictPath); - } catch (Exception e1) { - logger.catching(e); - throw e; - } - } - } - - Files.move(sourcePath, Paths.get(conflictPath.toString(), sourceFileName + (new Date()).getTime())); - } catch (Exception e1) { - logger.catching(e1); - logger.error(marker, "Hiba az '{}' állomány mappába másolásakor. A rendszer üzenete: {}", CONFLICT, e1.getMessage()); - } - throw new Exception("Az '" + sourceFileName + "' állomány már megtalálható az archívumban, archiválása nem lehetséges."); - } - } - - private Item createItem(ArchiveItem archiveItem) { - Item mediaCubeItem = getExistingItem(archiveItem.getItemHouseId(), archiveItem.getItemTitle()); - if (mediaCubeItem == null) - mediaCubeItem = itemManager.createItem(DEFAULT_MEDIATYPE, archiveItem.getItemTitle(), archiveItem.getItemDescription(), - archiveItem.getItemHouseId()); - return mediaCubeItem; - } - - private Media createMedia(ArchiveItem archiveItem, Item mediaCubeItem, String mediaType) { - Media mediaCubeMedia; - mediaCubeMedia = itemManager.createMedia(mediaType, archiveItem.getMediaTitle(), archiveItem.getMediaDescription(), archiveItem.getMediaHouseId()); - mediaCubeMedia.setLength(archiveItem.getDuration()); - mediaCubeItem.appendMedia(mediaCubeMedia); - - return mediaCubeMedia; - } - - @StepEntry - public Object[] execute(ArchiveItem archiveItem, IJobEngine jobEngine, IJobRuntime jobRuntime) throws Exception { - marker = jobRuntime.getSessionMarker(); - Media mediaCubeMedia = null; - itemManager = (ItemManager) jobEngine.getItemManager(); - if (itemManager == null) - throw new NullPointerException(ITEM_MANAGER_IS_NULL); - try { - File sourceMediaFile = new File(archiveItem.getMediaFile()); - String sourceFileName = sourceMediaFile.getName(); - checkDuplicates(archiveItem, sourceFileName); - Item mediaCubeItem = createItem(archiveItem); - jobRuntime.incrementProgress(50); - String mediaType = getCreateType(archiveItem); - mediaCubeMedia = createMedia(archiveItem, mediaCubeItem, mediaType); - //ha itemid 0 akkor merge, egyebkent media insert - - if (mediaCubeItem.getId() == 0) - itemManager.mergeItemStructure(mediaCubeItem); - else { - mediaCubeMedia.setItemId(mediaCubeItem.getId()); - mediaCubeMedia.add(); - } - - addTags(archiveItem, mediaCubeMedia); - - } catch (Exception e) { - logger.catching(e); - String fileName = new File(archiveItem.getMediaFile()).getName(); - logger.error(marker, "Az '{}' állomány nem archiválható, mert a metaadat transzformáció sikertelen. A rendszer üzenete: {}", fileName, - e.getMessage()); - if (!archiveItem.removeCatchedFile()) - logger.error(marker, "Az '{}' állomány .catched jelző állománya nem törölhető.", fileName); - throw e; - } finally { - jobRuntime.incrementProgress(100); - } - return new Object[] { mediaCubeMedia }; - } - - private String getCreateType(ArchiveItem archiveItem) { - String mediaType = archiveItem.getMediaType(); - if (mediaType == null || mediaType.length() == 0) - mediaType = DEFAULT_MEDIATYPE; - else { - ItemType mediaItemType = itemManager.getItemType(mediaType); - if (mediaItemType == null) - itemManager.createItemType(mediaType, mediaType).add(); - } - return mediaType; - } - - private Item getExistingItem(String itemHouseId, String itemTitle) { - Item[] result = new Item[] { null }; - String sql = String.format("select id from item where houseid='%s' and title='%s'", itemHouseId, itemTitle); - itemManager.executeQuery(sql, rs -> { - long id = rs.getLong("id"); - result[0] = itemManager.getItem(id); - return true; - }, null); - return result[0]; - } - -} diff --git a/server/-product/production/OMAR/jobs/steps/MetadataUpdater.java b/server/-product/production/OMAR/jobs/steps/MetadataUpdater.java deleted file mode 100644 index a28abfa2..00000000 --- a/server/-product/production/OMAR/jobs/steps/MetadataUpdater.java +++ /dev/null @@ -1,297 +0,0 @@ -package user.jobengine.server.steps; - -import java.io.IOException; -import java.nio.file.Path; -import java.nio.file.Paths; -import java.sql.ResultSet; -import java.text.SimpleDateFormat; -import java.util.Date; -import java.util.LinkedHashSet; -import java.util.List; -import java.util.Set; - -import org.apache.commons.io.FileUtils; -import org.apache.ibatis.jdbc.SQL; -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; -import org.apache.logging.log4j.Marker; -import org.apache.logging.log4j.MarkerManager; - -import com.ibm.nosql.json.api.BasicDBObject; - -import user.jobengine.db.IItemManager; -import user.jobengine.server.steps.shared.MetadataTypeDetector; -import user.mediacube.metadata.interfaces.IMetadata; -import user.mediacube.metadata.interfaces.IMetadataProvider; -import user.mediacube.metadata.interfaces.IMetadataProviderFactory; -import user.mediacube.metadata.interfaces.MetadataProviderType; -import user.mediacube.metadata.interfaces.MetadataType; -import user.mediacube.metadata.interfaces.PlanAirMetadataListOptions; - -public class MetadataUpdater extends JobStep { - private static final String EMPTY = ""; - private static final String DOT = "."; - private static final Logger logger = LogManager.getLogger(MetadataUpdater.class); - private Marker csvMarker = MarkerManager.getMarker("METADATA-UPDATER-CSV"); - - private static final String ITEM_TITLE = "itemTitle"; - private static final String ITEM_HOUSEID = "itemHouseId"; - private static final String MEDIA_HOUSEID = "mediaHouseId"; - private static final String MEDIA_TITLE = "mediaTitle"; - private static final String MEDIA_DESCRIPTION = "mediaDescription"; - private static final String MEDIA_TYPE = "mediaType"; - private static final String MEDIAFILE_HOUSEID = "mediaFileHouseId"; - - private SimpleDateFormat df = new SimpleDateFormat("yyyy.MM.dd HH:mm:ss"); - private Set includeList; - - public String createQuery() { - SQL isql = new SQL(); - isql.SELECT("id"); - isql.FROM("STORE"); - isql.WHERE("name = 'TSM'"); - - SQL sql = new SQL(); - sql.SELECT("i.id AS itemId"); - sql.SELECT("i.houseid AS itemHouseId"); - sql.SELECT("i.title AS itemTitle"); - //sql.SELECT("i.description AS itemdescription"); - sql.SELECT("m.id AS mediaId"); - sql.SELECT("m.houseid AS mediaHouseId"); - sql.SELECT("m.title AS mediaTitle"); - sql.SELECT("m.description AS mediaDescription"); - sql.SELECT("mf.houseid AS mediaFileHouseId"); - sql.FROM("MEDIAFILE mf"); - sql.LEFT_OUTER_JOIN("MEDIA m ON (m.id = mf.mediaid)"); - sql.LEFT_OUTER_JOIN("ITEM i ON (i.id = m.itemid)"); - sql.WHERE(String.format("mf.storeid = (%s)", isql.toString())); - return sql.toString(); - } - - public String createCountQuery() { - SQL sql = new SQL(); - sql.SELECT("COUNT(*) as count"); - sql.FROM("MEDIA"); - return sql.toString(); - } - - @StepEntry - public Object[] execute() throws Exception { - try { - - String location = "/opt/test-mediacube/file_list_original.txt"; - includeList = loadIncludeList(location); - - IItemManager manager = getManager(); - String sql = createCountQuery(); - long[] count = new long[1]; - count[0] = 0; - manager.executeQuery(sql, rs -> { - try { - count[0] = rs.getLong("count"); - logger.info(getSessionMarker(), "Processing rs"); - } catch (Exception e) { - logger.error(getMarker(), e.getMessage()); - } - return true; - }, null); - - logger.info(getSessionMarker(), "Count {}", count[0]); - - long[] current = new long[1]; - current[0] = 0; - - logger.info(csvMarker, - "Date;Name;isProgramById;includeContains;isMetadataEquals;itemHouseId;P itemHouseId;itemHouseIdEquals;itemTitle;P itemTitle;itemTitleEquals;mediaHouseId;" - + "P mediaHouseId;mediaHouseIdEquals;mediaTitle;P mediaTitle;mediaTitleEquals;" - + "mediaDescription;P mediaDescription;mediaDescriptionEquals;"); - - sql = createQuery(); - manager.executeQuery(sql, rs -> { - - if (getJobRuntime().isWaitingCancel()) - return false; - - current[0]++; - processRecord(rs); - int p = ((int) current[0] * 100) / ((int) count[0]); - setProgress(p); - return true; - }, null); - - } catch (Exception e) { - logger.error(getSessionMarker(), e.getMessage()); - } - return null; - } - - private String set(String value) { - return value == null ? EMPTY : value; - } - - private void processRecord(ResultSet rs) { - try { - String itemHouseId = set(rs.getString(ITEM_HOUSEID)); - String itemTitle = set(rs.getString(ITEM_TITLE)); - String mediaHouseId = set(rs.getString(MEDIA_HOUSEID)); - String mediaTitle = set(rs.getString(MEDIA_TITLE)); - String mediaDescription = set(rs.getString(MEDIA_DESCRIPTION)); - String pitemHouseId = EMPTY; - String pitemTitle = EMPTY; - String pmediaHouseId = EMPTY; - String pmediaTitle = EMPTY; - String pmediaDescription = EMPTY; - - String fileName = rs.getString(MEDIAFILE_HOUSEID); - String mediaFileHouseId = fileName; - if (mediaFileHouseId.contains(DOT)) - mediaFileHouseId = mediaFileHouseId.substring(0, mediaFileHouseId.lastIndexOf(DOT)); - - user.jobengine.server.steps.shared.MetadataType metadataType = MetadataTypeDetector - .GuessMetadataType(mediaFileHouseId); - - boolean isProgramById = false; - boolean includeContains = includeList.contains(fileName); - boolean itemHouseIdEquals = false; - boolean itemTitleEquals = false; - boolean mediaHouseIdEquals = false; - boolean mediaTitleEquals = false; - boolean mediaDescriptionEquals = false; - - logger.info(getMarker(), "{} {}", mediaFileHouseId, metadataType); - ArchiveItem archiveItem = getPlanAirMetadata(mediaFileHouseId); - if (archiveItem != null) { - isProgramById = true; - - if (isProgramById) { - - pitemHouseId = set(archiveItem.getItemHouseId()); - itemHouseIdEquals = pitemHouseId.equals(itemHouseId); - - pitemTitle = set(archiveItem.getItemTitle()); - itemTitleEquals = pitemTitle.equals(itemTitle); - - pmediaHouseId = set(archiveItem.getMediaHouseId()); - mediaHouseIdEquals = pmediaHouseId.equals(mediaHouseId); - - pmediaTitle = set(archiveItem.getMediaTitle()); - mediaTitleEquals = pmediaTitle.equals(mediaTitle); - - pmediaDescription = set(archiveItem.getMediaDescription()); - mediaDescriptionEquals = pmediaDescription.equals(mediaDescription); - - boolean isMetadataEquals = itemHouseIdEquals && itemTitleEquals && mediaHouseIdEquals - && mediaTitleEquals && mediaDescriptionEquals; - - Date now = new Date(System.currentTimeMillis()); - logger.info(csvMarker, "{};{};{};{};{};{};{};{};{};{};{};{};{};{};{};{};{};{};{};{};", D(now), - fileName, YN(isProgramById), YN(includeContains), YN(isMetadataEquals), itemHouseId, - pitemHouseId, itemHouseIdEquals, itemTitle, pitemTitle, itemTitleEquals, mediaHouseId, - pmediaHouseId, mediaHouseIdEquals, mediaTitle, pmediaTitle, mediaTitleEquals, - mediaDescription, pmediaDescription, mediaDescriptionEquals); - - logger.info(getSessionMarker(), "Processed {} {}{}{}", fileName, YN(isProgramById), - YN(includeContains), YN(isMetadataEquals)); - } - } - - } catch ( - - Exception e) { - logger.error(getSessionMarker(), e.getMessage()); - } - - } - - private ArchiveItem getPlanAirMetadata(String mediaHouseId) throws Exception { - PlanAirMetadataListOptions opt = new PlanAirMetadataListOptions(); - opt.setSearch(mediaHouseId); - opt.setType(MetadataType.Material); - - BasicDBObject json = null; - List data = null; - - IMetadataProviderFactory factory = getService(IMetadataProviderFactory.class); - if (factory == null) - logger.info(getSessionMarker(), "IMetadataProviderFactory is null"); - - IMetadataProvider planairProvider = factory.getProvider(MetadataProviderType.PLANAIR); - if (planairProvider == null) - logger.info(getSessionMarker(), "IMetadataProvider is null"); - - ArchiveItem result = null; - data = planairProvider.list(opt); - if (data.size() != 0) - json = data.get(0).asJSON(); - else { - opt.setType(MetadataType.Promo); - data = planairProvider.list(opt); - if (data.size() != 0) - json = data.get(0).asJSON(); - else { - opt.setType(MetadataType.AD); - data = planairProvider.list(opt); - if (data.size() != 0) - json = data.get(0).asJSON(); - } - } - if (json != null) { - result = new ArchiveItem(); - result.setItemHouseId(json.getString(ITEM_HOUSEID)); - result.setItemTitle(json.getString(ITEM_TITLE)); - result.setMediaHouseId(json.getString(MEDIA_HOUSEID)); - result.setMediaTitle(json.getString(MEDIA_TITLE)); - result.setMediaDescription(json.getString(MEDIA_DESCRIPTION)); - result.setMediaType(json.getString(MEDIA_TYPE)); - } - return result; - } - - private char YN(boolean value) { - return value ? 'Y' : 'N'; - } - - private String D(Date value) { - return value == null ? EMPTY : df.format(value); - } - - private Set loadIncludeList(String location) throws IOException { - logger.info(getSessionMarker(), "Loading include list {}", location); - Set result = new LinkedHashSet<>(); - - Path path = Paths.get(location); - List lines = FileUtils.readLines(path.toFile()); - - String lastDir = null; - for (String line : lines) { - line = line.trim(); - if (line.startsWith("Directory of")) { - lastDir = line; - lastDir = lastDir.replace("Directory of", EMPTY); - lastDir = lastDir.replace("X:", EMPTY); - lastDir = lastDir.replace("\\", "/"); - lastDir = lastDir.trim(); - } - - if (lastDir != null && lastDir.endsWith(".STATUS")) - continue; - - if (line.startsWith("2") && line.length() > 39) { - String file = line.substring(39).trim(); - - if (file.equals(DOT) || file.equals("..")) - continue; - - String len = line.substring(21, 39).trim(); - if (len.length() != 0) { - String fullpath = "/mnt/POLC/FINISHED_SHOWS" + lastDir + "/" + file; - result.add(Paths.get(fullpath).getFileName().toString()); - } - } - - } - - return result; - } - -} diff --git a/server/-product/production/OMAR/jobs/steps/MoveJpegToIsilonStep.java b/server/-product/production/OMAR/jobs/steps/MoveJpegToIsilonStep.java deleted file mode 100644 index 3dc75b55..00000000 --- a/server/-product/production/OMAR/jobs/steps/MoveJpegToIsilonStep.java +++ /dev/null @@ -1,165 +0,0 @@ -package user.jobengine.server.steps; - -import java.io.File; -import java.io.IOException; -import java.nio.file.FileVisitResult; -import java.nio.file.FileVisitor; -import java.nio.file.Files; -import java.nio.file.Path; -import java.nio.file.Paths; -import java.nio.file.SimpleFileVisitor; -import java.nio.file.StandardCopyOption; -import java.nio.file.attribute.BasicFileAttributes; -import java.text.ParseException; -import java.text.SimpleDateFormat; -import java.util.ArrayList; -import java.util.Date; -import java.util.List; - -import org.apache.commons.io.FilenameUtils; -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; - -import user.commons.StoreUri; -import user.commons.remotestore.RemoteStoreProtocol; - -public class MoveJpegToIsilonStep extends JobStep { - private static final Logger logger = LogManager.getLogger(); - - @StepEntry - public Object[] execute(String sourceUri, String sourceProtocol, String sourceFolder, String targetUri, - String targetProtocol, String targetFolder, String userName, String password, int port) throws Exception { - StoreUri sourceStoreUri = new StoreUri(sourceUri); - if (sourceFolder.endsWith("/")) { - sourceFolder = sourceFolder.substring(0, sourceFolder.length() - 1); - } - - List foundItems = new ArrayList<>(); - sourceStoreUri.setRootPath(sourceFolder); - sourceStoreUri.setPortNumber(port); - sourceStoreUri.setPassword(password); - sourceStoreUri.setUserName(userName); - sourceStoreUri.setProtocol(RemoteStoreProtocol.valueOf(sourceProtocol)); - - FileVisitor visitor = new SimpleFileVisitor() { - String fileNamePattern = "yyyymmdd"; - SimpleDateFormat dateFormatter = new SimpleDateFormat(fileNamePattern); - Date currentDate = new Date(); // initializes with the current date - Date dateFromFileName; - boolean isRootFolder = true; - - @Override - public FileVisitResult preVisitDirectory(Path folder, BasicFileAttributes attrs) throws IOException { - FileVisitResult result = null; - boolean isDateParseable = true; - - if (folder != null) { - String splitter = File.separator.replace("\\", "\\\\"); - String[] parentFolderSegments = folder.toString().split(splitter); - - try { - dateFromFileName = dateFormatter.parse(parentFolderSegments[parentFolderSegments.length - 1]); - - if (!(dateFromFileName.compareTo(currentDate) > 0)) { - if (isRootFolder) { - result = FileVisitResult.CONTINUE; - isRootFolder = false; - } else { - result = FileVisitResult.SKIP_SUBTREE; - } - } else { - result = FileVisitResult.CONTINUE; - } - } catch (ParseException e) { - isDateParseable = false; - logger.info("Illegal argument to parse as date: {}", - parentFolderSegments[parentFolderSegments.length - 1]); - } finally { - isRootFolder = false; - if (!isDateParseable) { - result = FileVisitResult.CONTINUE; - } - } - } - return result; - } - - @Override - public FileVisitResult visitFile(Path path, BasicFileAttributes attrs) throws IOException { - logger.info("path: {}", path.toString()); - try { - String fileExtension = FilenameUtils.getExtension(path.toString()); - - if (fileExtension.equals("jpg") || fileExtension.equals("jpeg")) { - foundItems.add(path.toString()); - StoreUri targetStoreUri = getManager() - .createStoreUri(RemoteStoreProtocol.valueOf(targetProtocol), targetUri); - targetStoreUri.setProtocol(RemoteStoreProtocol.valueOf(targetProtocol)); - targetStoreUri.setRootPath(targetFolder); - - copyFile(path, Paths.get(targetStoreUri.getRootPath())); - } - } catch (Exception e) { - logger.info("Exception: {}", e.getStackTrace()); - } - return FileVisitResult.CONTINUE; - } - - @Override - public FileVisitResult visitFileFailed(Path path, IOException exc) throws IOException { - return FileVisitResult.CONTINUE; - } - }; - - try { - Files.walkFileTree(Paths.get(sourceStoreUri.getRootPath()), visitor); - } catch (IOException e) { - logger.info("Error processing Paths.get(sourceStoreUri.getRootPath()) '{}'. System message: {}", - Paths.get(sourceStoreUri.getRootPath()), e.getStackTrace()); - logger.catching(e); - throw e; - } catch (SecurityException se) { - logger.info("SecurityException: {}", se.getStackTrace()); - } finally { - } - - return new Object[] { foundItems }; - } - - private void copyFile(Path sourceFullPath, Path targetRootPath) { - Path targetPath = Paths.get(targetRootPath.toString(), sourceFullPath.getFileName().toString()); - logger.info(getMarker(), "Root {} exists {}", targetRootPath, Files.exists(targetRootPath)); - - if (Files.exists(targetPath)) { - logger.info(getMarker(), "Skipping {}, target exists", targetPath); - return; - } - - logger.info(getMarker(), "Target {} synchronization required", targetPath); - - try { - Files.copy(sourceFullPath, targetPath, StandardCopyOption.REPLACE_EXISTING); - } catch (Exception e) { - logger.error(getMarker(), "Error synchronize {} to {}. System message: {}", sourceFullPath, targetPath, - e.getMessage()); - } - } - - protected String getPathUntilCurrentFile(String rootFolder, Path fullPath) { - String tempRootfolder = ""; - if (rootFolder.contains("/") && fullPath.toString().contains("\\")) { - tempRootfolder = rootFolder.replace('/', '\\'); - if (!tempRootfolder.endsWith("\\")) { - tempRootfolder = tempRootfolder.concat("\\"); - } - } - if (rootFolder.contains("\\") && fullPath.toString().contains("/")) { - tempRootfolder = rootFolder.replace('\\', '/'); - if (!tempRootfolder.endsWith("/")) { - tempRootfolder = tempRootfolder.concat("/"); - } - } - - return fullPath.toString().replace(rootFolder, "").substring(1); - } -} diff --git a/server/-product/production/OMAR/jobs/steps/OutputPathAndNameSelectorStep.java b/server/-product/production/OMAR/jobs/steps/OutputPathAndNameSelectorStep.java deleted file mode 100644 index f0203ca5..00000000 --- a/server/-product/production/OMAR/jobs/steps/OutputPathAndNameSelectorStep.java +++ /dev/null @@ -1,154 +0,0 @@ -package user.jobengine.server.steps; - -import java.io.IOException; -import java.nio.file.FileVisitResult; -import java.nio.file.FileVisitor; -import java.nio.file.Files; -import java.nio.file.Path; -import java.nio.file.Paths; -import java.nio.file.SimpleFileVisitor; -import java.nio.file.attribute.BasicFileAttributes; - -import org.apache.commons.lang.StringUtils; -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; -import org.apache.logging.log4j.Marker; - -import user.jobengine.db.ArchivedMedia; -import user.jobengine.server.IJobEngine; -import user.jobengine.server.IJobRuntime; -import user.jobengine.server.steps.shared.EscortFiles; -import user.jobengine.server.steps.shared.MetadataType; -import user.jobengine.server.steps.shared.MetadataTypeDetector; - -public class OutputPathAndNameSelectorStep extends JobStep { - - private static final String TARGETNAMEPATTERN = "-ARCH-%s"; - - private static final Logger logger = LogManager.getLogger(); - - private Marker marker; - - private void check(String localRetrievePath, String materialOutputFolder, String promoOutputFolder, String advertisementOutputFolder, - String octopusOutputFolder, String genericOutputFolder, String houseId, String targetPathType) { - if (StringUtils.isBlank(localRetrievePath)) { - logger.error(marker, "A folyamat 'localRetrievePath' bemeneti paramétere üres."); - throw new NullPointerException("System is not configured properly, 'localRetrievePath' input parameter missing."); - } - if (StringUtils.isBlank(materialOutputFolder)) { - logger.error(marker, "A folyamat 'materialOutputFolder' bemeneti paramétere üres."); - throw new NullPointerException("System is not configured properly, 'materialOutputFolder' input parameter missing."); - } - if (StringUtils.isBlank(promoOutputFolder)) { - logger.error(marker, "A folyamat 'promoOutputFolder' bemeneti paramétere üres."); - throw new NullPointerException("System is not configured properly, 'promoOutputFolder' input parameter missing."); - } - if (StringUtils.isBlank(advertisementOutputFolder)) { - logger.error(marker, "A folyamat 'advertisementOutputFolder' bemeneti paramétere üres."); - throw new NullPointerException("System is not configured properly, 'advertisementOutputFolder' input parameter missing."); - } - if (StringUtils.isBlank(octopusOutputFolder)) { - logger.error(marker, "A folyamat 'octopusOutputFolder' bemeneti paramétere üres."); - throw new NullPointerException("System is not configured properly, 'octopusOutputFolder' input parameter missing."); - } - if (StringUtils.isBlank(genericOutputFolder)) { - logger.error(marker, "A folyamat 'genericOutputFolder' bemeneti paramétere üres."); - throw new NullPointerException("System is not configured properly, 'genericOutputFolder' input parameter missing."); - } - if (StringUtils.isBlank(houseId)) { - logger.error(marker, "A folyamat 'houseId' bemeneti paramétere üres."); - throw new NullPointerException("System is not configured properly, 'houseId' input parameter missing."); - } - if (StringUtils.isBlank(targetPathType)) { - logger.error(marker, "A folyamat 'targetPathType' bemeneti paramétere üres."); - throw new NullPointerException("System is not configured properly, 'targetPathType' input parameter missing."); - } - } - - @StepEntry - public Object[] execute(String localRetrievePath, String materialOutputFolder, String promoOutputFolder, String advertisementOutputFolder, - String octopusOutputFolder, String genericOutputFolder, String onlineOutputFolder, String houseId, String targetPathType, - ArchivedMedia archivedMedia, IJobEngine jobEngine, IJobRuntime jobRuntime) throws Exception { - marker = jobRuntime.getSessionMarker(); - check(localRetrievePath, materialOutputFolder, promoOutputFolder, advertisementOutputFolder, octopusOutputFolder, genericOutputFolder, houseId, - targetPathType); - Object[] result = null; - switch (Integer.parseInt(targetPathType)) { - case 0: - String outputFolder = getFolderById(materialOutputFolder, promoOutputFolder, advertisementOutputFolder, octopusOutputFolder, genericOutputFolder, - houseId, archivedMedia); - result = localTargetInit(localRetrievePath, outputFolder, houseId, jobRuntime); - break; - case 1: - result = localTargetInit(localRetrievePath, onlineOutputFolder, houseId, jobRuntime); - break; - case 2: - - if (archivedMedia.getTcIn() != null && archivedMedia.getTcOut() != null) - result = new Object[] { genericOutputFolder, houseId, true }; - else - result = new Object[] { null, houseId, true }; - break; - } - return result; - } - - private String getFolderById(String materialOutputFolder, String promoOutputFolder, String advertisementOutputFolder, String octopusOutputFolder, - String genericOutputFolder, String houseId, ArchivedMedia archivedMedia) throws Exception { - String id = houseId.toUpperCase(); - MetadataType mdType = MetadataTypeDetector.GuessMetadataType(id); - String result = null; - - //a groovy nem latja enumnak, hanem az objektum tulajdonsaganak - switch (mdType.toString()) { - case "OctopusPlaceholder": - case "OctopusStory": - result = octopusOutputFolder; - break; - case "TrafficMaterial": - result = materialOutputFolder; - break; - case "TrafficPromo": - result = promoOutputFolder; - break; - case "TrafficAD": - result = advertisementOutputFolder; - break; - case "Generic": - result = genericOutputFolder; - break; - } - return result; - } - - private String getPossiblePath(String id, Path targetPath) throws IOException { - String[] result = new String[] { targetPath.toString() }; - FileVisitor matcherVisitor = new SimpleFileVisitor() { - @Override - public FileVisitResult preVisitDirectory(Path dir, BasicFileAttributes attrs) throws IOException { - String dirName = dir.getFileName().toString(); - if (dirName.startsWith(id + "-") || dirName.equals(id)) { - result[0] = dir.toString(); - return FileVisitResult.TERMINATE; - } - return FileVisitResult.CONTINUE; - } - }; - Files.walkFileTree(targetPath.getParent(), matcherVisitor); - return result[0]; - } - - private Object[] localTargetInit(String localRetrievePath, String outputFolder, String houseId, IJobRuntime jobRuntime) throws IOException { - String id = houseId.toUpperCase(); - String targetPath = getPossiblePath(id, Paths.get(localRetrievePath, outputFolder, id)).toString(); - String targetNamePattern = houseId + TARGETNAMEPATTERN; - try { - EscortFiles.ensureUNCFolder(Paths.get(targetPath)); - } catch (Exception e) { - logger.error(jobRuntime.getSessionMarker(), "A cél mappa '{}' nem létezik és nem hozható létre. A rendszer hibaüzenete: {}", targetPath, - e.getMessage()); - throw e; - } - return new Object[] { targetPath, targetNamePattern, false }; - } -} diff --git a/server/-product/production/OMAR/jobs/steps/PrepareMediaRestoreStep.java b/server/-product/production/OMAR/jobs/steps/PrepareMediaRestoreStep.java deleted file mode 100644 index 3708aa3e..00000000 --- a/server/-product/production/OMAR/jobs/steps/PrepareMediaRestoreStep.java +++ /dev/null @@ -1,37 +0,0 @@ -package user.jobengine.server.steps; - -import java.util.List; - -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; - -import user.commons.StoreUri; -import user.commons.remotestore.RemoteStoreProtocol; -import user.jobengine.db.Media; -import user.jobengine.db.MediaFile; -import user.jobengine.db.Store; - -public class PrepareMediaRestoreStep extends JobStep { - private static final Logger logger = LogManager.getLogger(); - - @StepEntry - public Object[] execute(Media media, String tempStoreName, String tempStoreProtocol) throws Exception { - StoreUri mediaStoreUri = null; - List mediaFiles = media.getMediaFiles(); - if (mediaFiles.size() > 1) - throw new Exception("Media " + media.getId() + " already has proxy"); - for (MediaFile mediaFile : mediaFiles) { - Store store = mediaFile.getStore(); - if (store.isSystem() && !store.isLowres()) { - mediaStoreUri = store.getStoreUri(RemoteStoreProtocol.TSM); - if (mediaStoreUri != null) - break; - } - } - - StoreUri tempStoreUri = getManager().getStoreUri(tempStoreName, Enum.valueOf(RemoteStoreProtocol.class, tempStoreProtocol)); - - return new Object[] { mediaStoreUri, tempStoreUri, media.getMediaFileRealName() }; - } - -} diff --git a/server/-product/production/OMAR/jobs/steps/PrepareRemoteTranscodeStep.java b/server/-product/production/OMAR/jobs/steps/PrepareRemoteTranscodeStep.java deleted file mode 100644 index 5116b1ef..00000000 --- a/server/-product/production/OMAR/jobs/steps/PrepareRemoteTranscodeStep.java +++ /dev/null @@ -1,48 +0,0 @@ -package user.jobengine.server.steps; - -import java.nio.file.Path; -import java.nio.file.Paths; - -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; - -import com.ibm.nosql.json.api.BasicDBObject; - -import user.commons.configuration.SystemConfiguration; - -public class PrepareRemoteTranscodeStep extends JobStep { - static private final Logger logger = LogManager.getLogger(); - private static boolean RANDOMIZE_ARCHIVES = SystemConfiguration.getInstance().value("tsm.randomize-archives", - false); - - @StepEntry - public Object[] execute(String profileName, String fileName) throws Exception { - String hiResRoot = "m:/"; - String lowResRoot = "m:/lowres"; - - Path inputPath = Paths.get(hiResRoot, fileName); - - String realFileName = fileName; - - if (RANDOMIZE_ARCHIVES) - realFileName = realFileName.substring(9); - - String outFileName = realFileName.substring(0, realFileName.lastIndexOf(".")) + ".mp4"; - - Path relativeOutputPath = null; - if (realFileName.length() > 2) - relativeOutputPath = Paths.get(realFileName.substring(0, 1), realFileName.substring(1, 2), - realFileName.substring(2, 3), outFileName); - else - relativeOutputPath = Paths.get("0", outFileName); - - BasicDBObject parameters = new BasicDBObject(); - parameters.put("profile", profileName); - parameters.put("input", inputPath.toString()); - parameters.put("output", Paths.get(lowResRoot, relativeOutputPath.toString()).toString()); - - logger.info("Prepared for remote transode {}", parameters); - return new Object[] { parameters, relativeOutputPath.toString() }; - } - -} diff --git a/server/-product/production/OMAR/jobs/steps/QueryMissingProxyMediaStep.java b/server/-product/production/OMAR/jobs/steps/QueryMissingProxyMediaStep.java deleted file mode 100644 index 7df2bed0..00000000 --- a/server/-product/production/OMAR/jobs/steps/QueryMissingProxyMediaStep.java +++ /dev/null @@ -1,74 +0,0 @@ -package user.jobengine.server.steps; - -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; - -import com.ibm.nosql.json.api.BasicDBObject; -import com.ibm.nosql.json.api.DB; -import com.ibm.nosql.json.api.DBCollection; - -import user.commons.log4j2.marker.MediaCubeUndoMarker; -import user.commons.nosql.NoSQLUtils; -import user.jobengine.db.IItemManager; -import user.jobengine.db.Media; -import user.jobengine.db.MediaFile; - -public class QueryMissingProxyMediaStep extends JobStep { - private static final Logger logger = LogManager.getLogger(); - - @StepEntry - public Object[] execute() throws Exception { - setDescription("Looking for missing proxy"); - DB db = NoSQLUtils.getNoSQLDB(); - DBCollection collection = db.getCollection("missing_lowres"); - Media media = getFirstUntranscodedMedia(collection); - - if (media == null) { - logger.info(new MediaCubeUndoMarker(getSessionMarker().getSessionID()), "Nincs feldolgozandó hiány."); - cancel(); - return null; - } - - String name = media.getMediaFileRealName(); - collection.save(new BasicDBObject("name", name)); - logger.info(getSessionMarker(), "Processing mediaID: {}", media.getId()); - return new Object[] { media }; - } - - private Media getFirstUntranscodedMedia(DBCollection collection) { - Media[] result = new Media[] { null }; - //MV - String query = "SELECT mediaid FROM VW_MISSING_PROXY_IDS WHERE HOUSEID like 'M%' or HOUSEID like 'P%' or HOUSEID like 'R%' ORDER BY modified DESC"; - - //HTV - //String query = "SELECT mediaid FROM VW_MISSING_PROXY_IDS"; - IItemManager manager = getManager(); - manager.executeQuery(query, rs -> { - try { - long mediaId = rs.getLong(1); - Media media = manager.getMedia(mediaId); - //a nevgeneralas miatt az eredeti MediaFilesName nem jo, a pontos nev kell nekunk - String name = media.getMediaFileRealName(); - //logger.info(getSessionMarker(), "Checking {}", name); - long existing = collection.find(new BasicDBObject("name", name)).count(); - if (existing > 0) { - //logger.info(getSessionMarker(), "{} is on missing_lowres list", name); - return true; - } - // 210617 proxy keszites tiltasa - MediaFile mf = manager.getSystemMediaFile(media); - if (mf.isDisableProxy()) { - //logger.info(getSessionMarker(), "Proxy disabled {}", name); - - return true; - } - - result[0] = media; - } catch (Exception e) { - logger.error(e); - } - return false; - }, null); - return result[0]; - } -} diff --git a/server/-product/production/OMAR/jobs/steps/RemoteJobStep.java b/server/-product/production/OMAR/jobs/steps/RemoteJobStep.java deleted file mode 100644 index d4900d76..00000000 --- a/server/-product/production/OMAR/jobs/steps/RemoteJobStep.java +++ /dev/null @@ -1,36 +0,0 @@ -package user.jobengine.server.steps; - -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; - -import com.ibm.nosql.json.api.BasicDBObject; - -import user.jobengine.server.steps.shared.MediaCubeClient; - -public class RemoteJobStep extends JobStep { - private static final Logger logger = LogManager.getLogger(); - - @StepEntry - public Object[] execute(String remoteServer, String template, String name, BasicDBObject remoteJobParameters) throws Exception { - setDescription("Executing on {}, template {}, profile {}", remoteServer, template, remoteJobParameters.getString("profile")); - MediaCubeClient mc = new MediaCubeClient(remoteServer); - long jobId = mc.startjob(template, name, remoteJobParameters); - logger.info(getMarker(), "Started {} on server {}", jobId, remoteServer); - while (true) { - BasicDBObject status = mc.getStatus(jobId); - if (status != null) - setProgress(status.getInt("progress")); - - Thread.sleep(2000); - - String jobStatus = status.getString("status"); - if ("SUSPENDED".equals(jobStatus)) - throw new Exception(status.getString("description")); - - if ("FINISHED".equals(jobStatus)) - break; - } - - return null; - } -} diff --git a/server/-product/production/OMAR/jobs/steps/SafeDeleteRecursiveStep.java b/server/-product/production/OMAR/jobs/steps/SafeDeleteRecursiveStep.java deleted file mode 100644 index 238aae23..00000000 --- a/server/-product/production/OMAR/jobs/steps/SafeDeleteRecursiveStep.java +++ /dev/null @@ -1,416 +0,0 @@ -package user.jobengine.server.steps; - -import java.io.File; -import java.io.IOException; -import java.nio.file.DirectoryStream; -import java.nio.file.FileVisitResult; -import java.nio.file.FileVisitor; -import java.nio.file.Files; -import java.nio.file.Path; -import java.nio.file.Paths; -import java.nio.file.attribute.BasicFileAttributes; -import java.text.SimpleDateFormat; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.Collections; -import java.util.Date; -import java.util.LinkedHashSet; -import java.util.List; -import java.util.Set; - -import org.apache.commons.io.FileUtils; -import org.apache.commons.io.FilenameUtils; -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; - -import com.ibm.nosql.json.api.BasicDBObject; - -import user.commons.IEntityBase; -import user.commons.RemoteFile; -import user.commons.StoreUri; -import user.commons.mediatool.MediaInfo; -import user.commons.remotestore.RemoteStoreProtocol; -import user.jobengine.db.Item; -import user.jobengine.db.Media; -import user.jobengine.db.MediaFile; -import user.jobengine.db.MediaFileDAO; -import user.jobengine.db.Store; -import user.jobengine.server.steps.shared.EscortFiles; -import user.mediacube.metadata.interfaces.IMetadata; -import user.mediacube.metadata.interfaces.IMetadataListOptions; -import user.mediacube.metadata.interfaces.IMetadataProvider; -import user.mediacube.metadata.interfaces.IMetadataProviderFactory; -import user.mediacube.metadata.interfaces.MetadataProviderType; -import user.mediacube.metadata.interfaces.MetadataType; -import user.mediacube.metadata.interfaces.PlanAirMetadataListOptions; - -public class SafeDeleteRecursiveStep extends JobStep implements FileVisitor { - private static final Logger logger = LogManager.getLogger(); - private static final String ITEM_TITLE = "itemTitle"; - private static final String ITEM_HOUSEID = "itemHouseId"; - private static final String MEDIA_HOUSEID = "mediaHouseId"; - private static final String MEDIA_TITLE = "mediaTitle"; - private static final String MEDIA_DESCRIPTION = "mediaDescription"; - private static final String MEDIA_TYPE = "mediaType"; - private static final String KILLDATEEXT = ".killdate"; - private SimpleDateFormat df = new SimpleDateFormat("yyyy.MM.dd HH:mm:ss"); - - private List skipPathNames = Arrays.asList("!ARCHIVALAS_ALATT", EscortFiles.STATUSFOLDER, - EscortFiles.CONFLICTFOLDER); - private Set includeList; - - private boolean canReadMediaInfo(Path mediaFilePath) { - boolean result = false; - try { - MediaInfo mi = new MediaInfo(mediaFilePath); - mi.process(); - result = true; - } catch (Exception e) { - logger.warn(getSessionMarker(), e.getMessage()); - } - return result; - } - - private boolean checkArchiveItem(ArchiveItem archiveItem, Item item, Media media) { - if (archiveItem == null) - return false; - - String itemHouseId = archiveItem.getItemHouseId() == null ? "" : archiveItem.getItemHouseId(); - if (!itemHouseId.equals(item.getHouseId())) { - logger.error(getSessionMarker(), "ItemHouseId"); - return false; - } - - String itemTitle = archiveItem.getItemTitle() == null ? "" : archiveItem.getItemTitle(); - if (!itemTitle.equals(item.getTitle())) { - logger.error(getSessionMarker(), "ItemTitle"); - return false; - } - String mediaHouseId = archiveItem.getMediaHouseId() == null ? "" : archiveItem.getMediaHouseId(); - if (!mediaHouseId.equals(media.getHouseId())) { - logger.error(getSessionMarker(), "MediaHouseId"); - return false; - } - String mediaTitle = archiveItem.getMediaTitle() == null ? "" : archiveItem.getMediaTitle(); - if (!mediaTitle.equals(media.getTitle())) { - logger.error(getSessionMarker(), "MediaTitle"); - return false; - } - - String mediaDescription = archiveItem.getMediaDescription() == null ? "" : archiveItem.getMediaDescription(); - String storedMediaDescription = media.getDescription() == null ? "" : media.getDescription(); - if (!mediaDescription.equals(storedMediaDescription)) { - logger.error(getSessionMarker(), "MediaDescription"); - return false; - } - - return true; - } - - private ArchiveItem createArchiveItem(Path filePath) throws Exception { - ArchiveItem result = null; - - String fileName = filePath.getFileName().toString(); - String mediaHouseId = FilenameUtils.removeExtension(fileName); - try { - result = getPlanAirMetadata(mediaHouseId); - - if (result != null) - result.setMediaFile(filePath.toString()); - - } catch (Exception e) { - logger.error(getSessionMarker(), "PlanAir metadata error", e); - } - - return result; - } - - @StepEntry - public Object[] execute(String sourcePath) throws Exception { - logger.info(getSessionMarker(), "Starting in {}", sourcePath); - logger.info(getSessionMarker(), "{};{};{};{};{};{};{};{};{};{};{};{};{};{};{};{};{};{};", "Napló időbélyeg", - "Neve", "Elérése", "Mérete", "Létrehozva", "Módosítva", "Archiválva (MC)", "Archiválva (TSM)", - "Méret (TSM)", ".catched", ".killdate", ".nomd", "Archivált (MC)", "Archivált (TSM)", - "Metaadat egyezés", "Méret egyezés", "Törölhető", "Kezelendő"); - - try { - String location = "/opt/test-mediacube/file_list_original.txt"; - includeList = loadIncludeList(location); - Files.walkFileTree(Paths.get(sourcePath), this); - } catch (Exception e) { - logger.error(getSessionMarker(), "Az '{}' mappa elérése sikertelen. A rendszer hibaüzenete: {}", sourcePath, - e.getMessage()); - } - return null; - } - - private ArchiveItem getPlanAirMetadata(String mediaHouseId) throws Exception { - PlanAirMetadataListOptions opt = new PlanAirMetadataListOptions(); - opt.setSearch(mediaHouseId); - opt.setType(MetadataType.Material); - - BasicDBObject json = null; - List data = null; - - IMetadataProviderFactory factory = getService(IMetadataProviderFactory.class); - if (factory == null) - logger.info(getSessionMarker(), "IMetadataProviderFactory is null"); - - IMetadataProvider planairProvider = factory.getProvider(MetadataProviderType.PLANAIR); - if (planairProvider == null) - logger.info(getSessionMarker(), "IMetadataProvider is null"); - - ArchiveItem result = null; - data = planairProvider.list(opt); - if (data.size() != 0) - json = data.get(0).asJSON(); - else { - opt.setType(MetadataType.Promo); - data = planairProvider.list(opt); - if (data.size() != 0) - json = data.get(0).asJSON(); - else { - opt.setType(MetadataType.AD); - data = planairProvider.list(opt); - if (data.size() != 0) - json = data.get(0).asJSON(); - } - } - if (json != null) { - result = new ArchiveItem(); - result.setItemHouseId(json.getString(ITEM_HOUSEID)); - result.setItemTitle(json.getString(ITEM_TITLE)); - result.setMediaHouseId(json.getString(MEDIA_HOUSEID)); - result.setMediaTitle(json.getString(MEDIA_TITLE)); - result.setMediaDescription(json.getString(MEDIA_DESCRIPTION)); - result.setMediaType(json.getString(MEDIA_TYPE)); - } - return result; - } - - @Override - public FileVisitResult postVisitDirectory(Path paramT, IOException paramIOException) throws IOException { - return FileVisitResult.CONTINUE; - } - - @Override - public FileVisitResult preVisitDirectory(Path dir, BasicFileAttributes paramBasicFileAttributes) - throws IOException { - Path dirName = dir.getFileName(); - - if (skipPathNames.contains(dirName.toString())) { - //logger.info(getSessionMarker(), "PreVisit skip {}", dir); - return FileVisitResult.SKIP_SUBTREE; - } //else - //logger.info(getSessionMarker(), "PreVisit {}", dir); - - return FileVisitResult.CONTINUE; - } - - private boolean processPathItem(Path mediaPath) throws Exception { - if (getJobRuntime().isWaitingCancel()) { - cancel(); - return false; - } - - BasicFileAttributes attr = Files.readAttributes(mediaPath, BasicFileAttributes.class); - File mediaFSFile = mediaPath.toFile(); - String fileName = mediaPath.getFileName().toString(); - - if (mediaFSFile.isDirectory()) - return true; - - if (fileName.startsWith(".") || fileName.endsWith(".nomd")) - return true; - - Path nomdFile = Paths.get(mediaPath.toString() + ".nomd"); - ArchiveItem archiveItem = createArchiveItem(mediaPath); - List killDateFiles = getKillDateFiles(mediaPath); - MediaFile mediaFile = getMediaFile(fileName); - - long size = mediaFSFile.length(); - Date lastModifiedDate = new Date(attr.lastModifiedTime().toMillis()); - //Date lastAccesDate = new Date(attr.lastAccessTime().toMillis()); - Date createDate = new Date(attr.creationTime().toMillis()); - boolean catchedExists = EscortFiles.isMediaCatched(mediaPath); - boolean noMDExists = Files.exists(nomdFile); - boolean killdateExists = killDateFiles.size() > 0; - //boolean mediaInfoAvailable = canReadMediaInfo(mediaPath); - boolean mcArchived = mediaFile != null; - boolean metadataEquals = false; - boolean tsmArchived = false; - boolean sizeEquals = false; - Date mcArchivedDate = null; - long tsmSize = 0; - Date tsmBackupDate = null; - boolean includeContains = includeList.contains(mediaPath.toString()); - - if (mcArchived) { - //metadata - Media media = getManager().getMedia(mediaFile.getMediaId()); - mcArchivedDate = media.getCreated(); - Item item = getManager().getItem(media.getItemId()); - metadataEquals = checkArchiveItem(archiveItem, item, media); - - //tsm - String tsmFileName = mediaFile.getRelativePath(); - RemoteFile tsmFile = getTSMFile(tsmFileName); - tsmArchived = tsmFile != null; - if (tsmArchived) { - tsmSize = tsmFile.getSize(); - tsmBackupDate = getTSMBackupDate(tsmFileName); - sizeEquals = tsmSize == size; - } - } - - boolean canDelete = !noMDExists && catchedExists && killdateExists && mcArchived && tsmArchived - && metadataEquals; - Date now = new Date(System.currentTimeMillis()); - logger.info(getSessionMarker(), "{};{};{};{};{};{};{};{};{};{};{};{};{};{};{};{};{};{};", D(now), fileName, - mediaPath.getParent(), size, D(createDate), D(lastModifiedDate), D(mcArchivedDate), D(tsmBackupDate), - tsmSize, YN(catchedExists), YN(killdateExists), YN(noMDExists), YN(mcArchived), YN(tsmArchived), - YN(metadataEquals), YN(sizeEquals), YN(canDelete), YN(includeContains)); - - return true; - } - - private Date getTSMBackupDate(String tsmFileName) throws Exception { - Date result = null; - IMetadataProviderFactory factory = getService(IMetadataProviderFactory.class); - if (factory == null) - return null; - IMetadataProvider provider = factory.getProvider(MetadataProviderType.TSM); - if (provider == null) - return null; - - IMetadataListOptions opt = provider.createOptions(new BasicDBObject("fileName", tsmFileName)); - - List tsmContents = provider.list(opt); - if (tsmContents != null && tsmContents.size() > 0) { - - for (IMetadata md : tsmContents) { - Date backupDate = md.asJSON().getDate("backupDate"); - if (result == null) - result = backupDate; - else { - if (backupDate.after(result)) - result = backupDate; - } - } - } - return result; - } - - private char YN(boolean value) { - return value ? 'Y' : 'N'; - } - - private String D(Date value) { - return value == null ? "" : df.format(value); - } - - @Override - public FileVisitResult visitFile(Path filePath, BasicFileAttributes paramBasicFileAttributes) throws IOException { - //logger.info(getSessionMarker(), "Will checked {}", filePath); - try { - if (!processPathItem(filePath)) - return FileVisitResult.TERMINATE; - } catch (Exception e) { - logger.catching(e); - } - return FileVisitResult.CONTINUE; - } - - @Override - public FileVisitResult visitFileFailed(Path filePath, IOException paramIOException) throws IOException { - logger.info("Error archive {}", filePath); - return FileVisitResult.CONTINUE; - } - - private List getKillDateFiles(Path filePath) { - String killDateFilePattern = String.format("%s.*%s", filePath.getFileName().toString(), KILLDATEEXT); - List result = new ArrayList<>(); - Path statusPath = null; - try { - statusPath = Paths.get(filePath.getParent().toString(), EscortFiles.STATUSFOLDER); - } catch (Exception e) { - logger.catching(e); - return null; - } - File statusPathFile = statusPath.toFile(); - if (statusPathFile.exists() && statusPathFile.isDirectory()) { - try (DirectoryStream stream = Files.newDirectoryStream(statusPath, killDateFilePattern)) { - stream.forEach(p -> result.add(p)); - } catch (Exception e) { - logger.catching(e); - } - } - Collections.sort(result); - return result; - } - - MediaFile getMediaFile(String fileName) { - MediaFile result = null; - MediaFileDAO mfDAO = (MediaFileDAO) getManager().getBaseDAO(MediaFile.class); - List mfList = mfDAO.getByHouseId(fileName); - if (mfList != null && mfList.size() == 1) - result = (MediaFile) mfList.get(0); - return result; - } - - RemoteFile getTSMFile(String mcFileName) { - RemoteFile result = null; - Store tsmStore = getManager().getSystemStore(false); - StoreUri tsmStoreUri = tsmStore.getSourceStoreUri(RemoteStoreProtocol.TSM); - try { - result = tsmStoreUri.getRemoteFile(mcFileName); - } catch (Exception e) { - logger.error(e); - } finally { - tsmStoreUri.cleanUp(); - } - return result; - } - - private Set loadIncludeList(String location) throws IOException { - Set result = new LinkedHashSet<>(); - - Path path = Paths.get(location); - List lines = FileUtils.readLines(path.toFile()); - - String lastDir = null; - for (String line : lines) { - line = line.trim(); - if (line.startsWith("Directory of")) { - lastDir = line; - lastDir = lastDir.replace("Directory of", ""); - lastDir = lastDir.replace("X:", ""); - lastDir = lastDir.replace("\\", "/"); - lastDir = lastDir.trim(); - // if (!lastDir.endsWith(".STATUS")) - // System.out.println(">> " + lastDir); - } - - if (lastDir != null && lastDir.endsWith(".STATUS")) - continue; - - if (line.startsWith("2") && line.length() > 39) { - String file = line.substring(39).trim(); - - if (file.equals(".") || file.equals("..")) - continue; - - String len = line.substring(21, 39).trim(); - if (len.length() != 0) { - String fullpath = "/mnt/POLC/FINISHED_SHOWS" + lastDir + "/" + file; - //System.out.println("/mnt/POLC/FINISHED_SHOWS" + lastDir + "/" + file + " : " + len); - result.add(fullpath); - } - } - - } - - return result; - } - -} diff --git a/server/-product/production/OMAR/jobs/steps/SaveMediaProxy.java b/server/-product/production/OMAR/jobs/steps/SaveMediaProxy.java deleted file mode 100644 index 3f0f667d..00000000 --- a/server/-product/production/OMAR/jobs/steps/SaveMediaProxy.java +++ /dev/null @@ -1,22 +0,0 @@ -package user.jobengine.server.steps; - -import user.jobengine.db.IItemManager; -import user.jobengine.db.Media; -import user.jobengine.db.MediaFile; -import user.jobengine.db.Store; - -public class SaveMediaProxy extends JobStep { - - @StepEntry - public Object[] execute(Media media, String proxyRelativePath) throws Exception { - IItemManager manager = getManager(); - Store store = manager.getCurrentLowresStore(); - MediaFile mediaFile = new MediaFile(); - mediaFile.setMedia(media); - mediaFile.setStore(store); - mediaFile.setFileType(manager.getFileType("Low-res")); - mediaFile.setRelativePath(proxyRelativePath); - manager.add(mediaFile); - return null; - } -} diff --git a/server/-product/production/OMAR/jobs/steps/TSMBackupStep.java b/server/-product/production/OMAR/jobs/steps/TSMBackupStep.java deleted file mode 100644 index b526f16f..00000000 --- a/server/-product/production/OMAR/jobs/steps/TSMBackupStep.java +++ /dev/null @@ -1,239 +0,0 @@ -package user.jobengine.server.steps; - -import java.io.File; -import java.io.IOException; -import java.sql.Timestamp; -import java.util.Date; -import java.util.List; - -import org.apache.commons.lang.RandomStringUtils; -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; -import org.apache.logging.log4j.Marker; -import org.apache.logging.log4j.message.Message; -import org.apache.logging.log4j.message.ParameterizedMessage; - -import user.commons.RemoteFile; -import user.commons.StoreUri; -import user.commons.configuration.SystemConfiguration; -import user.commons.mediatool.Timecode; -import user.commons.mediatool.Timecode.Type; -import user.commons.remotestore.IProgressEventListener; -import user.commons.remotestore.IStatusEventListener; -import user.commons.remotestore.ProgressEvent; -import user.commons.remotestore.RemoteStoreProtocol; -import user.commons.remotestore.StatusEvent; -import user.jobengine.db.FileType; -import user.jobengine.db.IItemManager; -import user.jobengine.db.Media; -import user.jobengine.db.MediaFile; -import user.jobengine.db.Store; -import user.jobengine.server.IJobEngine; -import user.jobengine.server.IJobRuntime; -import user.jobengine.server.JobEngineException; -import user.jobengine.server.steps.shared.EscortFiles; -import user.jobengine.server.steps.shared.ItemManagerExtensions; - -public class TSMBackupStep extends JobStep { - private static final String MXFEXT = ".MXF"; - private static final Logger logger = LogManager.getLogger(); - private static boolean RANDOMIZE_ARCHIVES = SystemConfiguration.getInstance().value("tsm.randomize-archives"); - private IItemManager manager; - private File sourceMediaFile; - private Store tsmStore; - private StoreUri targetUri; - private FileType fileType; - private Marker marker; - - @StepEntry - public Object[] execute(ArchiveItem archiveItem, Media mediaCubeMedia, int killDateDays) throws Exception { - marker = getSessionMarker(); - - File sourceMediaFile = new File(archiveItem.getMediaFile()); - String sourceFileName = sourceMediaFile.getName(); - long fileSize = sourceMediaFile.length(); - - try { - Timecode timecode = new Timecode(mediaCubeMedia.getLength(), Type.PAL); - String details = String.format("%s (%s, %d bytes)", sourceFileName, timecode.toString(), fileSize); - logger.info(marker, details); - getJobRuntime().setDescription(details); - } catch (Exception e) { - String details = String.format("%s (%d bytes)", sourceFileName, fileSize); - getJobRuntime().setDescription(details); - } - Timecode timecode = new Timecode(mediaCubeMedia.getLength(), Type.PAL); - String details = String.format("%s (%s, %d bytes)", sourceFileName, timecode.toString(), fileSize); - getJobRuntime().setDescription(details); - try { - - setAndCheck(archiveItem, mediaCubeMedia, getEngine()); - - // TODO mxf helyett az osszes kiterjesztest!!!!! - // A dupla ellenorzes a napon beluli ismetlesek miatt kell - long existingMediaId = archiveItem.getExistingMediaId(); - - if (fileSize == 0 && existingMediaId == 0) { - existingMediaId = ItemManagerExtensions.getExistingRundownMedia(manager, - sourceFileName.replace(MXFEXT, "")); - if (existingMediaId == 0) - existingMediaId = -1; - } - - if (existingMediaId == 0) - existingMediaId = ItemManagerExtensions.getExistingRundownMedia(manager, - sourceFileName.replace(MXFEXT, "")); - String targetFileName; - if (RANDOMIZE_ARCHIVES) { - // a-z, A-Z, 0-9. For example: WRMcpIk7, s57JwCVA - // veletlenszeru neveket adunk! - targetFileName = String.format("%s-%s", RandomStringUtils.randomAlphanumeric(8), sourceFileName); - } else - targetFileName = sourceFileName; - - if (existingMediaId == 0) { - StoreUri sourceUri = manager.createStoreUri(RemoteStoreProtocol.LOCAL, - sourceMediaFile.getParent().toString()); - - final IJobRuntime runtime = getJobRuntime(); - sourceUri.addProgressListener(new IProgressEventListener() { - @Override - public void progressChanged(ProgressEvent evt) { - runtime.incrementProgress(evt.getProgress()); - } - }); - sourceUri.addStatusListener(new IStatusEventListener() { - @Override - public void statusChanged(StatusEvent evt) { - evt.setCancel(!canContinue()); - } - }); - - RemoteFile remoteFile = sourceUri.transferFrom(targetUri, sourceFileName, targetFileName); - } - - if (existingMediaId > 0) - logger.info(marker, "Az '{}' TSM mentése nem szükséges, mert már megtalálható az archívumban.", - sourceFileName); - - // Fel kell szabadítani, hogy a kovetkezo archivalaskor is nekifusson - if (existingMediaId == -1) { - logger.info(marker, - "Az '{}' mentése jelenleg nem lehetséges, mert a szükséges metaadat még nem található meg az archívumban.", - sourceFileName); - if (!archiveItem.removeCatchedFile()) - logger.error(marker, - "Az '{}' állomány .catched jelző állománya nem törölhető. Az újabb archiválási kísérlethez annak kézi eltávolítása szükséges!", - sourceMediaFile.getName()); - mediaCubeMedia.remove(); - } else { - saveMetadata(mediaCubeMedia, sourceMediaFile, targetFileName, existingMediaId, fileSize, - archiveItem.isDisableProxy()); - logger.info(marker, "Az '{}' archiválása sikeres.", sourceFileName); - if (killDateDays != 0) - EscortFiles.createUNCKillDate(sourceMediaFile.getParent(), sourceFileName, killDateDays, marker); - } - - } catch (Exception e) { - logger.catching(e); - Message m = new ParameterizedMessage("Az '{}' állomány archiválása sikertelen. A rendszer hibaüzenete: {}", - details, e.getMessage()); - logger.error(marker, m); - if (!archiveItem.removeCatchedFile()) - logger.error(marker, - "Az '{}' állomány .catched jelző állománya nem törölhető. Az újabb archiválási kísérlethez annak kézi eltávolítása szükséges!", - sourceMediaFile.getName()); - throw new Exception(m.getFormattedMessage()); - } - return null; - } - - private void saveMetadata(Media mediaCubeMedia, File sourceFile, String targetFileName, long existingMediaId, - long fileSize, boolean disableProxy) { - - if (existingMediaId == 0) { - MediaFile mf = manager.createMediaFile(targetFileName, fileType, tsmStore, mediaCubeMedia); - mf.setHouseId(sourceFile.getName()); - mf.setFileSize(fileSize); - // 210617 proxy keszites tiltasa - mf.setDisableProxy(disableProxy); - mf.add(); - } else { - Media existingMedia = manager.getMedia(existingMediaId); - List mediaFiles = existingMedia.getMediaFiles(); - if (mediaFiles != null) { - for (MediaFile mf : mediaFiles) { - mf.setPersister(manager); - mf.setId(0); - mf.setMedia(mediaCubeMedia); - // mivel itt masolat keszul, nem allitunk at semmit -// mf.setFileSize(fileSize); -// mf.setDisableProxy(disableProxy); - mf.add(); - } - } - } - mediaCubeMedia.setPersister(manager); - - // 210614 megis maradjon az aktualis idopont - mediaCubeMedia.setArchived(new Timestamp(new Date().getTime())); - - /* - try { - BasicFileAttributes attr = Files.readAttributes(sourceFile.toPath(), BasicFileAttributes.class); - mediaCubeMedia.setArchived(new Timestamp(attr.creationTime().toMillis())); - } catch (IOException e) { - logger.catching(e); - } - */ - mediaCubeMedia.modify(); - } - - private void setAndCheck(ArchiveItem archiveItem, Media mediaCubeMedia, IJobEngine jobEngine) - throws JobEngineException, IOException { - if (jobEngine == null) { - logger.error(marker, "Az folyamatkezelő réteg nem elérhető."); - throw new NullPointerException("Internal error, missing JobEngine reference."); - } - manager = jobEngine.getItemManager(); - if (manager == null) { - logger.error(marker, "Az adatbáziskezelő réteg nem elérhető."); - throw new NullPointerException("Internal error, missing ItemManager reference."); - } - if (archiveItem == null) { - logger.error(marker, "A folyamat 'archiveItem' bemeneti paramétere üres."); - throw new NullPointerException("System is not configured properly, missing 'archiveItem' input parameter."); - } - sourceMediaFile = new File(archiveItem.getMediaFile()); - if (sourceMediaFile == null) { - logger.error(marker, "A folyamat 'archiveItem' bemeneti paraméter 'mediaFile' értéke üres."); - throw new NullPointerException( - "System is not configured properly, missing 'mediaFile' value in 'archiveItem' input parameter."); - } - if (!sourceMediaFile.exists()) { - logger.error(marker, "A(z) {} állomány nem létezik vagy nem érhető el.", sourceMediaFile.getName()); - throw new IOException(String.format("Input file {} does not exist or unreachable.", sourceMediaFile.getName())); - } - tsmStore = manager.getSystemStore(false); - if (tsmStore == null) { - logger.error(marker, "A TSM rendszer beállítás nem elérhető."); - throw new NullPointerException("System is not configured properly, missing TSM Store."); - } - targetUri = tsmStore.getSourceStoreUri(RemoteStoreProtocol.TSM); - if (targetUri == null) { - logger.error(marker, "A TSM rendszer beállítás paraméterei nem elérhetőek."); - throw new NullPointerException("System is not configured properly, missing TSM StoreUri."); - } - fileType = manager.getFileType("High-res"); - if (fileType == null) { - logger.error(marker, "Adatbázis bejegyzés hiba, a 'High-res' FileType nem található."); - throw new NullPointerException("System is not configured properly, missing 'High-res' FileType."); - } - if (mediaCubeMedia == null) { - logger.error(marker, "A folyamat 'mediaCubeMedia' bemeneti paramétere üres."); - throw new NullPointerException( - "System is not configured properly, 'mediaCubeMedia' input parameter missing."); - } - - } -} diff --git a/server/-product/production/OMAR/jobs/steps/TSMExtendedRetrieveStep.java b/server/-product/production/OMAR/jobs/steps/TSMExtendedRetrieveStep.java deleted file mode 100644 index fd222344..00000000 --- a/server/-product/production/OMAR/jobs/steps/TSMExtendedRetrieveStep.java +++ /dev/null @@ -1,95 +0,0 @@ -package user.jobengine.server.steps; - -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; - -import user.commons.StoreUri; -import user.commons.remotestore.RemoteStoreProtocol; -import user.jobengine.db.ArchivedMedia; -import user.jobengine.db.IItemManager; -import user.jobengine.server.IJobEngine; -import user.jobengine.server.IJobRuntime; - -/* -import user.jobengine.server.steps.EscortFiles; -*/ - -public class TSMExtendedRetrieveStep extends TSMRestoreStep { - private static String NEXIO_HOST = System.getProperty("nexio.host"); - // private static String NEXIO_HOST = SystemConfiguration.getInstance().value("services.nexio.host"); - private static final Logger logger = LogManager.getLogger(); - - private boolean useNexioTarget; - private int nexioPort; - private String nexioUserName, nexioPassword; - private String nexioAgency; - - @Override - protected void afterRestore(StoreUri targetUri, String targetPath, int killDateDays, String targetFileName) throws Exception { - if (useNexioTarget) { - EscortFiles.setNEXIOKillDate(killDateDays, targetFileName, nexioAgency, targetUri); - } else { - super.afterRestore(targetUri, targetPath, killDateDays, targetFileName); - } - } - - @Override - protected void beforeRestore(StoreUri targetURI, String targetName) throws Exception { - String newTargetName = targetName; - if (targetName.contains(".")) - newTargetName = targetName.substring(0, targetName.lastIndexOf('.')); - if (useNexioTarget) - if (targetURI.fileExists(newTargetName + ".mxf")) - throw new Exception(String.format("%s-The newly created file name is existed.", getClass().getSimpleName())); - } - - @Override - protected void checkTargetPath(String targetPath) { - if (!useNexioTarget) - super.checkTargetPath(targetPath); - } - - @Override - protected StoreUri createTargetUri(IItemManager manager, String targetPath) throws NullPointerException { - StoreUri result = null; - logger.info(getSessionMarker(), "Create target uri {}", targetPath); - if (useNexioTarget) { - if (NEXIO_HOST == null) { - throw new NullPointerException("Missing system property on 'nexio.host' name"); - } - result = manager.createStoreUri(RemoteStoreProtocol.FTP, NEXIO_HOST); - result.setPortNumber(nexioPort); - result.setUserName(nexioUserName); - result.setPassword(nexioPassword); - } else - result = super.createTargetUri(manager, targetPath); - return result; - } - - @StepEntry - public Object[] execute(ArchivedMedia archivedMedia, String targetPath, String targetNamePattern, String successRecipient, int killDateDays, - String localRetrievePath, String globalRetrievePath, boolean useNexioTarget, String nexioAgency, int nexioPort, String nexioUserName, - String nexioPassword, IJobEngine jobEngine, IJobRuntime jobRuntime) throws Exception { - this.useNexioTarget = useNexioTarget; - this.nexioAgency = nexioAgency; - this.nexioPort = nexioPort; - this.nexioUserName = nexioUserName; - this.nexioPassword = nexioPassword; - if (nexioPort == 0) { - throw new NullPointerException("System is not configured properly, 'nexioPort' input parameter missing."); - } - if (nexioUserName == null) { - throw new NullPointerException("System is not configured properly, 'nexioUserName' input parameter missing."); - } - if (nexioPassword == null) { - throw new NullPointerException("System is not configured properly, 'nexioPassword' input parameter missing."); - } - if (nexioAgency == null) { - throw new NullPointerException("System is not configured properly, 'nexioAgency' input parameter missing."); - } - - return super.execute(archivedMedia.getMedia(), targetPath, targetNamePattern, successRecipient, killDateDays, localRetrievePath, globalRetrievePath, - jobEngine, jobRuntime); - } - -} diff --git a/server/-product/production/OMAR/jobs/steps/TSMRestoreStep.java b/server/-product/production/OMAR/jobs/steps/TSMRestoreStep.java deleted file mode 100644 index 26d8607b..00000000 --- a/server/-product/production/OMAR/jobs/steps/TSMRestoreStep.java +++ /dev/null @@ -1,187 +0,0 @@ -package user.jobengine.server.steps; - -import java.io.IOException; -import java.nio.file.Paths; -import java.text.Normalizer; -import java.util.List; -import java.util.regex.Pattern; - -import org.apache.commons.lang.StringUtils; -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; -import org.apache.logging.log4j.Marker; -import org.apache.logging.log4j.message.Message; - -import user.commons.LogUtils; -import user.commons.RemoteFile; -import user.commons.StoreUri; -import user.commons.mediatool.Timecode; -import user.commons.mediatool.Timecode.Type; -import user.commons.remotestore.IProgressEventListener; -import user.commons.remotestore.IStatusEventListener; -import user.commons.remotestore.ProgressEvent; -import user.commons.remotestore.RemoteStoreProtocol; -import user.commons.remotestore.StatusEvent; -import user.jobengine.db.IItemManager; -import user.jobengine.db.Media; -import user.jobengine.db.MediaFile; -import user.jobengine.db.Store; -import user.jobengine.server.IJobEngine; -import user.jobengine.server.IJobRuntime; -import user.jobengine.server.steps.shared.EscortFiles; - -public class TSMRestoreStep extends JobStep { - private static final String DOT = "."; - public static final Pattern DIACRITICS_AND_FRIENDS = Pattern.compile("[\\p{InCombiningDiacriticalMarks}\\p{IsLm}\\p{IsSk}]+"); - private static final Logger logger = LogManager.getLogger(); - private IItemManager manager; - private StoreUri targetUri; - private StoreUri sourceUri; - private String sourceFileName; - private Marker marker; - - protected void afterRestore(StoreUri targetUri, String targetPath, int killDateDays, String targetFileName) throws IOException, Exception { - if (killDateDays != 0) - EscortFiles.createUNCKillDate(targetPath, targetFileName, killDateDays, marker); - } - - protected void beforeRestore(StoreUri targetURI, String targetFileName) throws Exception { - } - - protected void checkTargetPath(String targetPath) { - if (StringUtils.isBlank(targetPath)) { - logger.error(marker, "A folyamat 'targetPath' bemeneti paramétere üres."); - throw new NullPointerException("System is not configured properly, 'targetPath' input parameter missing."); - } - } - - protected StoreUri createTargetUri(IItemManager manager, String targetPath) { - return manager.createStoreUri(RemoteStoreProtocol.LOCAL, targetPath); - } - - @StepEntry - public Object[] execute(Media mediaCubeMedia, String targetPath, String targetNamePattern, String successRecipient, int killDateDays, - String localRetrievePath, String globalRetrievePath, IJobEngine jobEngine, IJobRuntime jobRuntime) throws Exception { - marker = jobRuntime.getSessionMarker(); - setAndCheck(mediaCubeMedia, targetPath, targetNamePattern, localRetrievePath, globalRetrievePath, jobEngine); - String targetFileName = String.format(targetNamePattern, sourceFileName); - //20210129 - //targetFileName = getMaximizedFileName(mediaCubeMedia, targetFileName, 120); - - Timecode timecode = new Timecode(mediaCubeMedia.getLength(), Type.PAL); - try { - String details = String.format("%s (%s)", sourceFileName, timecode.toString()); - jobRuntime.setDescription(details); - beforeRestore(targetUri, targetNamePattern); - final IJobRuntime runtime = jobRuntime; - sourceUri.addProgressListener(new IProgressEventListener() { - @Override - public void progressChanged(ProgressEvent evt) { - runtime.incrementProgress(evt.getProgress()); - } - }); - sourceUri.addStatusListener(new IStatusEventListener() { - @Override - public void statusChanged(StatusEvent evt) { - evt.setCancel(!canContinue()); - } - }); - RemoteFile result = sourceUri.transferFrom(targetUri, sourceFileName, targetFileName); - - String globalTargetPath = Paths.get(targetPath, targetFileName) - .getParent() - .toString() - .replace(Paths.get(localRetrievePath).toString(), globalRetrievePath); - - logger.info(marker, - "Az '{}' állomány visszatöltése sikeres volt '{}' néven. A célmappa a ide kattintva nyitható meg.", - sourceFileName, targetFileName, globalTargetPath); - afterRestore(targetUri, targetPath, killDateDays, targetFileName); - - } catch (Exception e) { - Message msg = LogUtils.format("Az '{}' állomány visszatöltése sikertelen. A rendszer hibaüzenete: {}", sourceFileName, e.getMessage()); - logger.error(marker, msg); - // logger.error(jobRuntime.marker, msg); - logger.catching(e); - throw e; - } - - return null; - } - - private String getMaximizedFileName(Media mediaCubeMedia, String targetFileName, int limit) { - String name = targetFileName; - String extension = ""; - if (name.contains(DOT)) { - extension = DOT + name.substring(name.lastIndexOf(DOT) + 1); - name = name.substring(0, name.lastIndexOf(DOT)); - } - String typeName = Normalizer.normalize(mediaCubeMedia.getItemType().getName(), Normalizer.Form.NFD); - typeName = DIACRITICS_AND_FRIENDS.matcher(typeName).replaceAll(""); - typeName = typeName.replace(" ", "_"); - - int allowedSize = limit - typeName.length() - 1 - extension.length(); - if (name.length() > allowedSize) - name = name.substring(0, allowedSize); - - return String.format("%s_%s%s", name, typeName, extension); - } - - private String getSourceFileName(Media mediaCubeMedia, Store store) { - List mediaFiles = mediaCubeMedia.getMediaFiles(); - if (mediaFiles == null) - return null; - for (MediaFile mediaFile : mediaFiles) { - if (mediaFile.getStore().getId() == store.getId()) - return mediaFile.getRelativePath(); - } - return null; - } - - private void setAndCheck(Media mediaCubeMedia, String targetPath, String targetNamePattern, String localRetrievePath, String globalRetrievePath, - IJobEngine jobEngine) { - if (jobEngine == null) { - logger.error(marker, "Az folyamatkezelő réteg nem elérhető."); - throw new NullPointerException("Internal error, missing JobEngine reference."); - } - manager = jobEngine.getItemManager(); - if (manager == null) { - logger.error(marker, "Az adatbáziskezelő réteg nem elérhető."); - throw new NullPointerException("Internal error, missing ItemManager reference."); - } - if (mediaCubeMedia == null) { - logger.error(marker, "A folyamat 'mediaCubeMedia' bemeneti paramétere üres."); - throw new NullPointerException("System is not configured properly, 'mediaCubeMedia' input parameter missing."); - } - checkTargetPath(targetPath); - if (StringUtils.isBlank(targetNamePattern)) { - logger.error(marker, "A folyamat 'targetNamePattern' bemeneti paramétere üres."); - throw new NullPointerException("System is not configured properly, 'targetNamePattern' input parameter missing."); - } - Store tsmStore = manager.getSystemStore(false); - if (tsmStore == null) { - logger.error(marker, "A TSM rendszer beállítás nem elérhető."); - throw new NullPointerException("System is not configured properly, missing TSM Store."); - } - sourceUri = tsmStore.getSourceStoreUri(RemoteStoreProtocol.TSM); - if (sourceUri == null) { - logger.error(marker, "A TSM rendszer beállítás paraméterei nem elérhetőek."); - throw new NullPointerException("System is not configured properly, missing TSM StoreUri."); - } - targetUri = createTargetUri(manager, targetPath); - sourceFileName = getSourceFileName(mediaCubeMedia, tsmStore); - if (sourceFileName == null) { - logger.error(marker, "Adatbázis bejegyzés hiba, a visszatöltendő fájl neve nem található."); - throw new NullPointerException("Database error, missing MediaFile 'relativePath'."); - } - - if (StringUtils.isBlank(localRetrievePath)) { - logger.error(marker, "A folyamat 'localRetrievePath' bemeneti paramétere üres."); - throw new NullPointerException("System is not configured properly, 'localRetrievePath' input parameter missing."); - } - if (StringUtils.isBlank(globalRetrievePath)) { - logger.error(marker, "A folyamat 'globalRetrievePath' bemeneti paramétere üres."); - throw new NullPointerException("System is not configured properly, 'globalRetrievePath' input parameter missing."); - } - } -} diff --git a/server/-product/production/OMAR/jobs/steps/TSMSimpleRestoreStep.java b/server/-product/production/OMAR/jobs/steps/TSMSimpleRestoreStep.java deleted file mode 100644 index ba63212c..00000000 --- a/server/-product/production/OMAR/jobs/steps/TSMSimpleRestoreStep.java +++ /dev/null @@ -1,57 +0,0 @@ -package user.jobengine.server.steps; - -import java.nio.file.Files; -import java.nio.file.Paths; - -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; - -import user.commons.StoreUri; -import user.commons.mediatool.Timecode; -import user.commons.mediatool.Timecode.Type; -import user.commons.remotestore.IProgressEventListener; -import user.commons.remotestore.IStatusEventListener; -import user.commons.remotestore.ProgressEvent; -import user.commons.remotestore.RemoteStoreProtocol; -import user.commons.remotestore.StatusEvent; -import user.jobengine.db.Media; -import user.jobengine.db.MediaFile; -import user.jobengine.db.Store; - -public class TSMSimpleRestoreStep extends JobStep { - private static final Logger logger = LogManager.getLogger(); - - @StepEntry - public Object[] execute(long mediaId, String targetPath) throws Exception { - String fileName = null; - try { - Media media = getManager().getMedia(mediaId); - MediaFile mediaFile = getManager().getSystemMediaFile(media); - fileName = mediaFile.getRelativePath(); - Timecode timecode = new Timecode(media.getLength(), Type.PAL); - getJobRuntime().setDescription(String.format("%s (%s)", fileName, timecode.toString())); - Store tsmStore = getManager().getSystemStore(false); - StoreUri sourceStoreUri = getManager().getStoreUri(tsmStore.getName(), RemoteStoreProtocol.TSM); - sourceStoreUri.addProgressListener(new IProgressEventListener() { - @Override - public void progressChanged(ProgressEvent evt) { - setProgress(evt.getProgress()); - } - }); - sourceStoreUri.addStatusListener(new IStatusEventListener() { - @Override - public void statusChanged(StatusEvent evt) { - evt.setCancel(!canContinue()); - } - }); - StoreUri targetStoreUri = getManager().createStoreUri(RemoteStoreProtocol.LOCAL, targetPath); - sourceStoreUri.transferFrom(targetStoreUri, fileName, fileName + ".part"); - Files.move(Paths.get(targetPath, fileName + ".part"), Paths.get(targetPath, fileName)); - } catch (Exception e) { - logger.error("Az '{}' állomány visszatöltése sikertelen. A rendszer üzenete: {}", fileName, e.getMessage()); - throw e; - } - - return null; - } -} diff --git a/server/-product/production/OMAR/jobs/steps/TestForkCancelableStep.java b/server/-product/production/OMAR/jobs/steps/TestForkCancelableStep.java deleted file mode 100644 index c1b49f76..00000000 --- a/server/-product/production/OMAR/jobs/steps/TestForkCancelableStep.java +++ /dev/null @@ -1,35 +0,0 @@ -package user.jobengine.server.steps; - -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; - -import user.commons.IJob; -import user.commons.JobStatus; -import user.commons.ListUtils; -import user.jobengine.server.IJobEngine; -import user.jobengine.server.IJobRuntime; - -public class TestForkCancelableStep extends JobStep { - private static final String CHILD_TITLE = "Párhuzamosított alfolyamat"; - // private static final String CHILD_TEMPLATE = "fake-concurrent.xml"; - private static final String CHILD_TEMPLATE = "cancelable.xml"; - private static final Logger logger = LogManager.getLogger(); - int count = 3; - - @StepEntry - public Object[] execute(IJobEngine jobEngine, IJobRuntime jobRuntime) throws Exception { - for (int i = 0; i < count; i++) { - IJobRuntime rt = getEngine().submit(null, e -> { - if (e.getStatus().equals(JobStatus.SUSPENDED)) { - IJobRuntime rt = (IJobRuntime) e.getSource(); - logger.info("Cleanup on SUSPEND {}, {}", rt.getId(), rt.isDisableRetry()); - } - }, CHILD_TEMPLATE, "JOB " + i, 0, IJobEngine.DEFAULT_OWNER, ListUtils.asMap("param", i)); - - rt.setRelated("TEST" + rt.getId()); - } - - logger.info("Done"); - return null; - } -} diff --git a/server/-product/production/OMAR/jobs/steps/TranscodeFFAStranStep.java b/server/-product/production/OMAR/jobs/steps/TranscodeFFAStranStep.java deleted file mode 100644 index d09685bc..00000000 --- a/server/-product/production/OMAR/jobs/steps/TranscodeFFAStranStep.java +++ /dev/null @@ -1,151 +0,0 @@ -package user.jobengine.server.steps; - -import java.io.File; -import java.io.IOException; -import java.nio.file.Files; -import java.nio.file.Path; -import java.nio.file.Paths; - -import org.apache.commons.io.FilenameUtils; -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; -import org.apache.logging.log4j.Marker; -import org.apache.logging.log4j.message.Message; -import org.apache.logging.log4j.message.ParameterizedMessage; - -import user.commons.FFAStransAPI; -import user.commons.IFFAStransAPI; -import user.commons.StoreUri; -import user.commons.mediatool.Timecode; -import user.commons.mediatool.Timecode.Type; -import user.commons.remotestore.RemoteStoreProtocol; -import user.jobengine.db.FileType; -import user.jobengine.db.IItemManager; -import user.jobengine.db.Media; -import user.jobengine.db.Store; -import user.jobengine.server.IJobEngine; -import user.jobengine.server.IJobRuntime; - -public class TranscodeFFAStranStep extends JobStep { - private static final int POLL_INTERVALL = 3000; - private static final String MP4EXT = ".MP4"; - private static final String MXFEXT = ".MXF"; - private static final String LOWRES_FILETYPE = "Low-res"; - private static final Logger logger = LogManager.getLogger("TranscodeFFAStranStep"); - private IItemManager manager; - private Store store; - private FileType fileType; - private Media mediaCubeMedia; - private Marker marker; - - @StepEntry - public Object[] execute(ArchiveItem archiveItem, Media mediaCubeMedia, String transcoderAddress, String transcoderTemplateName, - String globalHiresSourcePath, String localLowresTargetPath, boolean deleteSource, IJobEngine jobEngine, IJobRuntime jobRuntime) throws Exception { - - this.marker = jobRuntime.getSessionMarker(); - this.manager = jobEngine.getItemManager(); - this.store = check(manager.getCurrentLowresStore(), "lowres Store"); - this.fileType = check(manager.getFileType(LOWRES_FILETYPE), "lowres FileType"); - this.mediaCubeMedia = check(mediaCubeMedia, "mediaCubeMedia"); - check(archiveItem, "archiveItem"); - check(transcoderAddress, "transcoderAddress"); - check(transcoderTemplateName, "transcoderTemplateName"); - check(globalHiresSourcePath, "globalHiresSourcePath"); - check(localLowresTargetPath, "localLowresTargetPath"); - - File sourceMediaFile = new File(archiveItem.getMediaFile()); - logger.info("Transcoding {}", archiveItem.getMediaFile()); - String sourceFileName = sourceMediaFile.getName(); - Timecode timecode = new Timecode(mediaCubeMedia.getLength(), Type.PAL); - - String details = String.format("%s (%s, %d bytes)", sourceFileName, timecode.toString(), sourceMediaFile.length()); - - StoreUri storeUri = store.getTargetStoreUri(RemoteStoreProtocol.LOCAL); - if (storeUri == null) - throw new Exception("Can not detect proxy folder."); - - String webPath = storeUri.toString(true); - - Path targetPath = null; - try { - String targetFileName = FilenameUtils.removeExtension(sourceFileName) + MP4EXT; - targetPath = Paths.get(localLowresTargetPath, targetFileName); - if (!Files.exists(targetPath)) { - // jobRuntime.setDescription(String.format("%s: %s", jobRuntime.getDescription(), details)); - jobRuntime.setDescription(String.format("%s transzkódolása", details)); - String sourceFile = Paths.get(globalHiresSourcePath, sourceFileName).toString(); - IFFAStransAPI api = new FFAStransAPI(transcoderAddress, p -> { - if (p <= 100) - jobRuntime.incrementProgress(p); - }); - - api.submit(transcoderTemplateName, sourceFile); - api.monitor(POLL_INTERVALL); - } - - //a sikeres transzkod utan nem mindig van ott egybol a fajl - long started = System.currentTimeMillis(); - while (!Files.exists(targetPath)) { - long current = System.currentTimeMillis(); - //max 5 perc varakozas - if (current - started > 5 * 60 * 1000) - throw new Exception("Transcode job target file access timed out"); - Thread.sleep(POLL_INTERVALL); - } - - postprocess(targetPath, webPath); - - } catch (Exception e) { - logger.catching(e); - Message m = new ParameterizedMessage("{} átkódolás hiba: {}", sourceFileName, e.getMessage()); - logger.error(marker, m); - throw new Exception(m.getFormattedMessage()); - } finally { - try { - if (deleteSource && sourceMediaFile != null && sourceMediaFile.exists()) - sourceMediaFile.delete(); - } catch (Exception e) { - logger.catching(e); - } - try { - if (deleteSource && targetPath != null && Files.exists(targetPath)) - Files.delete(targetPath); - } catch (Exception e) { - logger.catching(e); - } - } - return null; - } - - private void postprocess(Path transcodedFilePath, String webPath) throws IOException { - Path lowresPath = null; - try { - String transcodedFileName = transcodedFilePath.getFileName().toString(); - String targetPath = null; - if (transcodedFileName.indexOf(".") > 2) { - Path subdir = Paths.get(transcodedFileName.substring(0, 1), transcodedFileName.substring(1, 2), transcodedFileName.substring(2, 3)); - EscortFiles.ensureUNCFolder(webPath, subdir.toString()); - targetPath = Paths.get(subdir.toString(), transcodedFileName).toString(); - } else { - targetPath = transcodedFileName; - } - lowresPath = Paths.get(webPath, targetPath); - int version = 1; - while (Files.exists(lowresPath)) { - String fileName = transcodedFileName + version + MP4EXT; - lowresPath = Paths.get(lowresPath.toString().replace(transcodedFileName, fileName)); - targetPath = targetPath.replace(transcodedFileName, fileName); - transcodedFileName = fileName; - version++; - } - - Files.move(transcodedFilePath, lowresPath); - manager.createMediaFile(targetPath, fileType, store, mediaCubeMedia).add(); - } catch (IOException e) { - logger.catching(e); - logger.error(marker, "A '{}' állomány mozgatása a '{}' helyre nem sikerült.", transcodedFilePath, lowresPath); - throw e; - } - } - -} diff --git a/server/-product/production/OMAR/jobs/steps/TransferStep.java b/server/-product/production/OMAR/jobs/steps/TransferStep.java deleted file mode 100644 index c10084fa..00000000 --- a/server/-product/production/OMAR/jobs/steps/TransferStep.java +++ /dev/null @@ -1,110 +0,0 @@ -package user.jobengine.server.steps; - -import java.nio.file.Path; -import java.nio.file.Paths; - -import org.apache.commons.net.ftp.FTPClient; -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; - -import user.commons.RemoteFile; -import user.commons.StoreUri; -import user.commons.remotestore.FtpDirectoryLister; -import user.commons.remotestore.RemoteStoreProtocol; -import user.jobengine.db.Store; - -public class TransferStep extends JobStep { - private static final String DOT_PART = ".part"; - private static final Logger logger = LogManager.getLogger(); - - private void copy(StoreUri sourceStoreUri, String sourceFileName, StoreUri targetStoreUri, String targetFileName) throws Exception { - String currentTargetFileName = targetFileName; - - boolean renameAfterCopy = false; - boolean renameAfterFTP = false; - if (getTmpExtension() != null) { - if (RemoteStoreProtocol.LOCAL.equals(targetStoreUri.getProtocol())) { - currentTargetFileName += getTmpExtension(); - renameAfterCopy = true; - } - Store targetStore = getManager().getStore(targetStoreUri.getStoreId()); - - if (RemoteStoreProtocol.FTP.equals(targetStoreUri.getProtocol()) && !"NEXIO1".equals(targetStore.getName()) - && !"NEXIO2".equals(targetStore.getName())) { - currentTargetFileName += getTmpExtension(); - renameAfterFTP = true; - } - } - - sourceStoreUri.transferFrom(targetStoreUri, sourceFileName, currentTargetFileName); - - logger.info(getMarker(), "Transfer of {} completed from {} to {}", sourceFileName, sourceStoreUri, currentTargetFileName); - - if (renameAfterCopy) { - Path tmpTargetFile = Paths.get(targetStoreUri.toString(true), currentTargetFileName); - Path targetFile = Paths.get(targetStoreUri.toString(true), targetFileName); - try { - logger.info(getMarker(), "Renaming LOCAL file from {} to {} on {}", currentTargetFileName, targetFileName, sourceStoreUri); - tmpTargetFile.toFile().renameTo(targetFile.toFile()); - } catch (Exception e) { - logger.error(getMarker(), e.getMessage()); - } - } - if (renameAfterFTP) { - try { - FtpDirectoryLister lister = (FtpDirectoryLister) targetStoreUri.getLister(); - FTPClient client = lister.connect(); - logger.info(getMarker(), "Renaming FTP file from {} to {} on {}", currentTargetFileName, targetFileName, sourceStoreUri); - client.rename(currentTargetFileName, targetFileName); - } catch (Exception e) { - logger.error(getMarker(), e.getMessage()); - } finally { - targetStoreUri.cleanUp(); - } - } - } - - @StepEntry - public Object[] execute(StoreUri sourceStoreUri, String sourceFileName, StoreUri targetStoreUri, String targetFileName) throws Exception { - try { - getJobRuntime().setCancelable(false); - - Store sourceStore = getManager().getStore(sourceStoreUri.getStoreId()); - Store targetStore = getManager().getStore(targetStoreUri.getStoreId()); - - setDescription("{} -> {} : {}", sourceStore.getName(), targetStore.getName(), sourceFileName); - - sourceStoreUri.addProgressListener(e -> setProgress(e.getProgress())); - - StoreUri currentTargetStoreUri = getTargetStoreUri(targetStoreUri); - - try { - RemoteFile remoteFile = currentTargetStoreUri.getRemoteFile(targetFileName); - if (remoteFile != null) { - logger.info("File {} already exists on target {}, skipping transfer", targetFileName, targetStore.getName()); - } - } catch (Exception e) { - } - - copy(sourceStoreUri, sourceFileName, currentTargetStoreUri, targetFileName); - } catch (Exception e) { - logger.error(getMarker(), "Error in transfer of {} when copying from {} to {}.", sourceFileName, sourceStoreUri, targetStoreUri); - throw e; - } finally { - getJobRuntime().setDescription(null); - if (sourceStoreUri != null) - sourceStoreUri.cleanUp(); - if (targetStoreUri != null) - targetStoreUri.cleanUp(); - } - return null; - } - - protected StoreUri getTargetStoreUri(StoreUri targetStoreUri) { - return targetStoreUri; - } - - protected String getTmpExtension() { - return DOT_PART; - } -} diff --git a/server/-product/production/OMAR/jobs/steps/UpdateGhostMediaDataStep.java b/server/-product/production/OMAR/jobs/steps/UpdateGhostMediaDataStep.java deleted file mode 100644 index 39cb7b30..00000000 --- a/server/-product/production/OMAR/jobs/steps/UpdateGhostMediaDataStep.java +++ /dev/null @@ -1,89 +0,0 @@ -package user.jobengine.server.steps; - -import java.util.List; - -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; -import org.apache.logging.log4j.Marker; - -import user.commons.remotestore.RemoteStoreProtocol; -import user.jobengine.db.IItemManager; -import user.jobengine.db.Media; -import user.jobengine.db.MediaFile; -import user.jobengine.db.Store; -import user.jobengine.server.IJobEngine; -import user.jobengine.server.IJobRuntime; -import user.jobengine.server.steps.shared.MetadataType; -import user.jobengine.server.steps.shared.MetadataTypeDetector; - -public class UpdateGhostMediaDataStep extends JobStep { - private static final Logger logger = LogManager.getLogger(); - private Marker marker; - - @StepEntry - public Object[] execute(Media mediaCubeMedia, IJobEngine jobEngine, IJobRuntime jobRuntime) throws Exception { - marker = jobRuntime.getSessionMarker(); - - IItemManager manager = jobEngine.getItemManager(); - //Refresh from db - List mediaFiles = manager.getMedia(mediaCubeMedia.getId()).getMediaFiles(); - if (mediaFiles != null && mediaFiles.size() == 2) { - MediaFile lowres = null; - MediaFile highres = null; - - for (MediaFile mf : mediaFiles) { - if (mf.getStore().getSourceStoreUri(RemoteStoreProtocol.HTTP) != null) - lowres = mf; - else - highres = mf; - } - - if (highres == null) { - logger.info(marker, "Nincs highres mediaId: {}", mediaCubeMedia.getId()); - return null; - } - if (lowres == null) { - logger.info(marker, "Nincs lowres mediaId: {}", mediaCubeMedia.getId()); - return null; - } - - String id = MetadataTypeDetector.truncateExtension(highres.getRelativePath()); - id = MetadataTypeDetector.truncateVersion(id); - boolean detect = MetadataTypeDetector.GuessMetadataType(id) == MetadataType.OctopusPlaceholder - || MetadataTypeDetector.GuessMetadataType(id) == MetadataType.OctopusStory; - if (!detect) { - logger.info(marker, "Nem bejátszó mediaId: {}, file: {}", mediaCubeMedia.getId(), highres.getRelativePath()); - return null; - } - - Store highresStore = manager.getSystemStore(false); - final long sourceMediaId = lowres.getId(); - final long highresMediaFileId = highres.getId(); - final String highresRealtivePath = highres.getRelativePath(); - - manager.executeQuery("SELECT mediaid FROM mediafile WHERE relativepath=? and storeid=? and id!=?", rs -> { - long mediaId = rs.getLong(1); - Media media = manager.getMedia(mediaId); - if (media.getMediaFilesCount() == 1) { - logger.info(marker, "Hiányzó szellem lowres hozzáadása {} alapján", media.getId()); - - MediaFile mf = (MediaFile) manager.get(MediaFile.class, sourceMediaId); - mf.setMedia(media); - mf.setId(0); - manager.add(mf); - media.setLength(mediaCubeMedia.getLength()); - manager.modify(media); - } - return true; - }, st -> { - st.setString(1, highresRealtivePath); - st.setLong(2, highresStore.getId()); - st.setLong(3, highresMediaFileId); - }); - - } - - return null; - } - -} diff --git a/server/-product/production/OMAR/jobs/steps/ValidateProResStep.java b/server/-product/production/OMAR/jobs/steps/ValidateProResStep.java deleted file mode 100644 index e36f784d..00000000 --- a/server/-product/production/OMAR/jobs/steps/ValidateProResStep.java +++ /dev/null @@ -1,32 +0,0 @@ -package user.jobengine.server.steps; - -import java.util.HashMap; -import java.util.Map; - -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; - -import user.commons.mediaarea.MediaArea; - -public class ValidateProResStep extends JobStep { - private static final Logger logger = LogManager.getLogger(); - - @StepEntry - public Object[] execute(Map proResFiles) { - Map files = new HashMap(); - - if (!proResFiles.isEmpty()) { - for (String fileName : proResFiles.keySet()) { - MediaArea mediaArea = proResFiles.get(fileName); - String videoFormat = mediaArea.getFormat(); - - if (mediaArea.getFrameRate() == 23.976 && videoFormat.equals("PRORES")) { - files.put(fileName, mediaArea); - } - } - } else { - logger.info("proResFiles is empty!"); - } - return new Object[] { files }; - } -} diff --git a/server/-product/production/OMAR/jobs/steps/shared/EscortFiles.java b/server/-product/production/OMAR/jobs/steps/shared/EscortFiles.java deleted file mode 100644 index c888f722..00000000 --- a/server/-product/production/OMAR/jobs/steps/shared/EscortFiles.java +++ /dev/null @@ -1,348 +0,0 @@ -package user.jobengine.server.steps.shared; - -import java.io.ByteArrayInputStream; -import java.io.File; -import java.io.IOException; -import java.io.InputStream; -import java.io.OutputStream; -import java.io.StringWriter; -import java.io.UnsupportedEncodingException; -import java.nio.file.Files; -import java.nio.file.Path; -import java.nio.file.Paths; -import java.nio.file.attribute.FileAttribute; -import java.nio.file.attribute.PosixFilePermission; -import java.nio.file.attribute.PosixFilePermissions; -import java.text.SimpleDateFormat; -import java.util.Calendar; -import java.util.Date; -import java.util.Set; - -import javax.xml.parsers.DocumentBuilder; -import javax.xml.parsers.DocumentBuilderFactory; -import javax.xml.transform.OutputKeys; -import javax.xml.transform.Transformer; -import javax.xml.transform.TransformerConfigurationException; -import javax.xml.transform.TransformerException; -import javax.xml.transform.TransformerFactory; -import javax.xml.transform.TransformerFactoryConfigurationError; -import javax.xml.transform.dom.DOMSource; -import javax.xml.transform.stream.StreamResult; - -import org.apache.commons.lang.StringUtils; -import org.apache.commons.net.ftp.FTPClient; -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; -import org.apache.logging.log4j.Marker; -import org.apache.logging.log4j.message.Message; -import org.w3c.dom.DOMImplementation; -import org.w3c.dom.Document; -import org.w3c.dom.Element; - -import com.ibm.nosql.json.api.BasicDBObject; -import com.ibm.nosql.json.util.JSON; - -import user.commons.CalendarUtils; -import user.commons.log4j2.marker.MediaCubeMarker; -import user.commons.StoreUri; -import user.commons.remotestore.FtpDirectoryLister; - -public class EscortFiles { - private static final String RECORDTIMESTAMP = "RecordTimeStamp"; - private static final String MODIFIEDTIMESTAMP = "ModifiedTimeStamp"; - public static final String DOT_CATCHED = ".catched"; - public static final String DOT_JSON = ".json"; - private static final Logger logger = LogManager.getLogger(); - private static final String EXTENDEDAGENCY = "ExtendedAgency"; - private static final String EXTENDEDDESCRIPTION = "ExtendedDescription"; - private static final String KILLDATE = "KillDate"; - private static final String FORMAT_KILLDATE = "MM-dd-yyyy"; - private static final String EXTENDEDID = "extendedId"; - private static final String ID = "ID"; - private static final String KILLDATE_FILENAME = "%s.%s.killdate"; - private static final String FORMAT_KILLDATENAME = "yyyyMMdd"; - public static final String STATUSFOLDER = ".STATUS"; - public static final String CONFLICTFOLDER = ".CONFLICT"; - - public static String composeKillDate(int days) { - Calendar killDate = Calendar.getInstance(); - killDate.add(Calendar.DAY_OF_YEAR, days); - SimpleDateFormat dateFormat = new SimpleDateFormat(FORMAT_KILLDATENAME); - return dateFormat.format(killDate.getTime()); - } - /* - * - * 02-02-2018 TEST - * TEST AGENT AGENT - */ - - private static String composeKillDateFileName(String fileName, int days) { - return String.format(KILLDATE_FILENAME, fileName, composeKillDate(days)); - } - - public static void createCatchedFile(Path escortFile) throws IOException { - Path catchedFilePath = Paths.get(escortFile.toString() + DOT_CATCHED); - Files.createFile(catchedFilePath); - } - - public static void createFellow(String escortFile, String extension) throws IOException { - Files.copy(Paths.get(escortFile), Paths.get(escortFile + "." + extension)); - } - - /*** - * A media eleresi utjan alapjan a .STATUS almappaban letrehozza a .catch fajlt. - * - * @param mediaFile - * @throws IOException - */ - public static void createMediaCatch(Path mediaFile) throws IOException { - Path catchedFile = createMediaCatchFilePath(mediaFile); - ensureUNCFolder(catchedFile.getParent()); - Files.createFile(catchedFile); - } - - public static Path createMediaCatchFilePath(Path mediaFile) { - String fileName = mediaFile.getFileName().toString() + DOT_CATCHED; - return Paths.get(mediaFile.getParent().toString(), STATUSFOLDER, fileName); - } - - public static void createMetadata(String filePath, String fileName, String metadata) throws IOException { - ensureUNCFolder(filePath, STATUSFOLDER); - String metadataFileName = fileName + DOT_JSON; - Path metadataPath = Paths.get(filePath, STATUSFOLDER, metadataFileName); - Files.write(metadataPath, metadata.getBytes()); - } - - public static boolean createMetadataIfNotExists(String filePath, String fileName, String metadata) - throws IOException { - boolean result = false; - if (!EscortFiles.isMetadataExists(filePath, fileName)) { - EscortFiles.createMetadata(filePath, fileName, metadata); - result = true; - } - return result; - } - - public static void createMorpheusXML(String filePath, String fileName, String content) throws IOException { - ensureUNCFolder(filePath, STATUSFOLDER); - Path xmlPath = Paths.get(filePath, fileName); - if (Files.exists(xmlPath)) - throw new IOException(String.format("Az '%s' állomány már létezik.", xmlPath)); - Files.write(xmlPath, content.getBytes()); - } - - public static byte[] createNEXIODatesMeta(String fileName, Date recorded, Date modified) throws Exception { - DocumentBuilderFactory dbf = DocumentBuilderFactory.newInstance(); - DocumentBuilder db = dbf.newDocumentBuilder(); - DOMImplementation impl = db.getDOMImplementation(); - Document xmlDocument = impl.createDocument(null, null, null); - - Element root = xmlDocument.createElement(ID); - root.setAttribute(EXTENDEDID, fileName); - // 07-13-2020 (19:36:52) - // 05-18-2013 (18:52:24) - SimpleDateFormat df = new SimpleDateFormat("MM-dd-yyyy (HH:mm:ss)"); - root.appendChild(xmlDocument.createElement(MODIFIEDTIMESTAMP)) - .appendChild(xmlDocument.createTextNode(df.format(modified))); - root.appendChild(xmlDocument.createElement(RECORDTIMESTAMP)) - .appendChild(xmlDocument.createTextNode(df.format(recorded))); - xmlDocument.appendChild(root); - - return xmDocumentToString(xmlDocument); - } - - public static byte[] createNEXIOKillDateFile(String fileName, Date killDate, String description, String agency) - throws Exception { - DocumentBuilderFactory dbf = DocumentBuilderFactory.newInstance(); - DocumentBuilder db = dbf.newDocumentBuilder(); - DOMImplementation impl = db.getDOMImplementation(); - Document xmlDocument = impl.createDocument(null, null, null); - - Element root = xmlDocument.createElement(ID); - root.setAttribute(EXTENDEDID, fileName); - if (killDate != null) { - String sKillDate = CalendarUtils.toString(CalendarUtils.createCalendar(killDate), FORMAT_KILLDATE); - root.appendChild(xmlDocument.createElement(KILLDATE)).appendChild(xmlDocument.createTextNode(sKillDate)); - } - - if (StringUtils.isNotBlank(description)) - root.appendChild(xmlDocument.createElement(EXTENDEDDESCRIPTION)) - .appendChild(xmlDocument.createTextNode(description)); - if (StringUtils.isNotBlank(agency)) - root.appendChild(xmlDocument.createElement(EXTENDEDAGENCY)).appendChild(xmlDocument.createTextNode(agency)); - xmlDocument.appendChild(root); - - return xmDocumentToString(xmlDocument); - } - - public static Document createNEXIOMeta(byte[] content) throws Exception { - DocumentBuilderFactory dbf = DocumentBuilderFactory.newInstance(); - DocumentBuilder db = dbf.newDocumentBuilder(); - DOMImplementation impl = db.getDOMImplementation(); - Document xmlDocument = null; - - try (InputStream is = new ByteArrayInputStream(content)) { - xmlDocument = db.parse(is); - } catch (Exception e) { - logger.catching(e); - } - - return xmlDocument; - } - - public static void createUNCKillDate(String filePath, String fileName, int days, Marker marker) throws IOException { - ensureUNCFolder(filePath, STATUSFOLDER); - String killDateFileName = composeKillDateFileName(fileName, days); - Path killDatePath = Paths.get(filePath, STATUSFOLDER, killDateFileName); - if (Files.exists(killDatePath)) - logger.warn(marker, "Az '{}' állomány már létezik.", killDatePath); - else - Files.createFile(killDatePath); - } - - @SuppressWarnings("unchecked") - public static T decode(Path escortFile) { - T result = null; - try { - byte[] bytes = Files.readAllBytes(escortFile); - String content = new String(bytes); - result = (T) JSON.parse(content); - } catch (Exception e) { - logger.error("Decode error. System message is: ", e.getMessage()); - } - return result; - } - - public static void ensureUNCFolder(Path filePath) throws IOException { - File folder = filePath.toFile(); - if (!folder.exists() || !folder.isDirectory()) { - try { - Set perms = PosixFilePermissions.fromString("rwxrwxrwx"); - FileAttribute> attr = PosixFilePermissions.asFileAttribute(perms); - Files.createDirectories(filePath, attr); - } catch (Exception e) { - // logger.catching(e); - try { - Files.createDirectories(filePath); - } catch (Exception e1) { - logger.catching(e1); - throw e1; - } - } - } - } - - public static void ensureUNCFolder(String filePath, String folderName) throws IOException { - Path statusPath = Paths.get(filePath, folderName); - ensureUNCFolder(statusPath); - } - - public static boolean isCatchedFileExists(Path escortFile) { - Path catchedFilePath = Paths.get(escortFile.toString() + DOT_CATCHED); - return Files.exists(catchedFilePath); - } - - /*** - * A media elérési útján alapján a .STATUS almappában vizsgálja .catch fajl - * létezését. - * - * @param mediaFile - * @return - */ - public static boolean isMediaCatched(Path mediaFile) { - Path catchedFile = createMediaCatchFilePath(mediaFile); - return Files.exists(catchedFile); - } - - public static boolean isMetadataExists(String filePath, String fileName) throws IOException { - boolean result = false; - String metadataFileName = fileName + DOT_JSON; - Path metadataPath = Paths.get(filePath, STATUSFOLDER, metadataFileName); - result = Files.exists(metadataPath); - return result; - } - - public static void notifyRecipient(Path escortFile, Logger logger, Message msg) { - if (Files.exists(escortFile)) { - try { - BasicDBObject downloadable = EscortFiles.decode(escortFile); - String recipientKey = "recipient"; - if (downloadable.containsKey(recipientKey)) { - String recipient = downloadable.getString(recipientKey); - logger.info(new MediaCubeMarker(recipient, "MediaCube rendszerüzenet"), msg); - } - } catch (Exception e) { - logger.catching(e); - } - - } - } - - public static void remove(Path file) { - try { - file.toFile().delete(); - } catch (Exception e) { - logger.error("Unable to delete {}", file.toAbsolutePath().toString()); - } - } - - public static void removeCatchedFile(Path escortFile) { - remove(Paths.get(escortFile.toString() + DOT_CATCHED)); - } - - /*** - * A media eleresi utjan alapjan a .STATUS almappabol torli a .catch fajlt. - * - * @param mediaFile - * @throws IOException - */ - public static void removeMediaCatch(Path mediaFile) { - Path catchedFile = createMediaCatchFilePath(mediaFile); - remove(catchedFile); - } - - public static void setNEXIOKillDate(int killDateDays, String targetFileName, String nexioAgency, StoreUri targetUri) - throws Exception { - OutputStream outStream = null; - try { - FTPClient targetFTP = ((FtpDirectoryLister) targetUri.getLister()).connect(); - Calendar killDate = CalendarUtils.createCalendar(new Date()); - killDate.add(Calendar.DAY_OF_YEAR, killDateDays); - if (targetFileName.toLowerCase().contains(".mxf")) - targetFileName = targetFileName.substring(0, targetFileName.lastIndexOf('.')); - byte[] killDateFile = EscortFiles.createNEXIOKillDateFile(targetFileName, killDate.getTime(), null, - nexioAgency); - String xml = targetFileName + ".xml"; - outStream = targetFTP.storeFileStream(xml); - if (outStream == null) { - throw new NullPointerException( - "Can not open: " + targetFileName.substring(0, targetFileName.lastIndexOf('.')) + ".xml" - + " Reply:" + targetFTP.getReplyString()); - } - outStream.write(killDateFile); - outStream.flush(); - } catch (Exception e) { - throw e; - } finally { - if (outStream != null) - outStream.close(); - targetUri.cleanUp(); - } - } - - private static byte[] xmDocumentToString(Document xmlDocument) throws TransformerFactoryConfigurationError, - TransformerConfigurationException, TransformerException, IOException, UnsupportedEncodingException { - DOMSource domSource = new DOMSource(xmlDocument); - TransformerFactory tf = TransformerFactory.newInstance(); - Transformer transformer = tf.newTransformer(); - transformer.setOutputProperty(OutputKeys.METHOD, "xml"); - transformer.setOutputProperty(OutputKeys.ENCODING, "UTF-16"); - transformer.setOutputProperty(OutputKeys.INDENT, "yes"); - StringWriter sw = new StringWriter(); - StreamResult sr = new StreamResult(sw); - transformer.transform(domSource, sr); - String result = sw.toString(); - sw.close(); - return result.getBytes("UTF-16"); - } -} diff --git a/server/-product/production/OMAR/jobs/steps/shared/ExternalCommand.java b/server/-product/production/OMAR/jobs/steps/shared/ExternalCommand.java deleted file mode 100644 index 307f36d0..00000000 --- a/server/-product/production/OMAR/jobs/steps/shared/ExternalCommand.java +++ /dev/null @@ -1,79 +0,0 @@ -package user.jobengine.server.steps.shared; - -import java.io.BufferedReader; -import java.io.InputStreamReader; -import java.util.ArrayList; -import java.util.List; - -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; - -public class ExternalCommand { - private static final Logger logger = LogManager.getLogger(); - private ExternalProfile profile; - - public ExternalCommand(ExternalProfile profile) { - this.profile = profile; - } - - public String execute(String input, String output, boolean firstResponse, IExternalCallback responseCallBack) throws Exception { - List arguments = getArguments(input, output); - List command = new ArrayList<>(); - command.add(profile.getExecutable()); - command.addAll(arguments); - - ProcessBuilder processBuilder = new ProcessBuilder(); - processBuilder.command(command); - - String result = null; - try { - logger.info("Executing : {}", processBuilder.command()); - - Process process = processBuilder.start(); - try (BufferedReader reader = new BufferedReader(new InputStreamReader(process.getInputStream()))) { - String line = null; - while ((line = reader.readLine()) != null) { - logger.debug("Process response: {}", line); - if (responseCallBack != null) - responseCallBack.onResponse(line); - //System.out.println(line); - if (line != null && line.length() > 0) { - result = line; - if (firstResponse) - break; - } - } - int exitCode = process.waitFor(); - if (exitCode != 0) { - StringBuilder msg = new StringBuilder(); - try (BufferedReader errReader = new BufferedReader(new InputStreamReader(process.getErrorStream()))) { - String errline = null; - while ((errline = errReader.readLine()) != null) { - msg.append(errline); - } - } catch (Exception ex) { - } - - throw new Exception("Exited with error code : " + exitCode + ". " + msg); - } - } catch (Exception e) { - throw e; - } - } catch (Exception e) { - logger.error(e); - throw e; - } - - return result; - } - - private List getArguments(String input, String output) { - List result = new ArrayList<>(); - - profile.getArguments().forEach(i -> { - result.add(i.replace("%i", input).replace("%o", output)); - }); - return result; - } - -} \ No newline at end of file diff --git a/server/-product/production/OMAR/jobs/steps/shared/ExternalCommandExecutor.java b/server/-product/production/OMAR/jobs/steps/shared/ExternalCommandExecutor.java deleted file mode 100644 index a34ff7f0..00000000 --- a/server/-product/production/OMAR/jobs/steps/shared/ExternalCommandExecutor.java +++ /dev/null @@ -1,32 +0,0 @@ -package user.jobengine.server.steps.shared; - -import user.commons.configuration.SystemConfiguration; - -public class ExternalCommandExecutor { - - public void execute(String profileName, String input, String output, IExternalCallback responseCallBack) throws Exception { - ExternalCommand externalCommand = getExternalCommand(profileName); - externalCommand.execute(input, output, false, responseCallBack); - } - - private ExternalCommand getExternalCommand(String profileName) throws Exception { - ExternalProfilesConfig config = SystemConfiguration.getInstance().load("settings/external-commands.yaml", ExternalProfilesConfig.class); - - if (config == null) - throw new Exception("Missing external-commands.yaml configuration"); - - ExternalProfile selectedProfile = null; - for (ExternalProfile profile : config.getProfiles()) { - if (profileName.equals(profile.getName())) { - selectedProfile = profile; - break; - } - } - - if (selectedProfile == null) - throw new Exception("Missing profile " + profileName + " in external-commands.yaml configuration"); - - return new ExternalCommand(selectedProfile); - } - -} diff --git a/server/-product/production/OMAR/jobs/steps/shared/ExternalProfile.java b/server/-product/production/OMAR/jobs/steps/shared/ExternalProfile.java deleted file mode 100644 index 68e22f4c..00000000 --- a/server/-product/production/OMAR/jobs/steps/shared/ExternalProfile.java +++ /dev/null @@ -1,33 +0,0 @@ -package user.jobengine.server.steps.shared; - -import java.util.List; - -public class ExternalProfile { - private String executable; - private String name; - private List arguments; - - public List getArguments() { - return arguments; - } - - public String getExecutable() { - return executable; - } - - public String getName() { - return name; - } - - public void setArguments(List arguments) { - this.arguments = arguments; - } - - public void setExecutable(String executable) { - this.executable = executable; - } - - public void setName(String name) { - this.name = name; - } -} \ No newline at end of file diff --git a/server/-product/production/OMAR/jobs/steps/shared/ExternalProfilesConfig.java b/server/-product/production/OMAR/jobs/steps/shared/ExternalProfilesConfig.java deleted file mode 100644 index 7ef77291..00000000 --- a/server/-product/production/OMAR/jobs/steps/shared/ExternalProfilesConfig.java +++ /dev/null @@ -1,15 +0,0 @@ -package user.jobengine.server.steps.shared; - -import java.util.List; - -public class ExternalProfilesConfig { - private List profiles; - - public List getProfiles() { - return profiles; - } - - public void setProfiles(List profiles) { - this.profiles = profiles; - } -} \ No newline at end of file diff --git a/server/-product/production/OMAR/jobs/steps/shared/FileSearchFilterOptions.java b/server/-product/production/OMAR/jobs/steps/shared/FileSearchFilterOptions.java deleted file mode 100644 index b5a8d0e8..00000000 --- a/server/-product/production/OMAR/jobs/steps/shared/FileSearchFilterOptions.java +++ /dev/null @@ -1,41 +0,0 @@ -package user.jobengine.server.steps.shared; - -import java.nio.file.Path; -import java.util.regex.Matcher; -import java.util.regex.Pattern; - -import com.ibm.nosql.json.api.BasicDBObject; - -public class FileSearchFilterOptions { - - private BasicDBObject filter; - - public FileSearchFilterOptions(BasicDBObject filter) { - this.filter = filter; - } - - public boolean acceptFile(Path file) { - if (filter == null) - return true; - - if (filter.containsKey("fileName")) { - //.*\.(sh|ini|conf|vhost|xml|php)$ - String fileNamePattern = filter.getString("fileName"); - if (fileNamePattern == null || fileNamePattern.trim().length() == 0) - return true; - - Pattern pattern = Pattern.compile(fileNamePattern, Pattern.CASE_INSENSITIVE); - - Matcher matcher = pattern.matcher(file.getFileName().toString()); - if (matcher.find()) - return true; - - } - - return false; - } - - public boolean preAcceptDirectory(Path file) { - return true; - } -} diff --git a/server/-product/production/OMAR/jobs/steps/shared/IExternalCallback.java b/server/-product/production/OMAR/jobs/steps/shared/IExternalCallback.java deleted file mode 100644 index 1e071913..00000000 --- a/server/-product/production/OMAR/jobs/steps/shared/IExternalCallback.java +++ /dev/null @@ -1,5 +0,0 @@ -package user.jobengine.server.steps.shared; - -public interface IExternalCallback { - void onResponse(String data); -} \ No newline at end of file diff --git a/server/-product/production/OMAR/jobs/steps/shared/ItemManagerExtensions.java b/server/-product/production/OMAR/jobs/steps/shared/ItemManagerExtensions.java deleted file mode 100644 index 32f17cb5..00000000 --- a/server/-product/production/OMAR/jobs/steps/shared/ItemManagerExtensions.java +++ /dev/null @@ -1,94 +0,0 @@ -package user.jobengine.server.steps.shared; - -import java.nio.file.Path; - -import com.ibm.nosql.json.api.BasicDBObject; - -import user.commons.RemoteFile; -import user.commons.StoreUri; -import user.commons.remotestore.RemoteStoreProtocol; -import user.jobengine.db.IItemManager; -import user.jobengine.db.IResultSetConsumer; -import user.jobengine.db.Store; - -public class ItemManagerExtensions { - - public static BasicDBObject getArchiveInfo(IItemManager manager, long houseid) { - final BasicDBObject[] result = { null }; - StringBuilder query = new StringBuilder(); - query.append("select count(*) as count, sum(length) as duration FROM media"); - query.append(" "); - query.append(String.format("where houseid='%d' and itemtypeid = 82", houseid)); - query.append(" "); - query.append("group by houseid"); - IResultSetConsumer consumer = rs -> { - BasicDBObject o = new BasicDBObject(); - o.put("count", rs.getLong("count")); - o.put("duration", rs.getLong("duration")); - result[0] = o; - return false; - }; - manager.executeQuery(query.toString(), consumer, null); - return result[0]; - } - - public static long getExistingRundownMedia(IItemManager manager, String houseid) { - final long[] result = new long[] { 0 }; - final String[] idToCheck = new String[] { houseid }; - int pos = houseid.lastIndexOf("-"); - //a hivas a CopyForArchiveNEXIOMaterialsStep-bol is johet, ott meg nincs idobelyegezve a nev! - if (pos > 0 && houseid.length() - pos > 4) - idToCheck[0] = houseid.substring(0, pos); - MetadataType metadataType = MetadataTypeDetector.GuessMetadataType(idToCheck[0]); - if (metadataType == MetadataType.OctopusPlaceholder) { - StringBuilder query = new StringBuilder(); - query.append("select mediaid, mediafilehouseid, filename"); - query.append(" "); - query.append(String.format("from vw_rundown_items where mediafilehouseid like '%s%%'", idToCheck[0])); - query.append(" "); - query.append("order by filename, mediaid"); - IResultSetConsumer consumer = rs -> { - String fileName = rs.getString("filename"); - if (idToCheck[0].equals(fileName)) { - result[0] = rs.getLong("mediaid"); - return false; - } else - return true; - }; - manager.executeQuery(query.toString(), consumer, null); - } - - return result[0]; - } - - static public boolean isArchived(IItemManager manager, Path filePath) { - boolean result = false; - String name = filePath.getFileName().toString(); - String[] tsmName = new String[] { null }; - String query = String.format("SELECT relativepath FROM MEDIAFILE WHERE houseid = '%s'", name); - manager.executeQuery(query, rs -> { - tsmName[0] = rs.getString("relativepath"); - return false; - }, null); - - Store tsmStore = manager.getSystemStore(false); - if (tsmStore == null) - throw new NullPointerException("A TSM bejegyzés nem található!"); - - StoreUri tsmStoreUri = tsmStore.getSourceStoreUri(RemoteStoreProtocol.TSM); - if (tsmStoreUri == null) - throw new NullPointerException("A TSM forrás elérése nem található!"); - - if (tsmName[0] != null) { - try { - RemoteFile remoteFile = tsmStoreUri.getRemoteFile(tsmName[0]); - result = remoteFile != null; - } catch (Exception e) { - result = false; - } finally { - tsmStoreUri.cleanUp(); - } - } - return result; - } -} diff --git a/server/-product/production/OMAR/jobs/steps/shared/MediaCubeClient.java b/server/-product/production/OMAR/jobs/steps/shared/MediaCubeClient.java deleted file mode 100644 index 0915cc33..00000000 --- a/server/-product/production/OMAR/jobs/steps/shared/MediaCubeClient.java +++ /dev/null @@ -1,77 +0,0 @@ -package user.jobengine.server.steps.shared; - -import javax.ws.rs.client.Entity; -import javax.ws.rs.client.Invocation.Builder; -import javax.ws.rs.core.MediaType; -import javax.ws.rs.core.MultivaluedMap; -import javax.ws.rs.core.Response; -import javax.ws.rs.core.Response.Status; - -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; -import org.jboss.resteasy.client.jaxrs.ResteasyClient; -import org.jboss.resteasy.client.jaxrs.ResteasyClientBuilder; -import org.jboss.resteasy.client.jaxrs.ResteasyWebTarget; -import org.jboss.resteasy.specimpl.MultivaluedMapImpl; - -import com.ibm.nosql.json.JSONUtil; -import com.ibm.nosql.json.api.BasicDBObject; - -public class MediaCubeClient { - private static Logger logger = LogManager.getLogger(); - private ResteasyWebTarget webTarget; - - public MediaCubeClient(String address) { - ResteasyClient client = new ResteasyClientBuilder().build(); - webTarget = client.target(address); - } - - BasicDBObject getDbObject(String json) { - BasicDBObject result = (BasicDBObject) JSONUtil.jsonToDbObject(json); - - if (result == null) - throw new NullPointerException("API Result is null!"); - - if (result.containsKey("exception")) { - BasicDBObject e = (BasicDBObject) result.get("exception"); - throw new RuntimeException(e.getString("message")); - } - //{"exception":{"message":"Invalid credentials.","publicName":"AuthenticationFailedException"}} - return result; - } - - public BasicDBObject getStatus(long jobId) { - MultivaluedMap vars = new MultivaluedMapImpl<>(); - vars.add("jobId", jobId); - Response response = query("services/rest/jobengine/jobstatus", vars).get(); - if (response.getStatus() != Status.OK.getStatusCode()) { - logger.error(response.readEntity(String.class)); - System.out.println(response.readEntity(String.class)); - return null; - } - String result = response.readEntity(String.class); - return getDbObject(result); - } - - private Builder query(String path, MultivaluedMap vars) { - ResteasyWebTarget target = webTarget.path(path).queryParams(vars); - Builder result = target.request(); - return result; - } - - public long startjob(String template, String name, BasicDBObject jobParams) throws Exception { - MultivaluedMap vars = new MultivaluedMapImpl<>(); - vars.add("template", template); - vars.add("name", name); - Response response = query("services/rest/jobengine/startjob", vars).post(Entity.entity(jobParams.toString(), MediaType.APPLICATION_JSON)); - - if (response.getStatus() != Status.OK.getStatusCode()) { - logger.error(response.readEntity(String.class)); - return 0; - } - - String resultObject = response.readEntity(String.class); - return Long.parseLong(resultObject); - } - -} diff --git a/server/-product/production/OMAR/jobs/steps/shared/MediaFileSearchFilterOptions.java b/server/-product/production/OMAR/jobs/steps/shared/MediaFileSearchFilterOptions.java deleted file mode 100644 index ee467e5b..00000000 --- a/server/-product/production/OMAR/jobs/steps/shared/MediaFileSearchFilterOptions.java +++ /dev/null @@ -1,42 +0,0 @@ -package user.jobengine.server.steps.shared; - -import java.nio.file.Path; -import java.util.regex.Matcher; -import java.util.regex.Pattern; - -import com.ibm.nosql.json.api.BasicDBObject; - -public class MediaFileSearchFilterOptions extends FileSearchFilterOptions { - - private BasicDBObject filter; - - public MediaFileSearchFilterOptions(BasicDBObject filter) { - super(filter); - } - - @Override - public boolean acceptFile(Path file) { - if (filter == null) - return true; - - if (filter.containsKey("fileName")) { - // .*\.(sh|ini|conf|vhost|xml|php)$ - String fileNamePattern = filter.getString("fileName"); - if (fileNamePattern == null || fileNamePattern.trim().length() == 0) - return true; - - Pattern pattern = Pattern.compile(fileNamePattern, Pattern.CASE_INSENSITIVE); - - Matcher matcher = pattern.matcher(file.getFileName().toString()); - if (matcher.find()) - return true; - } - - return false; - } - - @Override - public boolean preAcceptDirectory(Path file) { - return true; - } -} diff --git a/server/-product/production/OMAR/jobs/steps/shared/MetadataType.java b/server/-product/production/OMAR/jobs/steps/shared/MetadataType.java deleted file mode 100644 index 0e27bf34..00000000 --- a/server/-product/production/OMAR/jobs/steps/shared/MetadataType.java +++ /dev/null @@ -1,5 +0,0 @@ -package user.jobengine.server.steps.shared; - -public enum MetadataType { - TrafficMaterial, TrafficPromo, TrafficAD, OctopusStory, OctopusPlaceholder, Generic -} diff --git a/server/-product/production/OMAR/jobs/steps/shared/MetadataTypeDetector.java b/server/-product/production/OMAR/jobs/steps/shared/MetadataTypeDetector.java deleted file mode 100644 index 9e9e6be9..00000000 --- a/server/-product/production/OMAR/jobs/steps/shared/MetadataTypeDetector.java +++ /dev/null @@ -1,49 +0,0 @@ -package user.jobengine.server.steps.shared; - -import org.apache.commons.lang.StringUtils; - -public class MetadataTypeDetector { - - private static final String HYPHEN = "-"; - private static final String DOT = "."; - - private static final String REGEXP_TRAFFICMATERIALID = "^M{1}[0-9]{6}[A-Z]{1}"; - private static final String REGEXP_TRAFFICADID = "^R{1}[0-9]{6}[A-Z]{1}"; - private static final String REGEXP_TRAFFICPROMOID = "^P{1}[0-9]{6}[A-Z]{1}"; - private static final String REGEXP_OCTOPUSSTORYID = "^[0-9]+"; - private static final String REGEXP_OCTOPUSPLACEHOLDERID = "^[0-9]+_[0-9]+"; - private static final String REGEXP_OCTOPUSPLACEHOLDERVERSIONEDID = "^[0-9]+_[0-9]+-[0-9]{3}"; - - public static MetadataType GuessMetadataType(String id) { - if (StringUtils.isBlank(id)) - return MetadataType.Generic; - if (id.matches(REGEXP_TRAFFICMATERIALID)) - return MetadataType.TrafficMaterial; - if (id.matches(REGEXP_TRAFFICPROMOID)) - return MetadataType.TrafficPromo; - if (id.matches(REGEXP_TRAFFICADID)) - return MetadataType.TrafficAD; - if (id.matches(REGEXP_OCTOPUSSTORYID)) - return MetadataType.OctopusStory; - if (id.matches(REGEXP_OCTOPUSPLACEHOLDERID)) - return MetadataType.OctopusPlaceholder; - if (id.matches(REGEXP_OCTOPUSPLACEHOLDERVERSIONEDID)) - return MetadataType.OctopusPlaceholder; - return MetadataType.Generic; - } - - public static String truncateExtension(String name) { - String result = name; - if (result != null && result.contains(DOT)) - result = result.substring(0, result.lastIndexOf(DOT)); - return result; - } - - public static String truncateVersion(String name) { - String result = name; - if (result != null && result.contains(HYPHEN)) - result = result.split(HYPHEN)[0]; - return result; - } - -} diff --git a/server/-product/production/OMAR/jobs/steps/shared/TestLib.java b/server/-product/production/OMAR/jobs/steps/shared/TestLib.java deleted file mode 100644 index d12738ae..00000000 --- a/server/-product/production/OMAR/jobs/steps/shared/TestLib.java +++ /dev/null @@ -1,9 +0,0 @@ -package user.jobengine.server.steps.shared; - -public class TestLib { - - public void hello() { - TestLib1 lib = new TestLib1(); - lib.helo(); - } -} diff --git a/server/-product/production/OMAR/jobs/steps/shared/TestLib1.java b/server/-product/production/OMAR/jobs/steps/shared/TestLib1.java deleted file mode 100644 index 99d47e5f..00000000 --- a/server/-product/production/OMAR/jobs/steps/shared/TestLib1.java +++ /dev/null @@ -1,8 +0,0 @@ -package user.jobengine.server.steps.shared; - -public class TestLib1 { - - public void hello() { - System.out.println("Hello from lib1"); - } -} diff --git a/server/-product/production/OMAR/jobs/templates/archive-limited.xml b/server/-product/production/OMAR/jobs/templates/archive-limited.xml deleted file mode 100644 index 6ea81306..00000000 --- a/server/-product/production/OMAR/jobs/templates/archive-limited.xml +++ /dev/null @@ -1,43 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - \ No newline at end of file diff --git a/server/-product/production/OMAR/jobs/templates/archive-material.xml b/server/-product/production/OMAR/jobs/templates/archive-material.xml deleted file mode 100644 index 973ac9fa..00000000 --- a/server/-product/production/OMAR/jobs/templates/archive-material.xml +++ /dev/null @@ -1,49 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - \ No newline at end of file diff --git a/server/-product/production/OMAR/jobs/templates/archive-ondemand.xml b/server/-product/production/OMAR/jobs/templates/archive-ondemand.xml deleted file mode 100644 index 54555f6c..00000000 --- a/server/-product/production/OMAR/jobs/templates/archive-ondemand.xml +++ /dev/null @@ -1,43 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - \ No newline at end of file diff --git a/server/-product/production/OMAR/jobs/templates/archive-recursive.xml b/server/-product/production/OMAR/jobs/templates/archive-recursive.xml deleted file mode 100644 index 02b97884..00000000 --- a/server/-product/production/OMAR/jobs/templates/archive-recursive.xml +++ /dev/null @@ -1,25 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - \ No newline at end of file diff --git a/server/-product/production/OMAR/jobs/templates/batch-retrieve-ondemand.xml b/server/-product/production/OMAR/jobs/templates/batch-retrieve-ondemand.xml deleted file mode 100644 index 20332cf0..00000000 --- a/server/-product/production/OMAR/jobs/templates/batch-retrieve-ondemand.xml +++ /dev/null @@ -1,29 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - \ No newline at end of file diff --git a/server/-product/production/OMAR/jobs/templates/calculatemd5.xml b/server/-product/production/OMAR/jobs/templates/calculatemd5.xml deleted file mode 100644 index 637c7638..00000000 --- a/server/-product/production/OMAR/jobs/templates/calculatemd5.xml +++ /dev/null @@ -1,19 +0,0 @@ - - - - - - - - - - - - - - - - - - - \ No newline at end of file diff --git a/server/-product/production/OMAR/jobs/templates/cancelable.xml b/server/-product/production/OMAR/jobs/templates/cancelable.xml deleted file mode 100644 index 2be289b3..00000000 --- a/server/-product/production/OMAR/jobs/templates/cancelable.xml +++ /dev/null @@ -1,17 +0,0 @@ - - - - - - - - - - - - - - - - - \ No newline at end of file diff --git a/server/-product/production/OMAR/jobs/templates/common-copy.xml b/server/-product/production/OMAR/jobs/templates/common-copy.xml deleted file mode 100644 index 7abb6fcf..00000000 --- a/server/-product/production/OMAR/jobs/templates/common-copy.xml +++ /dev/null @@ -1,41 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - \ No newline at end of file diff --git a/server/-product/production/OMAR/jobs/templates/create-lowres-ondemand.xml b/server/-product/production/OMAR/jobs/templates/create-lowres-ondemand.xml deleted file mode 100644 index f7922734..00000000 --- a/server/-product/production/OMAR/jobs/templates/create-lowres-ondemand.xml +++ /dev/null @@ -1,106 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - \ No newline at end of file diff --git a/server/-product/production/OMAR/jobs/templates/create-proxy-ffmpeg.xml b/server/-product/production/OMAR/jobs/templates/create-proxy-ffmpeg.xml deleted file mode 100644 index acb2cb62..00000000 --- a/server/-product/production/OMAR/jobs/templates/create-proxy-ffmpeg.xml +++ /dev/null @@ -1,25 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - \ No newline at end of file diff --git a/server/-product/production/OMAR/jobs/templates/delete-materials.xml b/server/-product/production/OMAR/jobs/templates/delete-materials.xml deleted file mode 100644 index 365fe19c..00000000 --- a/server/-product/production/OMAR/jobs/templates/delete-materials.xml +++ /dev/null @@ -1,17 +0,0 @@ - - - - - - - - - - - - - - - - - \ No newline at end of file diff --git a/server/-product/production/OMAR/jobs/templates/dummy-test-job.xml b/server/-product/production/OMAR/jobs/templates/dummy-test-job.xml deleted file mode 100644 index 2c5d5a5d..00000000 --- a/server/-product/production/OMAR/jobs/templates/dummy-test-job.xml +++ /dev/null @@ -1,47 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - \ No newline at end of file diff --git a/server/-product/production/OMAR/jobs/templates/integration-test.xml b/server/-product/production/OMAR/jobs/templates/integration-test.xml deleted file mode 100644 index 6593e5d7..00000000 --- a/server/-product/production/OMAR/jobs/templates/integration-test.xml +++ /dev/null @@ -1,6 +0,0 @@ - - - - - - \ No newline at end of file diff --git a/server/-product/production/OMAR/jobs/templates/metadata-updater.xml b/server/-product/production/OMAR/jobs/templates/metadata-updater.xml deleted file mode 100644 index f516c3e2..00000000 --- a/server/-product/production/OMAR/jobs/templates/metadata-updater.xml +++ /dev/null @@ -1,6 +0,0 @@ - - - - - - \ No newline at end of file diff --git a/server/-product/production/OMAR/jobs/templates/migrate-hsm.xml b/server/-product/production/OMAR/jobs/templates/migrate-hsm.xml deleted file mode 100644 index faec8724..00000000 --- a/server/-product/production/OMAR/jobs/templates/migrate-hsm.xml +++ /dev/null @@ -1,21 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - \ No newline at end of file diff --git a/server/-product/production/OMAR/jobs/templates/move-jpeg-to-isilon.xml b/server/-product/production/OMAR/jobs/templates/move-jpeg-to-isilon.xml deleted file mode 100644 index ae74dab8..00000000 --- a/server/-product/production/OMAR/jobs/templates/move-jpeg-to-isilon.xml +++ /dev/null @@ -1,51 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - \ No newline at end of file diff --git a/server/-product/production/OMAR/jobs/templates/prores-archive.xml b/server/-product/production/OMAR/jobs/templates/prores-archive.xml deleted file mode 100644 index 2d49bcba..00000000 --- a/server/-product/production/OMAR/jobs/templates/prores-archive.xml +++ /dev/null @@ -1,76 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - \ No newline at end of file diff --git a/server/-product/production/OMAR/jobs/templates/remote-transcode.xml b/server/-product/production/OMAR/jobs/templates/remote-transcode.xml deleted file mode 100644 index ce4c77b7..00000000 --- a/server/-product/production/OMAR/jobs/templates/remote-transcode.xml +++ /dev/null @@ -1,111 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - \ No newline at end of file diff --git a/server/-product/production/OMAR/jobs/templates/retrieve-ondemand.xml b/server/-product/production/OMAR/jobs/templates/retrieve-ondemand.xml deleted file mode 100644 index b58d8fdf..00000000 --- a/server/-product/production/OMAR/jobs/templates/retrieve-ondemand.xml +++ /dev/null @@ -1,150 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - \ No newline at end of file diff --git a/server/-product/production/OMAR/jobs/templates/retrieve.xml b/server/-product/production/OMAR/jobs/templates/retrieve.xml deleted file mode 100644 index e12fd950..00000000 --- a/server/-product/production/OMAR/jobs/templates/retrieve.xml +++ /dev/null @@ -1,21 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - \ No newline at end of file diff --git a/server/-product/production/OMAR/jobs/templates/safe-delete-recursive.xml b/server/-product/production/OMAR/jobs/templates/safe-delete-recursive.xml deleted file mode 100644 index bfa8e371..00000000 --- a/server/-product/production/OMAR/jobs/templates/safe-delete-recursive.xml +++ /dev/null @@ -1,17 +0,0 @@ - - - - - - - - - - - - - - - - - \ No newline at end of file diff --git a/server/-product/production/OMAR/jobs/templates/sync-subtitles.xml b/server/-product/production/OMAR/jobs/templates/sync-subtitles.xml deleted file mode 100644 index f68c2781..00000000 --- a/server/-product/production/OMAR/jobs/templates/sync-subtitles.xml +++ /dev/null @@ -1,23 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - \ No newline at end of file diff --git a/server/-product/production/OMAR/jobs/templates/sys-recreate-lowres.xml b/server/-product/production/OMAR/jobs/templates/sys-recreate-lowres.xml deleted file mode 100644 index 9ef84b31..00000000 --- a/server/-product/production/OMAR/jobs/templates/sys-recreate-lowres.xml +++ /dev/null @@ -1,118 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - \ No newline at end of file diff --git a/server/-product/production/OMAR/jobs/templates/test-fork-cancelable.xml b/server/-product/production/OMAR/jobs/templates/test-fork-cancelable.xml deleted file mode 100644 index 8c88d84b..00000000 --- a/server/-product/production/OMAR/jobs/templates/test-fork-cancelable.xml +++ /dev/null @@ -1,6 +0,0 @@ - - - - - - \ No newline at end of file diff --git a/server/-product/production/OMAR/log/.gitignore b/server/-product/production/OMAR/log/.gitignore deleted file mode 100644 index c474102b..00000000 --- a/server/-product/production/OMAR/log/.gitignore +++ /dev/null @@ -1,5 +0,0 @@ -/delete-materials.log -/markered-mediacube.log -/mediacube-err.log -/mediacube.log -/movtest.log diff --git a/server/-product/production/OMAR/log/2022-05/.gitignore b/server/-product/production/OMAR/log/2022-05/.gitignore deleted file mode 100644 index 5e8754b0..00000000 --- a/server/-product/production/OMAR/log/2022-05/.gitignore +++ /dev/null @@ -1,6 +0,0 @@ -/markered-mediacube-05-05-2022-1.log.gz -/markered-mediacube-05-06-2022-1.log.gz -/mediacube-05-05-2022-1.log.gz -/mediacube-05-06-2022-1.log.gz -/mediacube-err-05-05-2022-1.log.gz -/mediacube-err-05-06-2022-1.log.gz diff --git a/server/-product/production/OMAR/log/2022-05/markered-mediacube-05-09-2022-1.log.gz b/server/-product/production/OMAR/log/2022-05/markered-mediacube-05-09-2022-1.log.gz deleted file mode 100644 index 53250356..00000000 Binary files a/server/-product/production/OMAR/log/2022-05/markered-mediacube-05-09-2022-1.log.gz and /dev/null differ diff --git a/server/-product/production/OMAR/log/2022-05/mediacube-05-09-2022-1.log.gz b/server/-product/production/OMAR/log/2022-05/mediacube-05-09-2022-1.log.gz deleted file mode 100644 index b4179fcc..00000000 Binary files a/server/-product/production/OMAR/log/2022-05/mediacube-05-09-2022-1.log.gz and /dev/null differ diff --git a/server/-product/production/OMAR/log/2022-05/mediacube-err-05-09-2022-1.log.gz b/server/-product/production/OMAR/log/2022-05/mediacube-err-05-09-2022-1.log.gz deleted file mode 100644 index 605ced81..00000000 Binary files a/server/-product/production/OMAR/log/2022-05/mediacube-err-05-09-2022-1.log.gz and /dev/null differ diff --git a/server/-product/production/OMAR/mediacube.bat b/server/-product/production/OMAR/mediacube.bat deleted file mode 100644 index 24a39d6c..00000000 --- a/server/-product/production/OMAR/mediacube.bat +++ /dev/null @@ -1,17 +0,0 @@ -@echo off -if exist tmp rmdir tmp /s /q -mkdir tmp -java ^ --Dorg.eclipse.epp.logging.aeri.skipReports=true ^ --Declipse.ignoreApp=true ^ --Dosgi.noShutdown=true ^ --Dlog4j.configurationFile=settings/log4j2.xml ^ --Djetty.home=settings ^ --Djetty.etc.config.urls=jetty.xml ^ --Djava.io.tmpdir=tmp ^ --Dfile.encoding=UTF-8 ^ --Dgosh.home=configuration ^ --jar ../../target/products/MediaCube/linux/gtk/x86_64/plugins/org.eclipse.equinox.launcher_1.3.201.v20161025-1711.jar ^ --Xms512m ^ --Xmx1024m ^ --console diff --git a/server/-product/production/OMAR/settings/application.yaml b/server/-product/production/OMAR/settings/application.yaml deleted file mode 100644 index c870af39..00000000 --- a/server/-product/production/OMAR/settings/application.yaml +++ /dev/null @@ -1,45 +0,0 @@ -datasource: - mediacube: - url: jdbc:db2://localhost:50000/mc - user: db2admin - password: password - external-indexer: true - simple-search: true - login-timeout: 3 - pool-size: 10 - mediacube-nosql: - url: jdbc:db2://localhost:50000/mc - user: db2admin - password: password - schema: test - login-timeout: 3 - hsm: - url: jdbc:db2://10.11.1.89:51500/tsmdb1 - user: tsminst1 - password: tsminst1 - planair: - url: jdbc:sqlserver://10.11.254.86;databaseName=PA_Vivantis; - user: MAM - password: VDani -services: - ffmpeg: - executable-location: /opt/ffmpeg/ffmpeg - mediacube: - proxy-root: /data/mediacube - nexio: -# host: 10.10.1.55 -# collection-name: nexioclips -# use-mos-gateway: true - disabled: true -jobs: - validate-transfers: false - copy-buffer-size: 32768 - scheduled-execution-disabled: true -tsm: - randomize-archives: false - delimiter: / - node-name: JOBENGINE - fs-name: /JOBENGINE - alternate-fs-name: /JOBENGINE - hl-name: /JOBENGINE - \ No newline at end of file diff --git a/server/-product/production/OMAR/settings/dsm.opt b/server/-product/production/OMAR/settings/dsm.opt deleted file mode 100644 index 90fcbdd2..00000000 --- a/server/-product/production/OMAR/settings/dsm.opt +++ /dev/null @@ -1 +0,0 @@ -SErvername tsm.in.useribm.hu diff --git a/server/-product/production/OMAR/settings/dsmopt.lock b/server/-product/production/OMAR/settings/dsmopt.lock deleted file mode 100644 index e69de29b..00000000 diff --git a/server/-product/production/OMAR/settings/external-commands.yaml b/server/-product/production/OMAR/settings/external-commands.yaml deleted file mode 100644 index 5f3a3a63..00000000 --- a/server/-product/production/OMAR/settings/external-commands.yaml +++ /dev/null @@ -1,27 +0,0 @@ -profiles: -- - name: proxy - executable: /Programs/ffmpeg/bin/ffmpeg.exe - arguments: - - -y - - -v - - error - - -progress - - pipe:1 - - -i - - "%i" - - "%o" -- - name: length - executable: /Programs/ffmpeg/bin/ffprobe.exe - arguments: - - -v - - error - - -select_streams - - v:0 - - -show_entries - - stream=nb_frames - - -of - - default=noprint_wrappers=1:nokey=1 - - "%i" - \ No newline at end of file diff --git a/server/-product/production/OMAR/settings/jetty.xml b/server/-product/production/OMAR/settings/jetty.xml deleted file mode 100644 index 30bdeb08..00000000 --- a/server/-product/production/OMAR/settings/jetty.xml +++ /dev/null @@ -1,45 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - \ No newline at end of file diff --git a/server/-product/production/OMAR/settings/log4j2.xml b/server/-product/production/OMAR/settings/log4j2.xml deleted file mode 100644 index 5f514527..00000000 --- a/server/-product/production/OMAR/settings/log4j2.xml +++ /dev/null @@ -1,105 +0,0 @@ - - - - - - log - ${logPath}/mediacube.log - ${logPath}/movtest.log - ${logPath}/$${date:yyyy-MM}/movtest-%d{MM-dd-yyyy}-%i.log.gz - ${logPath}/delete-materials.log - ${logPath}/$${date:yyyy-MM}/delete-materials-%d{MM-dd-yyyy}-%i.log.gz - ${logPath}/$${date:yyyy-MM}/mediacube-%d{MM-dd-yyyy}-%i.log.gz - ${logPath}/markered-mediacube.log - ${logPath}/$${date:yyyy-MM}/markered-mediacube-%d{MM-dd-yyyy}-%i.log.gz - ${logPath}/mediacube-err.log - ${logPath}/$${date:yyyy-MM}/mediacube-err-%d{MM-dd-yyyy}-%i.log.gz - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - \ No newline at end of file diff --git a/server/-product/production/OMAR/settings/maestro.yaml b/server/-product/production/OMAR/settings/maestro.yaml deleted file mode 100644 index b8091a2f..00000000 --- a/server/-product/production/OMAR/settings/maestro.yaml +++ /dev/null @@ -1,38 +0,0 @@ -sourceStoreUri: - name: Default - protocol: LOCAL - uri: "/mnt/NLE" - fileFilter: "*.mxf" - showDirectories: true -alternateSourceStoreUris: -- name: NLE1 - protocol: LOCAL - uri: "/mnt/NLE/NLE1" - fileFilter: "*.mxf" - showDirectories: true -- name: NLE2 - protocol: LOCAL - uri: "/mnt/NLE/NLE2" - fileFilter: "*.mxf" - showDirectories: true -- name: NLE3 - protocol: LOCAL - uri: "/mnt/NLE/NLE3" - fileFilter: "*.mxf" - showDirectories: true -- name: NLE4 - protocol: LOCAL - uri: "/mnt/NLE/NLE4" - fileFilter: "*.mxf" - showDirectories: true -- name: POLC - protocol: LOCAL - uri: "/mnt/POLC" - fileFilter: "*.mxf" - showDirectories: true -targets: -- name: FINISHED_SHOWS - killDateDays: 7 - storeUri: - protocol: LOCAL - uri: "/mnt/PROMISE/FINISHED_SHOWS" diff --git a/server/-product/production/OMAR/settings/mediacube.yaml b/server/-product/production/OMAR/settings/mediacube.yaml deleted file mode 100644 index 347ca896..00000000 --- a/server/-product/production/OMAR/settings/mediacube.yaml +++ /dev/null @@ -1,53 +0,0 @@ -jobQueuePollInterval: 1000 -disableHelp: true -maestroDisabled: false -alternateRetrieveSelector: false -disableStatistics: true -disableEditor: false -topTypeFilters: -- name: Hír bejátszó -- name: Hír nyers -- name: Visszarögzített -- name: Egyéb -bottomTypeFilters: -- name: Műsor -- name: Műsor nyers -- name: Promo -- name: Promo nyers -- name: Reklám -- name: Reklám nyers -authentication: - authEnabled: true - defaultUser: root - defaultPassword: password - adHost: intra.mediavivantis.hu - adNonSecurePort: 3268 - adBaseDn: DC=intra,DC=mediavivantis,DC=hu - adAdminMap: - - G_MV_U_MUSZAK - - G_MV_U_INGEST - adSubmitterMap: - - G_ECH_U_INFORMATIKUSOK - - G_ECH_U_MUSZAKVEZETOK - - ECH-ISILON-ADMINS - adEditorMap: - - G_ECH_U_INFORMATIKUSOK - - G_ECH_U_MUSZAKVEZETOK - - ECH-ISILON-ADMINS - localAccounts: - - user: user - password: 5F4DCC3B5AA765D61D8327DEB882CF99 - email: - - user: lebony - password: 4E25B117B14D86D7DCECB4E433CF932C - email: - - user: root - password: 5F4DCC3B5AA765D61D8327DEB882CF99 - email: vasary@elgekko.net - localAdmins: - - root - - admin - localSubmitters: - - lebony - localEditors: - - editor diff --git a/server/hu.user.mediacube.executors.tests/.settings/org.eclipse.core.resources.prefs b/server/hu.user.mediacube.executors.tests/.settings/org.eclipse.core.resources.prefs index 9b0d260f..fd2d9940 100644 --- a/server/hu.user.mediacube.executors.tests/.settings/org.eclipse.core.resources.prefs +++ b/server/hu.user.mediacube.executors.tests/.settings/org.eclipse.core.resources.prefs @@ -3,3 +3,4 @@ encoding//src/hu/user/mediacube/executors/tests/SupportAttachLowres.java=UTF-8 encoding//src/hu/user/mediacube/executors/tests/SupportCheckDelete.java=UTF-8 encoding//src/hu/user/mediacube/executors/tests/SupportCheckItem.java=UTF-8 encoding//src/hu/user/mediacube/executors/tests/SupportCreateItems.java=UTF-8 +encoding//src/hu/user/mediacube/executors/tests/SupportSyncMCToTSM.java=UTF-8 diff --git a/server/hu.user.mediacube.executors.tests/src/hu/user/mediacube/executors/tests/SmallTests.java b/server/hu.user.mediacube.executors.tests/src/hu/user/mediacube/executors/tests/SmallTests.java index 9e5bb23d..19bb773f 100644 --- a/server/hu.user.mediacube.executors.tests/src/hu/user/mediacube/executors/tests/SmallTests.java +++ b/server/hu.user.mediacube.executors.tests/src/hu/user/mediacube/executors/tests/SmallTests.java @@ -1071,19 +1071,37 @@ public class SmallTests { // htv errors @Test public void test9999() throws Exception { - Set tsm = new LinkedHashSet<>(Files.readAllLines(Paths.get("/opt/HTV/htv-tsm.txt"))); + List lines = Files.readAllLines(Paths.get("/opt/HTV/htv-tsm.txt")); + + Set tsm = new LinkedHashSet<>(); + lines.forEach(x -> { + if (tsm.contains(x)) { + System.out.println("Already added " + x); + } else { + tsm.add(x); + } + + }); + + System.out.println("TSM lines " + tsm.size()); Set mc = new LinkedHashSet<>(Files.readAllLines(Paths.get("/opt/HTV/htv-mc.txt"))); + Map mcMap = new HashMap<>(); + mc.forEach(x -> { + String[] tokens = x.split("\t"); + mcMap.put(tokens[0].replace("\"", ""), tokens[1].replace("'", "")); + }); int errorCount = 0; int current = 0; - for (String mcfile : mc) { + Set mcIds = mcMap.keySet(); + for (String mcfile : mcIds) { current++; boolean tsmContains = tsm.contains(mcfile); if (!tsmContains) { errorCount++; - int p = current * 100 / mc.size(); - System.out.println("Missing " + mcfile + " %" + p); + int p = current * 100 / mcIds.size(); + System.out.println(mcfile + "\t" + mcMap.get(mcfile)); } } System.out.println("All missing: " + errorCount); diff --git a/server/user.jobengine.osgi.commons/src/user/commons/remotestore/TSMLister.java b/server/user.jobengine.osgi.commons/src/user/commons/remotestore/TSMLister.java index 472ac53a..30ef7358 100644 --- a/server/user.jobengine.osgi.commons/src/user/commons/remotestore/TSMLister.java +++ b/server/user.jobengine.osgi.commons/src/user/commons/remotestore/TSMLister.java @@ -50,8 +50,7 @@ public class TSMLister implements IDirectoryLister { client = new TSMBufferedClient(NODENAME); // client.connect(storeUri.getUserName(), storeUri.getPassword(), "\\"); client.connect(storeUri.getUserName(), storeUri.getPassword()); - logger.info("TSMBufferedClient connected, parameters: {} {}/{} {}, separator: {}", NODENAME, FSNAME, - ALTERNATE_FSNAME, HLNAME, File.separator); + logger.info("TSMBufferedClient connected, parameters: {} {}/{} {}, separator: {}", NODENAME, FSNAME, ALTERNATE_FSNAME, HLNAME, File.separator); } @Override @@ -66,6 +65,11 @@ public class TSMLister implements IDirectoryLister { @Override public boolean exists(String fileName) throws Exception { + /* + * TSMBackupFileObject backupFileObject = + * tsmClient.getActiveBackupFileObject(tsmProperties.getFsName(), + * tsmProperties.getHlName(), tsmProperties.getDelimiter() + id); + */ return false; } @@ -76,8 +80,7 @@ public class TSMLister implements IDirectoryLister { connect(); String currentFileName = fileName; logger.info("Getting {}, {}, {}", FSNAME, HLNAME, DELIMITER + currentFileName); - TSMBackupFileObject backupFileObject = client.getActiveBackupFileObject(FSNAME, HLNAME, - DELIMITER + currentFileName); + TSMBackupFileObject backupFileObject = client.getActiveBackupFileObject(FSNAME, HLNAME, DELIMITER + currentFileName); // probaljuk meg a masik tarbol if (backupFileObject == null) { diff --git a/server/user.jobengine.osgi.server/src/user/jobengine/server/JobEngineConfiguration.java b/server/user.jobengine.osgi.server/src/user/jobengine/server/JobEngineConfiguration.java index 1c44347b..6298b58a 100644 --- a/server/user.jobengine.osgi.server/src/user/jobengine/server/JobEngineConfiguration.java +++ b/server/user.jobengine.osgi.server/src/user/jobengine/server/JobEngineConfiguration.java @@ -101,8 +101,7 @@ public class JobEngineConfiguration implements IJobEngineConfiguration { public IJobStep createJobStep(String stepUnitName) throws Exception { IJobStep result = null; - boolean isGroovyClass = stepUnitName.toLowerCase().endsWith(".java") - || stepUnitName.toLowerCase().endsWith(".groovy"); + boolean isGroovyClass = stepUnitName.toLowerCase().endsWith(".java") || stepUnitName.toLowerCase().endsWith(".groovy"); logger.info("Looking for {} step ClassLoader requirement", stepUnitName); if (resetStepClassLoader) { @@ -135,8 +134,7 @@ public class JobEngineConfiguration implements IJobEngineConfiguration { if (isGroovyClass) { if (dynamicStepsLoader != null) - stepClass = (Class) dynamicStepsLoader.loadClassFromSourceCode(groovyClassLoader, - stepUnitName); + stepClass = (Class) dynamicStepsLoader.loadClassFromSourceCode(groovyClassLoader, stepUnitName); } else stepClass = (Class) stepsClassLoader.loadClass(stepUnitName); @@ -242,7 +240,7 @@ public class JobEngineConfiguration implements IJobEngineConfiguration { /** * Visszadja az ütemezett feladatokat tartalmazó collection-t (a * scheduledjobs.json fájl tartalmát) - * + * */ @Override public ArrayList> getSchedules() { @@ -322,8 +320,7 @@ public class JobEngineConfiguration implements IJobEngineConfiguration { int newMaxConcurrent = executor.getMaxConcurrent(); if (currentMaxConcurrent != newMaxConcurrent) { stepExecutor.setMaxConcurrent(newMaxConcurrent); - logger.info("Executor maxConcurrent changed from {} to {}", currentMaxConcurrent, - newMaxConcurrent); + logger.info("Executor maxConcurrent changed from {} to {}", currentMaxConcurrent, newMaxConcurrent); } } @@ -360,8 +357,8 @@ public class JobEngineConfiguration implements IJobEngineConfiguration { BasicDBList scheduledJobs = NoSQLUtils.asDBList(dbo, "joblist"); scheduledJobs.forEach(o -> { + BasicDBObject j = (BasicDBObject) o; try { - BasicDBObject j = (BasicDBObject) o; String template = j.getString("template"); String xml = loadTemplateXml(template); j.put("xml", xml); @@ -379,7 +376,7 @@ public class JobEngineConfiguration implements IJobEngineConfiguration { schedules.add(new SimpleEntry(template, j)); } catch (Exception e) { - logger.error("Scheduled job error! {}", e.getMessage()); + logger.error("Scheduled job error! {}. Details: {}", e.getMessage(), j.toPrettyString("")); } }); } @@ -547,8 +544,7 @@ public class JobEngineConfiguration implements IJobEngineConfiguration { if (!Files.isDirectory(templatesPath)) throw new FileNotFoundException(templatesPath + " is not a directory!"); if (templateToSave.getFileName() != null) { - filePath = templatesPath.toString() + FileSystems.getDefault().getSeparator() - + templateToSave.getFileName(); + filePath = templatesPath.toString() + FileSystems.getDefault().getSeparator() + templateToSave.getFileName(); setFileAsWritable(Paths.get(filePath)); FileOutputStream fos = new FileOutputStream(filePath); logger.info("Saving job template to {}", filePath); @@ -595,15 +591,13 @@ public class JobEngineConfiguration implements IJobEngineConfiguration { InputStream is = null; try { templatesPath = Paths.get(systemConfig.getConfig(DIR_TEMPLATES)); - duplicateFileName = FilenameUtils.getName(selectedJob.getString("template")).replace(".xml", "") - .concat("-copy.xml"); + duplicateFileName = FilenameUtils.getName(selectedJob.getString("template")).replace(".xml", "").concat("-copy.xml"); filePath = templatesPath.toString() + FileSystems.getDefault().getSeparator() + duplicateFileName; while (Files.exists(Paths.get(filePath))) { duplicateFileName = FilenameUtils.getName(duplicateFileName).replace(".xml", "").concat("-copy.xml"); filePath = templatesPath.toString() + FileSystems.getDefault().getSeparator() + duplicateFileName; } - is = new FileInputStream(templatesPath.toString() + FileSystems.getDefault().getSeparator() - + selectedJob.getString("template")); + is = new FileInputStream(templatesPath.toString() + FileSystems.getDefault().getSeparator() + selectedJob.getString("template")); Files.copy(is, Paths.get(filePath)); } catch (FileNotFoundException e) { logger.error("File not found: {}", filePath); @@ -638,6 +632,7 @@ public class JobEngineConfiguration implements IJobEngineConfiguration { resetStepClassLoader = true; } + @Override public ArrayList getSteps() { return steps; } diff --git a/server/user.mediacube.gui/pages/jobs.zul b/server/user.mediacube.gui/pages/jobs.zul index 97d559b9..4dfe4b52 100644 --- a/server/user.mediacube.gui/pages/jobs.zul +++ b/server/user.mediacube.gui/pages/jobs.zul @@ -6,7 +6,7 @@ - + diff --git a/server/user.mediacube.gui/pages/menu.zul b/server/user.mediacube.gui/pages/menu.zul index 0b955abf..a24895aa 100644 --- a/server/user.mediacube.gui/pages/menu.zul +++ b/server/user.mediacube.gui/pages/menu.zul @@ -8,7 +8,7 @@ + disabled="${(not sessionScope.userPrincipal.pooler and not sessionScope.userPrincipal.submitter) or sessionScope.userPrincipal.anonymous}" /> diff --git a/server/user.mediacube.gui/resources/i3-label_hu.properties b/server/user.mediacube.gui/resources/i3-label_hu.properties index 708ddd80..cfe25737 100644 --- a/server/user.mediacube.gui/resources/i3-label_hu.properties +++ b/server/user.mediacube.gui/resources/i3-label_hu.properties @@ -1,4 +1,4 @@ -version=2.8.0 +version=2.8.1 #!!!2.8.0 Keresofelulet modositas (hardkodolt) #2.7.9 Keresofelulet modositas (hardkodolt) #2.7.8 Classloader mechanizmus atalakitasa, metadata editor history diff --git a/server/user.mediacube.gui/src/user/jobengine/zk/model/AlternateRetrieveBatchSelectorModel.java b/server/user.mediacube.gui/src/user/jobengine/zk/model/AlternateRetrieveBatchSelectorModel.java index aaff0397..541a67de 100644 --- a/server/user.mediacube.gui/src/user/jobengine/zk/model/AlternateRetrieveBatchSelectorModel.java +++ b/server/user.mediacube.gui/src/user/jobengine/zk/model/AlternateRetrieveBatchSelectorModel.java @@ -20,6 +20,7 @@ import org.zkoss.zul.ListModelList; import org.zkoss.zul.Messagebox; import org.zkoss.zul.Window; +import user.commons.ListUtils; import user.jobengine.db.ArchivedMedia; import user.jobengine.db.ItemManager; import user.jobengine.db.Store; @@ -75,7 +76,8 @@ public class AlternateRetrieveBatchSelectorModel extends BaseModel { if (scheduledJob == null) throw new Exception("A sablon nem található: " + JOBTEMPLATE); - //String escortStoreName, String targetStoreName, List basket, String recipient + // String escortStoreName, String targetStoreName, List basket, + // String recipient Map parameters = scheduledJob.getJobParameters(); parameters.put(TARGETSTORENAME, stores.getSelection().iterator().next().getName()); parameters.put(RECIPIENT, email); @@ -119,13 +121,25 @@ public class AlternateRetrieveBatchSelectorModel extends BaseModel { List targetStores = ItemManager.getInstance().getTargetStores(); if (targetStores != null) { - List filters = SessionUtil.getMediaCubeConfig().getTargetRestoreFilters(); + List filters = SessionUtil.getMediaCubeConfig().getTargetRestoreFilters(); if (filters == null || filters.size() == 0) { stores.addAll(targetStores); } else { for (Store store : targetStores) { - if (filters.contains(store.getName())) - stores.add(store); + filters.forEach(filter -> { + if (!store.getName().equals(filter.getName())) { + return; + } + List allowedGroups = filter.getAllowedGroups(); + if (ListUtils.isEmpty(allowedGroups)) { + stores.add(store); + } else { + if (allowedGroups.contains(SessionUtil.getUserPrincipal().getGroupName())) { + stores.add(store); + } + } + + }); } } diff --git a/server/user.mediacube.gui/src/user/jobengine/zk/model/MediaCubeAuthentication.java b/server/user.mediacube.gui/src/user/jobengine/zk/model/MediaCubeAuthentication.java index 204ecce5..732344ba 100644 --- a/server/user.mediacube.gui/src/user/jobengine/zk/model/MediaCubeAuthentication.java +++ b/server/user.mediacube.gui/src/user/jobengine/zk/model/MediaCubeAuthentication.java @@ -14,6 +14,7 @@ public class MediaCubeAuthentication { private List adEditorMap; private List localAccounts; private List localAdmins; + private List localPoolers; private List localSubmitters; private List localEditors; @@ -121,4 +122,12 @@ public class MediaCubeAuthentication { this.localSubmitters = localSubmitters; } + public List getLocalPoolers() { + return localPoolers; + } + + public void setLocalPoolers(List localPoolers) { + this.localPoolers = localPoolers; + } + } diff --git a/server/user.mediacube.gui/src/user/jobengine/zk/model/MediaCubeConfig.java b/server/user.mediacube.gui/src/user/jobengine/zk/model/MediaCubeConfig.java index 5844fbfb..c89c4dcd 100644 --- a/server/user.mediacube.gui/src/user/jobengine/zk/model/MediaCubeConfig.java +++ b/server/user.mediacube.gui/src/user/jobengine/zk/model/MediaCubeConfig.java @@ -3,11 +3,12 @@ package user.jobengine.zk.model; import java.util.List; public class MediaCubeConfig { + private int jobQueuePollInterval = 10; private MediaCubeAuthentication authentication; private List topTypeFilters; private List bottomTypeFilters; - private List targetRestoreFilters; + private List targetRestoreFilters; private boolean maestroDisabled; private boolean alternateRetrieveSelector; private boolean disableStatistics; @@ -26,7 +27,7 @@ public class MediaCubeConfig { return jobQueuePollInterval; } - public List getTargetRestoreFilters() { + public List getTargetRestoreFilters() { return targetRestoreFilters; } @@ -86,7 +87,7 @@ public class MediaCubeConfig { this.maestroDisabled = maestroDisabled; } - public void setTargetRestoreFilters(List targetRestoreFilters) { + public void setTargetRestoreFilters(List targetRestoreFilters) { this.targetRestoreFilters = targetRestoreFilters; } diff --git a/server/user.mediacube.gui/src/user/jobengine/zk/util/AdminAuthInitiator.java b/server/user.mediacube.gui/src/user/jobengine/zk/util/AdminAuthInitiator.java index f97844be..edba7f49 100644 --- a/server/user.mediacube.gui/src/user/jobengine/zk/util/AdminAuthInitiator.java +++ b/server/user.mediacube.gui/src/user/jobengine/zk/util/AdminAuthInitiator.java @@ -12,7 +12,7 @@ public class AdminAuthInitiator implements Initiator { if (!SessionUtil.isAuthenticated()) SessionUtil.redirect("/pages/login.zul"); - if (!(SessionUtil.isAdmin() || SessionUtil.isSubmitter())) + if (!(SessionUtil.isAdmin() || SessionUtil.isSubmitter() || SessionUtil.isPooler())) SessionUtil.redirect("/pages/access-denied.zul"); } diff --git a/server/user.mediacube.gui/src/user/jobengine/zk/util/LocalUserHandler.java b/server/user.mediacube.gui/src/user/jobengine/zk/util/LocalUserHandler.java index 47bac7f2..623a4ddb 100644 --- a/server/user.mediacube.gui/src/user/jobengine/zk/util/LocalUserHandler.java +++ b/server/user.mediacube.gui/src/user/jobengine/zk/util/LocalUserHandler.java @@ -48,11 +48,18 @@ public class LocalUserHandler { boolean isAdmin = isMemberOf(account, mediaCubeAuthentication.getLocalAdmins()); boolean isSubmitter = isMemberOf(account, mediaCubeAuthentication.getLocalSubmitters()); boolean isEditor = isMemberOf(account, mediaCubeAuthentication.getLocalEditors()); + boolean isPooler = isMemberOf(account, mediaCubeAuthentication.getLocalPoolers()); + result.setAdmin(isAdmin); if (isAdmin || isSubmitter) result.setSubmitter(true); + if (isAdmin || isEditor) result.setEditor(true); + + if (isAdmin || isPooler) + result.setPooler(true); + } return result; } diff --git a/server/user.mediacube.gui/src/user/jobengine/zk/util/SessionUtil.java b/server/user.mediacube.gui/src/user/jobengine/zk/util/SessionUtil.java index b0828a0c..644abe28 100644 --- a/server/user.mediacube.gui/src/user/jobengine/zk/util/SessionUtil.java +++ b/server/user.mediacube.gui/src/user/jobengine/zk/util/SessionUtil.java @@ -36,7 +36,7 @@ import user.jobengine.zk.model.IndexModel; import user.jobengine.zk.model.MaestroConfig; import user.jobengine.zk.model.MediaCubeConfig; -public class SessionUtil implements GlobalConstants{ +public class SessionUtil implements GlobalConstants { private static Logger logger = LogManager.getLogger(); private static final String MEDIACUBE_CONFIG = "mediaCubeConfig"; private static final String MAESTRO_CONFIG = "maestroConfig"; @@ -55,16 +55,17 @@ public class SessionUtil implements GlobalConstants{ public static final String GUI_STAT_DISABLE = "gui.statistics.disable"; public static final String GUI_HELP_DISABLE = "gui.help.disable"; public static final String GUI_EDITOR_DISABLE = "gui.editor.disable"; - - // private static MediaCubeConfig mediaCubeConfig; + + // private static MediaCubeConfig mediaCubeConfig; // - // static { - // try { - // mediaCubeConfig = JSONConfig.read(JOBENGINE_MEDIACUBE_CONFIG, MediaCubeConfig.class); - // } catch (Exception e) { - // logger.error("Critical error", e); - // } - // } + // static { + // try { + // mediaCubeConfig = JSONConfig.read(JOBENGINE_MEDIACUBE_CONFIG, + // MediaCubeConfig.class); + // } catch (Exception e) { + // logger.error("Critical error", e); + // } + // } public static boolean authenticate(String account, String password) { MediaCubeConfig mediaCubeConfig = getMediaCubeConfig(); @@ -107,11 +108,12 @@ public class SessionUtil implements GlobalConstants{ } static public void cleanup() { - // IJobChangedListener listener = (IJobChangedListener) getAttribute(SessionUtil.JOBLISTENER); - // if (listener != null) { - // JobEngine.getInstance().removeJobChangedEventListener(listener); - // setAttribute(SessionUtil.JOBLISTENER, null); - // } + // IJobChangedListener listener = (IJobChangedListener) + // getAttribute(SessionUtil.JOBLISTENER); + // if (listener != null) { + // JobEngine.getInstance().removeJobChangedEventListener(listener); + // setAttribute(SessionUtil.JOBLISTENER, null); + // } // setAttribute(SessionUtil.MENUMODEL, null); // setAttribute(SessionUtil.USERPRINCIPAL, null); @@ -159,7 +161,7 @@ public class SessionUtil implements GlobalConstants{ IConfiguration systemConfig = ComponentBinder.getSystemConfig(); String configFilePath = systemConfig.getConfig(JobEngineConfiguration.CONF_MAESTRO); result = systemConfig.load(configFilePath, MaestroConfig.class); - //result = JSONConfig.read(configFilePath, MaestroConfig.class); + // result = JSONConfig.read(configFilePath, MaestroConfig.class); setAttribute(MAESTRO_CONFIG, result); } } catch (Exception e) { @@ -176,7 +178,7 @@ public class SessionUtil implements GlobalConstants{ IConfiguration systemConfig = ComponentBinder.getSystemConfig(); String configFilePath = ComponentBinder.getSystemConfig().getConfig(JobEngineConfiguration.CONF_MEDIACUBE); result = systemConfig.load(configFilePath, MediaCubeConfig.class); - //result = JSONConfig.read(configFilePath, MediaCubeConfig.class); + // result = JSONConfig.read(configFilePath, MediaCubeConfig.class); setAttribute(MEDIACUBE_CONFIG, result); } } catch (Exception e) { @@ -291,6 +293,11 @@ public class SessionUtil implements GlobalConstants{ return userPrincipal != null && userPrincipal.isSubmitter(); } + static public boolean isPooler() { + UserPrincipal userPrincipal = getUserPrincipal(); + return userPrincipal != null && userPrincipal.isPooler(); + } + public static void logout() { setAttribute(USER_PRINCIPAL, null); setAttribute(HISTORY, null); @@ -381,9 +388,9 @@ public class SessionUtil implements GlobalConstants{ } static public void registerJobChangedListener(IJobChangedListener listener) { - // cleanup(); - // JobEngine.getInstance().addJobChangedEventListener(listener); - // setAttribute(JOBLISTENER, listener); + // cleanup(); + // JobEngine.getInstance().addJobChangedEventListener(listener); + // setAttribute(JOBLISTENER, listener); IJobEngine jobEngine = ComponentBinder.getJobEngine(); if (jobEngine != null) @@ -403,13 +410,13 @@ public class SessionUtil implements GlobalConstants{ if (session != null) session.setAttribute(name, obj); } - + public static void publish(Event event) { EventQueues.lookup(EVENTQUEUE_NAME, EventQueues.SESSION, true).publish(event); } public static void subscribe(EventListener eventListener) { - EventQueues.lookup(EVENTQUEUE_NAME, EventQueues.SESSION, true).subscribe(eventListener); + EventQueues.lookup(EVENTQUEUE_NAME, EventQueues.SESSION, true).subscribe(eventListener); } public static void unsubscribe(EventListener eventListener) { diff --git a/server/user.mediacube.gui/src/user/jobengine/zk/util/UserPrincipal.java b/server/user.mediacube.gui/src/user/jobengine/zk/util/UserPrincipal.java index 9b489e85..4b28e14b 100644 --- a/server/user.mediacube.gui/src/user/jobengine/zk/util/UserPrincipal.java +++ b/server/user.mediacube.gui/src/user/jobengine/zk/util/UserPrincipal.java @@ -14,20 +14,23 @@ public class UserPrincipal implements java.io.Serializable { private boolean anonymous; private boolean submitter; private boolean editor; + private boolean pooler; private String account; private String displayName; private String email; + private String groupName; public UserPrincipal(String account) { - this(account, false, false, false); + this(account, false, false, false, false); } - public UserPrincipal(String account, boolean isAdmin, boolean isSubmitter, boolean isAnonymous) { + public UserPrincipal(String account, boolean isAdmin, boolean isSubmitter, boolean isAnonymous, boolean isPooler) { this.account = account; this.displayName = account; - this.admin = isAdmin; - this.submitter = isSubmitter; - this.anonymous = isAnonymous; + this.setAdmin(isAdmin); + this.setSubmitter(isSubmitter); + this.setAnonymous(isAnonymous); + this.setPooler(isPooler); } public String getAccount() { @@ -54,16 +57,22 @@ public class UserPrincipal implements java.io.Serializable { return editor; } + public boolean isPooler() { + return pooler; + } + public boolean isSubmitter() { return submitter; } public void setAdmin(boolean admin) { this.admin = admin; + this.groupName = "Admin"; } public void setAnonymous(boolean anonymous) { this.anonymous = anonymous; + this.groupName = "Anonymous"; } public void setDisplayName(String displayName) { @@ -72,6 +81,7 @@ public class UserPrincipal implements java.io.Serializable { public void setEditor(boolean editor) { this.editor = editor; + this.groupName = "Editor"; } public void setEmail(String email) { @@ -80,6 +90,7 @@ public class UserPrincipal implements java.io.Serializable { public void setSubmitter(boolean submitter) { this.submitter = submitter; + this.groupName = "Submitter"; } @Override @@ -92,4 +103,13 @@ public class UserPrincipal implements java.io.Serializable { } return String.format("Display name: %s, email: %s, membership: %s", String.valueOf(displayName), String.valueOf(email), membership.toString()); } + + public void setPooler(boolean pooler) { + this.pooler = pooler; + this.groupName = "Pooler"; + } + + public String getGroupName() { + return groupName; + } }