From: Vásáry Dániel Date: Wed, 28 Feb 2018 10:22:23 +0000 (+0000) Subject: git-tfs-id: [http://tfs.userrendszerhaz.hu:8080/tfs/DefaultCollection]$/MediaCube... X-Git-Url: http://git.useribm.hu/?a=commitdiff_plain;h=86e0ca834bae823cf6bb57aa5718f015c29e32d9;p=mediacube.git git-tfs-id: [tfs.userrendszerhaz.hu:8080/tfs/DefaultCollection]$/MediaCube;C30935 --- diff --git a/server/-configuration/scheduledjobs.json b/server/-configuration/scheduledjobs.json index 076f1c8b..7cd76f45 100644 --- a/server/-configuration/scheduledjobs.json +++ b/server/-configuration/scheduledjobs.json @@ -18,6 +18,7 @@ "cronexpression": "0 0 0/1 1/1 * ? *", "parameters": [ {"name": "sourcePath", "value": "\\\\10.10.254.74\\temp_isilon\\NEXIO", "type": "java.lang.String"}, + {"name": "targetPath", "value": "\\\\10.10.1.74\\MAM-Proxy_input", "type": "java.lang.String"}, {"name": "nexioPort", "value": 2098, "type": "java.lang.Integer"}, {"name": "nexioUserName", "value": "administrator", "type": "java.lang.String"}, {"name": "nexioPassword", "value": "system", "type": "java.lang.String"} diff --git a/server/user.jobengine.executors/deploy-steps-to-bsh.bat b/server/user.jobengine.executors/deploy-steps-to-bsh.batx similarity index 100% rename from server/user.jobengine.executors/deploy-steps-to-bsh.bat rename to server/user.jobengine.executors/deploy-steps-to-bsh.batx diff --git a/server/user.jobengine.executors/deploy-steps-to-bsh2.bat b/server/user.jobengine.executors/deploy-steps-to-bsh2.batx similarity index 100% rename from server/user.jobengine.executors/deploy-steps-to-bsh2.bat rename to server/user.jobengine.executors/deploy-steps-to-bsh2.batx diff --git a/server/user.jobengine.executors/jobtemplates/archive-recording.xml b/server/user.jobengine.executors/jobtemplates/archive-recording.xml index 4afd9bda..b18b03ba 100644 --- a/server/user.jobengine.executors/jobtemplates/archive-recording.xml +++ b/server/user.jobengine.executors/jobtemplates/archive-recording.xml @@ -1,22 +1,31 @@ - + - - - - + + + + + + + + + + + + + - + @@ -46,11 +55,89 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/server/user.jobengine.executors/src/user/jobengine/server/steps/DownloadRecordingFromNexioStep.java b/server/user.jobengine.executors/src/user/jobengine/server/steps/DownloadRecordingFromNexioStep.java new file mode 100644 index 00000000..10caec80 --- /dev/null +++ b/server/user.jobengine.executors/src/user/jobengine/server/steps/DownloadRecordingFromNexioStep.java @@ -0,0 +1,165 @@ +package user.jobengine.server.steps; + +import java.io.File; +import java.nio.file.Paths; + +import org.apache.commons.lang.StringUtils; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.apache.logging.log4j.Marker; +import org.apache.logging.log4j.message.Message; +import org.apache.logging.log4j.message.ParameterizedMessage; + +import com.ibm.nosql.json.api.BasicDBObject; +import com.ibm.nosql.json.api.DB; +import com.ibm.nosql.json.api.DBCollection; + +import user.commons.RemoteFile; +import user.commons.StoreUri; +import user.commons.nosql.NoSQLUtils; +import user.commons.remotestore.IProgressEventListener; +import user.commons.remotestore.IStatusEventListener; +import user.commons.remotestore.ProgressEvent; +import user.commons.remotestore.RemoteStoreProtocol; +import user.commons.remotestore.StatusEvent; +import user.jobengine.db.IItemManager; +import user.jobengine.server.IJobEngine; +import user.jobengine.server.IJobRuntime; + +public class DownloadRecordingFromNexioStep extends JobStep { + private static final String MXFEXT = ".MXF"; + private static final String NEXIOCLIPS = "nexioclips"; + private static final String LONGNAMEID = "longnameid"; + private static final String DURATION = "duration"; + + private static final Logger logger = LogManager.getLogger(); + + private IItemManager manager; + private StoreUri targetUri; + private StoreUri sourceUri; + private Marker marker; + private DBCollection clipsCollection; + + private int check(int value, String name) { + if (value == 0) { + logger.error(marker, "A folyamat '{}' bemeneti paramétere 0.", name); + throw new NullPointerException(String.format("System is not configured properly, missing '%s' input parameter.", name)); + } + return value; + } + + private String check(String value, String name) { + if (StringUtils.isBlank(value)) { + logger.error(marker, "A folyamat '{}' bemeneti paramétere üres.", name); + throw new NullPointerException(String.format("System is not configured properly, missing '%s' input parameter.", name)); + } + return value; + } + + @StepEntry + public Object[] execute(ArchiveItem archiveItem, String targetPath, String targetFileName, int nexioPort, String nexioUserName, String nexioPassword, + IJobEngine jobEngine, IJobRuntime jobRuntime) throws Exception { + marker = jobRuntime.getMarker(); + manager = jobEngine.getItemManager(); + setAndCheck(archiveItem, targetPath, targetFileName, nexioPort, nexioUserName, nexioPassword); + String sourceFileName = targetFileName; + try { + final IJobRuntime runtime = jobRuntime; + targetUri.addProgressListener(new IProgressEventListener() { + @Override + public void progressChanged(ProgressEvent evt) { + runtime.incrementProgress(evt.getProgress()); + } + }); + targetUri.addStatusListener(new IStatusEventListener() { + @Override + public void statusChanged(StatusEvent evt) { + evt.setCancel(!canContinue()); + } + }); + + String targetName = targetFileName + MXFEXT; + + File targetFile = Paths.get(targetPath, targetName).toFile(); + if (targetFile.exists()) + throw new Exception("Exists!"); + + RemoteFile remoteFile = sourceUri.transferFrom(targetUri, sourceFileName, targetName); + + logger.info(marker, "Az '{}' állomány letöltése sikeres volt '{}' néven.", sourceFileName, targetFile); + + BasicDBObject clip = (BasicDBObject) clipsCollection.findOne(new BasicDBObject(LONGNAMEID, sourceFileName)); + if (clip == null) + throw new Exception("Clip not exists in NEXIO"); + + long duration = NoSQLUtils.asLong(clip, DURATION); + if (duration == 0) + throw new Exception("Clip duration is 0"); + + archiveItem.setDuration(duration); + archiveItem.setMediaFile(targetFile.toString()); + + RemoteFile sourceRemoteFile = sourceUri.getRemoteFile(sourceFileName); + sourceUri.delete(sourceRemoteFile); + + } catch (Exception e) { + logger.catching(e); + if (!archiveItem.removeCatchedFile()) + logger.error(getMarker(), "A {} állomány .catched jelző állománya nem törölhető.", new File(archiveItem.getMediaFile()).getName()); + Message m = new ParameterizedMessage("Az '{}' állomány feldolgozása sikertelen. A rendszer hibaüzenete: {}", sourceFileName, e.getMessage()); + logger.error(marker, m); + throw new Exception(m.getFormattedMessage()); + } + return new Object[] { 0 }; + } + + private void setAndCheck(ArchiveItem archiveItem, String targetPath, String targetFileName, int nexioPort, String nexioUserName, String nexioPassword) + throws Exception { + DB db = NoSQLUtils.getNoSQLDB(); + if (db == null) { + logger.error(marker, "Az NoSQL adatkezelő réteg nem elérhető."); + throw new NullPointerException("Internal error, missing NoSQL DB reference."); + } + + clipsCollection = db.getCollection(NEXIOCLIPS); + + if (archiveItem == null) { + logger.error(marker, "A folyamat 'archiveItem' bemeneti paramétere üres."); + throw new NullPointerException("Internal error, missing 'archiveItem'."); + } + + if (archiveItem.getMediaFile() == null) { + logger.error(marker, "A folyamat 'archiveItem.mediaFile' paramétere üres."); + throw new NullPointerException("Internal error, missing 'archiveItem.mediaFile'."); + } + + check(targetFileName, "targetFileName"); + check(targetPath, "targetPath"); + + String nexioHost = System.getProperty("nexio.host"); + if (StringUtils.isBlank(nexioHost)) { + logger.error(marker, "A 'nexio.host' rendszer paraméter nem található."); + throw new NullPointerException("System is not configured properly, 'jobengine.selenio.address' startup parameter missing."); + } + check(nexioPort, "nexioPort"); + check(nexioUserName, "nexioUserName"); + check(nexioPassword, "nexioPassword"); + + sourceUri = manager.createStoreUri(RemoteStoreProtocol.FTP, nexioHost); + sourceUri.setPortNumber(nexioPort); + sourceUri.setUserName(nexioUserName); + sourceUri.setPassword(nexioPassword); + if (sourceUri == null) { + logger.error(marker, "A forrás nem elérhető."); + throw new NullPointerException("Internal error, missing 'sourceUri'."); + } + + targetUri = manager.createStoreUri(RemoteStoreProtocol.LOCAL, targetPath); + if (targetUri == null) { + logger.error(marker, "A cél nem elérhető."); + throw new NullPointerException("Internal error, missing 'targetUri'."); + } + + } + +} diff --git a/server/user.jobengine.executors/src/user/jobengine/server/steps/RecordingsArchiveItemBuilderStep.java b/server/user.jobengine.executors/src/user/jobengine/server/steps/RecordingsArchiveItemBuilderStep.java index 599a73c9..f1aa4e9b 100644 --- a/server/user.jobengine.executors/src/user/jobengine/server/steps/RecordingsArchiveItemBuilderStep.java +++ b/server/user.jobengine.executors/src/user/jobengine/server/steps/RecordingsArchiveItemBuilderStep.java @@ -29,7 +29,7 @@ import user.jobengine.server.IJobEngine; import user.jobengine.server.IJobRuntime; public class RecordingsArchiveItemBuilderStep extends JobStep { - private static final String RECORDING = "Visszarögzítés"; + private static final String MEDIATYPE = "Visszarögzített"; private static final Logger logger = LogManager.getLogger(); private static final String STATUSFOLDER = ".STATUS"; private static final String LXFEXT = ".lxf"; @@ -53,21 +53,14 @@ public class RecordingsArchiveItemBuilderStep extends JobStep { logger.error(marker, "A '{}' anyaghoz nem található tükör '{}' kezdéssel, ezért nem archiválható.", clipName, scheduledStart); return null; } - //DB db = NoSQLUtils.getNoSQLDB(); - - // - // BasicDBObject dbObject = (BasicDBObject) JSONUtil.jsonToDbObject(new String(readAllBytes)); - // if (dbObject == null) - // throw new NullPointerException("Can not parse JSON file: " + jsonFilePath); result = processRundow(octopusAPI, rundown); if (result == null) return null; result.setMediaTitle(clipName); - result.setMediaType(RECORDING); + result.setMediaType(MEDIATYPE); result.setMediaFile(mediaFilePath.toString()); result.setCatchedFile(catchedFilePath.toString()); - // result.setDuration(NoSQLUtils.asLong(dbObject, DURATION)); } catch (Exception e) { logger.catching(e); logger.error(getJobRuntime().getMarker(), "A metaadat nem elérhető. A rendszer üzenete: {}", e.getMessage()); @@ -77,11 +70,11 @@ public class RecordingsArchiveItemBuilderStep extends JobStep { return result; } - private void createCatchedFile(Path catchedFilePath) throws Exception { + private void createCatchedFile(Path catchedFilePath) throws IOException { try { EscortFiles.ensureUNCFolder(catchedFilePath.getParent()); Files.createFile(catchedFilePath); - } catch (Exception e) { + } catch (IOException e) { logger.catching(e); logger.error(marker, "A '{}' jelzőfájl nem hozható létre. A rendszer üzenete: {}", catchedFilePath, e.getMessage()); throw e; @@ -89,7 +82,7 @@ public class RecordingsArchiveItemBuilderStep extends JobStep { } @StepEntry - public Object[] execute(String sourcePath, IJobEngine jobEngine, IJobRuntime jobRuntime) { + public Object[] execute(String sourcePath, IJobEngine jobEngine, IJobRuntime jobRuntime) throws Exception { final ArchiveItem[] archiveItems = { null }; marker = getJobRuntime().getMarker(); try { @@ -124,9 +117,10 @@ public class RecordingsArchiveItemBuilderStep extends JobStep { ArchiveItem item = null; try { item = processPathItem(file); - } catch (Exception e) { + } catch (IOException e) { logger.catching(e); logger.error(marker, "Az '{}' állomány feldolgozása sikertelen. A rendszer hibaüzenete: {}", file, e.getMessage()); + throw e; } if (item == null) { result = FileVisitResult.CONTINUE; @@ -142,14 +136,16 @@ public class RecordingsArchiveItemBuilderStep extends JobStep { } catch (Exception e) { logger.catching(e); logger.error(marker, "Az '{}' mappa elérése sikertelen. A rendszer hibaüzenete: {}", sourcePath, e.getMessage()); + throw e; } finally { } ArchiveItem archiveItem = archiveItems[0]; String targetFileName = null; - if (archiveItem == null || archiveItem.getMediaFile() == null) + if (archiveItem == null || archiveItem.getMediaFile() == null) { logger.warn(marker, "Az archiváló folyamat nem talált új anyagot."); - else { + throw new Exception("No media to archive"); + } else { String mediaFile = archiveItem.getMediaFile(); String name = new File(mediaFile).getName(); int extPos = name.toLowerCase().lastIndexOf(LXFEXT); @@ -182,7 +178,7 @@ public class RecordingsArchiveItemBuilderStep extends JobStep { return CalendarUtils.createCalendar(CalendarUtils.createCalendar(recordDate), timePart).getTime(); } - private ArchiveItem processPathItem(Path mediaFilePath) throws Exception { + private ArchiveItem processPathItem(Path mediaFilePath) throws IOException { File mediaFile = mediaFilePath.toFile(); Path dotStorePath = Paths.get(mediaFilePath.getParent().toString(), STATUSFOLDER); @@ -276,6 +272,9 @@ public class RecordingsArchiveItemBuilderStep extends JobStep { } result.setMediaHouseId(result.getItemHouseId()); result.setMediaDescription(sb.toString()); + //TODO + if (result.getMediaDescription() != null && result.getMediaDescription().length() > 5000) + result.setMediaDescription(result.getMediaDescription().substring(0, 5000)); return result; } } diff --git a/server/user.jobengine.executors/src/user/jobengine/server/steps/TranscodeFFAStranStep.java b/server/user.jobengine.executors/src/user/jobengine/server/steps/TranscodeFFAStranStep.java new file mode 100644 index 00000000..93c5f4cd --- /dev/null +++ b/server/user.jobengine.executors/src/user/jobengine/server/steps/TranscodeFFAStranStep.java @@ -0,0 +1,94 @@ +package user.jobengine.server.steps; + +import java.io.File; +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.apache.logging.log4j.Marker; +import org.apache.logging.log4j.message.Message; +import org.apache.logging.log4j.message.ParameterizedMessage; + +import user.commons.ffastrans.FFAStransAPI; +import user.commons.ffastrans.IFFAStransAPI; +import user.jobengine.db.FileType; +import user.jobengine.db.IItemManager; +import user.jobengine.db.Media; +import user.jobengine.db.Store; +import user.jobengine.server.IJobEngine; +import user.jobengine.server.IJobRuntime; + +public class TranscodeFFAStranStep extends JobStep { + private static final String MP4EXT = ".mp4"; + private static final String MXFEXT = ".MXF"; + private static final String LOWRES_FILETYPE = "Low-res"; + private static final Logger logger = LogManager.getLogger(); + private IItemManager manager; + private Store store; + private FileType fileType; + private Media mediaCubeMedia; + private Marker marker; + + @StepEntry + public Object[] execute(ArchiveItem archiveItem, Media mediaCubeMedia, String transcoderAddress, String transcoderTemplateName, String localHiresSourcePath, + String globalHiresSourcePath, String localLowresTargetPath, String webPath, String sourcePath, boolean deleteSource, IJobEngine jobEngine, + IJobRuntime jobRuntime) throws Exception { + marker = jobRuntime.getMarker(); + manager = jobEngine.getItemManager(); + store = check(manager.getSystemStore(true), "lowres Store"); + fileType = check(manager.getFileType(LOWRES_FILETYPE), "lowres FileType"); + mediaCubeMedia = check(mediaCubeMedia, "mediaCubeMedia"); + check(archiveItem, "archiveItem"); + check(transcoderAddress, "transcoderAddress"); + check(transcoderTemplateName, "transcoderTemplateName"); + check(localHiresSourcePath, "localHiresSourcePath"); + check(globalHiresSourcePath, "globalHiresSourcePath"); + check(localLowresTargetPath, "localLowresTargetPath"); + check(webPath, "webPath"); + + File sourceMediaFile = new File(archiveItem.getMediaFile()); + String sourceFileName = sourceMediaFile.getName(); + String details = String.format("%s (%d bytes)", sourceFileName, sourceMediaFile.length()); + try { + String sourceFile = sourceMediaFile.toString().replace(localHiresSourcePath, globalHiresSourcePath); + IFFAStransAPI api = new FFAStransAPI(transcoderAddress, p -> { + jobRuntime.incrementProgress(p); + }); + api.submit(transcoderTemplateName, sourceFile); + jobRuntime.setDescription(String.format("%s: %s", jobRuntime.getDescription(), details)); + api.monitor(1000); + Path targetPath = Paths.get(localLowresTargetPath, sourceFileName.replace(MXFEXT, MP4EXT)); + postprocess(targetPath, webPath); + if (deleteSource) + sourceMediaFile.delete(); + } catch (Exception e) { + logger.catching(e); + Message m = new ParameterizedMessage("Az '{}' állomány átkódolása sikertelen. A rendszer hibaüzenete: {}", details, e.getMessage()); + logger.error(marker, m); + throw new Exception(m.getFormattedMessage()); + } + return null; + } + + private void postprocess(Path transcodedFilePath, String webPath) { + try { + String transcodedFileName = transcodedFilePath.getFileName().toString(); + if (transcodedFileName.indexOf(".") > 2) { + Path subdir = Paths.get(transcodedFileName.substring(0, 1), transcodedFileName.substring(1, 2), transcodedFileName.substring(2, 3)); + manager.createMediaFile(Paths.get(subdir.toString(), transcodedFileName).toString(), fileType, store, mediaCubeMedia).add(); + EscortFiles.ensureUNCFolder(webPath, subdir.toString()); + Files.move(transcodedFilePath, Paths.get(webPath, subdir.toString(), transcodedFileName)); + } else { + manager.createMediaFile(transcodedFileName, fileType, store, mediaCubeMedia).add(); + Files.move(transcodedFilePath, Paths.get(webPath, transcodedFileName)); + } + } catch (IOException e) { + logger.catching(e); + logger.error(marker, "A '{}' állomány mozgatása a '{}' mappába nem sikerült.", transcodedFilePath, webPath); + } + } + +} diff --git a/server/user.jobengine.executors/src/user/jobengine/server/steps/UploadRecordingToNexioStep.java b/server/user.jobengine.executors/src/user/jobengine/server/steps/UploadRecordingToNexioStep.java index d5e3b790..47cc941d 100644 --- a/server/user.jobengine.executors/src/user/jobengine/server/steps/UploadRecordingToNexioStep.java +++ b/server/user.jobengine.executors/src/user/jobengine/server/steps/UploadRecordingToNexioStep.java @@ -15,7 +15,6 @@ import com.ibm.nosql.json.api.DB; import user.commons.RemoteFile; import user.commons.StoreUri; import user.commons.nosql.NoSQLUtils; -import user.commons.octopus.OctopusAPI; import user.commons.remotestore.IProgressEventListener; import user.commons.remotestore.IStatusEventListener; import user.commons.remotestore.ProgressEvent; @@ -28,7 +27,6 @@ import user.jobengine.server.IJobRuntime; public class UploadRecordingToNexioStep extends JobStep { private static final Logger logger = LogManager.getLogger(); - private OctopusAPI octopusAPI; private IItemManager manager; private DB db; private StoreUri sourceUri; @@ -57,12 +55,14 @@ public class UploadRecordingToNexioStep extends JobStep { marker = jobRuntime.getMarker(); manager = jobEngine.getItemManager(); setAndCheck(archiveItem, targetFileName, nexioPort, nexioUserName, nexioPassword); - String sourceFileName = new File(archiveItem.getMediaFile()).getName(); + File sourceFile = new File(archiveItem.getMediaFile()); + String sourceFileName = sourceFile.getName(); try { + final IJobRuntime runtime = jobRuntime; sourceUri.addProgressListener(new IProgressEventListener() { @Override public void progressChanged(ProgressEvent evt) { - setProgress(evt.getProgress()); + runtime.incrementProgress(evt.getProgress()); } }); sourceUri.addStatusListener(new IStatusEventListener() { @@ -83,14 +83,17 @@ public class UploadRecordingToNexioStep extends JobStep { RemoteFile remoteFile = sourceUri.transferFrom(targetUri, sourceFileName, targetFileName); - logger.info(marker, "Az '{}' állomány feltöltése sikeres volt '{}' néven.", sourceFileName, targetFileName); + logger.info(marker, "Az '{}' állomány feltöltése sikeres volt '{}' néven.", sourceFile, targetFileName); } catch (Exception e) { logger.catching(e); - Message m = new ParameterizedMessage("Az '{}' állomány feltöltése sikertelen. A rendszer hibaüzenete: {}", sourceFileName, e.getMessage()); + if (!archiveItem.removeCatchedFile()) + logger.error(getMarker(), "A {} állomány .catched jelző állománya nem törölhető.", new File(archiveItem.getMediaFile()).getName()); + Message m = new ParameterizedMessage("Az '{}' állomány feltöltése '{}' néven sikertelen. A rendszer hibaüzenete: {}", sourceFile, targetFileName, + e.getMessage()); logger.error(marker, m); throw new Exception(m.getFormattedMessage()); } - return null; + return new Object[] {}; } private void setAndCheck(ArchiveItem archiveItem, String targetFileName, int nexioPort, String nexioUserName, String nexioPassword) throws Exception { @@ -122,6 +125,7 @@ public class UploadRecordingToNexioStep extends JobStep { check(nexioPassword, "nexioPassword"); targetUri = manager.createStoreUri(RemoteStoreProtocol.FTP, nexioHost); + targetUri.setRootPath("LXF"); targetUri.setPortNumber(nexioPort); targetUri.setUserName(nexioUserName); targetUri.setPassword(nexioPassword); diff --git a/server/user.jobengine.executors/test-deploy-steps-to-bsh-bkup.bat b/server/user.jobengine.executors/test-deploy-steps-to-bsh-bkup.bat index ce7ba733..8ad41633 100644 --- a/server/user.jobengine.executors/test-deploy-steps-to-bsh-bkup.bat +++ b/server/user.jobengine.executors/test-deploy-steps-to-bsh-bkup.bat @@ -1,32 +1,30 @@ @ECHO OFF SET REMOTE_HOST=10.10.1.29 -SET REMOTE_LOCATION=/opt/test-mediacube/configuration/executors SET REMOTE_SERVER_HOSTKEY=ssh-ed25519 256 ea:ab:67:70:79:63:2f:6a:34:81:48:e2:b9:dd:ca:d4 SET REMOTE_SERVER_ADDRESS=scp://root:password@%REMOTE_HOST% + SET LOCAL_STEPS_LOCATION=bin/user/jobengine/server/steps -SET LOCAL_CONFIG_LOCATION=config +SET REMOTE_STEPS_LOCATION=/opt/test-mediacube/configuration/executors + +SET LOCAL_TEMPLATES_LOCATION=jobtemplates +SET REMOTE_TEMPLATES_LOCATION=/opt/test-mediacube/configuration/jobtemplates - ECHO *** Begin deploy steps to server %REMOTE_HOST% *** - ECHO --- Deploying - + ECHO *** Deploy steps to server %REMOTE_HOST% *** WinSCP.com /command ^ "open %REMOTE_SERVER_ADDRESS% -hostkey=""%REMOTE_SERVER_HOSTKEY%"" -timeout=60" ^ - "cd %REMOTE_LOCATION%" ^ "lcd %LOCAL_STEPS_LOCATION%" ^ + "cd %REMOTE_STEPS_LOCATION%" ^ "synchronize remote" ^ - "lcd ..\" ^ - "lcd ..\" ^ - "lcd ..\" ^ - "lcd ..\" ^ - "lcd ..\" ^ - "lcd %LOCAL_CONFIG_LOCATION%" ^ - "synchronize remote -filemask=config.xml" ^ "exit" - - ECHO *** Completed *** -@ECHO ON -REM "cd .." ^ -REM "synchronize remote -filemask=scheduledjobs.json" ^ + ECHO *** Deploy templates to server %REMOTE_HOST% *** + WinSCP.com /command ^ + "open %REMOTE_SERVER_ADDRESS% -hostkey=""%REMOTE_SERVER_HOSTKEY%"" -timeout=60" ^ + "lcd %LOCAL_TEMPLATES_LOCATION%" ^ + "cd %REMOTE_TEMPLATES_LOCATION%" ^ + "synchronize remote" ^ + "exit" + +@ECHO ON diff --git a/server/user.jobengine.executors/test-deploy-steps-to-bsh-main.bat b/server/user.jobengine.executors/test-deploy-steps-to-bsh-main.batx similarity index 100% rename from server/user.jobengine.executors/test-deploy-steps-to-bsh-main.bat rename to server/user.jobengine.executors/test-deploy-steps-to-bsh-main.batx diff --git a/server/user.jobengine.osgi.commons/META-INF/MANIFEST.MF b/server/user.jobengine.osgi.commons/META-INF/MANIFEST.MF index 51f1af6f..02b6b369 100644 --- a/server/user.jobengine.osgi.commons/META-INF/MANIFEST.MF +++ b/server/user.jobengine.osgi.commons/META-INF/MANIFEST.MF @@ -19,6 +19,7 @@ DynamicImport-Package: * Bundle-ClassPath: . Service-Component: OSGI-INF/nexioAPI.xml, OSGI-INF/octopusAPI.xml Export-Package: user.commons, + user.commons.ffastrans, user.commons.ftp, user.commons.logging, user.commons.morpheus, diff --git a/server/user.jobengine.osgi.commons/src/user/commons/ffastrans/FFAStransAPI.java b/server/user.jobengine.osgi.commons/src/user/commons/ffastrans/FFAStransAPI.java index 99f420e0..7d47d4d8 100644 --- a/server/user.jobengine.osgi.commons/src/user/commons/ffastrans/FFAStransAPI.java +++ b/server/user.jobengine.osgi.commons/src/user/commons/ffastrans/FFAStransAPI.java @@ -6,6 +6,7 @@ import javax.ws.rs.client.Entity; import javax.ws.rs.core.MediaType; import javax.ws.rs.core.Response; +import org.apache.commons.lang.StringUtils; import org.jboss.resteasy.client.jaxrs.ResteasyClientBuilder; import org.jboss.resteasy.client.jaxrs.ResteasyWebTarget; @@ -16,42 +17,49 @@ import user.commons.nosql.NoSQLUtils; public class FFAStransAPI implements IFFAStransAPI { - public static void main(String[] args) throws Exception { + public interface IProgressChangedListener { + void onProgressChanged(int progress); + } - IFFAStransAPI api = new FFAStransAPI("http://10.10.1.74:65445/api/json/v1/"); - List workflows = api.getWorkflows(); - for (BasicDBObject workflow : workflows) { - //System.out.println(workflow.toPrettyString(null)); - if (!"MAM-Proxy".equals(workflow.getString("wf_name"))) - continue; - long wfID = workflow.getLong("wf_id"); - System.out.println("MAM-Proxy wf_id: " + wfID); - - String path = "\\\\PROXY-TRANSCODER-01\\MAM-Proxy_input\\test_09.mxf"; - String jobID = api.submit(wfID, path); - System.out.println("Submited: " + jobID); - - String jobStart = null; - while (true) { - Thread.sleep(1000); - BasicDBObject status = api.getStatus(jobID); - if (status != null) { - System.out.println("Status: " + status.toPrettyString(null)); - if (jobStart == null) - jobStart = status.getString("job_start"); - } else { - BasicDBObject history = api.getHistory(jobStart); - System.out.println("History: " + history.toPrettyString(null)); - break; - } + public static void main(String[] args) throws Exception { + for (int i = 0; i < 1; i++) { + IFFAStransAPI api = new FFAStransAPI("http://10.10.1.74:65445/api/json/v1/", p -> { + System.out.println("Progress: " + p); + }); + + String inputFile = "\\\\10.10.254.74\\temp_isilon\\NEXIO\\HIRADO\\180201\\0700_hirado_CLN_-_CS.lxf"; + api.submit("MAM_proxy_copy", inputFile); + try { + api.monitor(1000); + } catch (Exception e) { + System.out.println(e.getMessage()); } + // try { + // + // String outputFile = "\\\\10.10.1.74\\MAM-Proxy_output\\20180201-0700_hirado_TEST-_CS.mp4"; + // Path outputFilePath = Paths.get(outputFile); + // File file = outputFilePath.toFile(); + // if (!file.exists()) + // Thread.sleep(2000); + // Files.delete(outputFilePath); + // } catch (Exception e) { + // System.out.println(e.getMessage()); + // } + } } private ResteasyWebTarget webTarget; - public FFAStransAPI(String apiAddress) { + private IProgressChangedListener listener; + + private String inputFile; + + private String jobId; + + public FFAStransAPI(String apiAddress, IProgressChangedListener listener) { + this.listener = listener; webTarget = new ResteasyClientBuilder().build().target(apiAddress); } @@ -69,7 +77,9 @@ public class FFAStransAPI implements IFFAStransAPI { for (BasicDBObject job : jobs) { if (job == null) continue; - if (jobStart.equals(job.getString("job_start"))) { + + String job_start = NoSQLUtils.asString(job, "job_start"); + if (jobStart.equals(job_start)) { result = job; break; } @@ -109,6 +119,15 @@ public class FFAStransAPI implements IFFAStransAPI { return result; } + private long getWorkflowId(String workflowName, List workflows, long wfID) { + for (BasicDBObject workflow : workflows) { + if (!workflowName.equals(workflow.getString("wf_name"))) + continue; + wfID = workflow.getLong("wf_id"); + } + return wfID; + } + @Override public List getWorkflows() { ResteasyWebTarget target = webTarget.path("workflows"); @@ -121,14 +140,75 @@ public class FFAStransAPI implements IFFAStransAPI { } @Override - public String submit(long wfID, String path) { - BasicDBObject job = new BasicDBObject("wf_id", wfID).append("inputfile", path); + public void monitor(int pollIntervall) throws InterruptedException, Exception { + String jobStart = null; + int progress = 0; + while (true) { + Thread.sleep(pollIntervall); + BasicDBObject status = getStatus(jobId); + if (status != null) { + //System.out.println("Status: " + status.toPrettyString(null)); + if (jobStart == null) + jobStart = status.getString("job_start"); + + List splits = NoSQLUtils.asList(status, "splits"); + if (splits != null && splits.size() > 0) { + String processor = NoSQLUtils.asString(splits.get(0), "processor"); + if (StringUtils.isNotBlank(processor) && "Generate text file".equals(processor)) + continue; + String prg = NoSQLUtils.asString(splits.get(0), "progress"); + int current = (int) Float.parseFloat(prg); + //System.out.println(String.format("%s %s %s", current, progress, current != progress)); + if (current != progress) { + progress = current; + listener.onProgressChanged(progress); + } + } + //Status: {"job_id":"20180226-162821-217-A7E91DC625BD","job_start":"2018/02/26 16:28:21","file":"\\\\PROXY-TRANSCODER-01\\MAM-Proxy_input\\20180201-0700_hirado_TEST-_CS.MXF","wf_name":"MAM_proxy","splits":[{"steps":"4 / 5","processor":"Folder","status":"Waiting for next processor resources...","node":"PROXY-TRANSCODE","progress":"78.5"}]} + + } else { + //System.out.println("Progress: " + 100); + listener.onProgressChanged(100); + BasicDBObject history = getHistory(jobStart); + //System.out.println("History: " + history.toPrettyString(null)); + if (history == null || NoSQLUtils.asLong(history, "state") != 1) { + String error = NoSQLUtils.asString(history, "outcome"); + throw new Exception("Transcode error: " + error); + } else { + //System.out.println("Transcode completed"); + break; + } + + //History: {"wf_name":"MAM_proxy","job_start":"2018/02/26 16:28:21","job_end":"2018/02/26 16:28:36","file":"20180201-0700_hirado_TEST-_CS.MXF","outcome":"Success","state":1} + } + + } + } + + @Override + public void submit(String workflowName, String inputFile) throws Exception { + this.inputFile = inputFile; + List workflows = getWorkflows(); + if (workflows == null) + throw new Exception("No workflows"); + + long wfID = -1; + wfID = getWorkflowId(workflowName, workflows, wfID); + if (wfID < 0) + throw new Exception("Workflow not exists: " + workflowName); + + BasicDBObject job = new BasicDBObject("wf_id", wfID).append("inputfile", inputFile); ResteasyWebTarget target = webTarget.path("jobs"); Response apiResponse = target.request().post(Entity.entity(job.toString(), MediaType.APPLICATION_JSON)); if (apiResponse.getStatus() != 202) - return null; + throw new Exception("Can not submit, response status is: " + apiResponse.getStatus()); String json = apiResponse.readEntity(String.class); + + if (StringUtils.isBlank(json)) + throw new Exception("Can not submit, response JSON is empty"); BasicDBObject resultObject = (BasicDBObject) JSONUtil.jsonToDbObject(json); - return resultObject.getString("job_id"); + if (resultObject == null) + throw new Exception("Can not submit, response object is null"); + jobId = resultObject.getString("job_id"); } } diff --git a/server/user.jobengine.osgi.commons/src/user/commons/ffastrans/IFFAStransAPI.java b/server/user.jobengine.osgi.commons/src/user/commons/ffastrans/IFFAStransAPI.java index 1bb9ca10..6c59e5b6 100644 --- a/server/user.jobengine.osgi.commons/src/user/commons/ffastrans/IFFAStransAPI.java +++ b/server/user.jobengine.osgi.commons/src/user/commons/ffastrans/IFFAStransAPI.java @@ -12,5 +12,7 @@ public interface IFFAStransAPI { List getWorkflows(); - String submit(long wfID, String path); + void monitor(int pollIntervall) throws InterruptedException, Exception; + + void submit(String workflowName, String path) throws Exception; } diff --git a/server/user.jobengine.osgi.db/sql/5-CreateFTIndex.db2 b/server/user.jobengine.osgi.db/sql/5-CreateFTIndex.db2 index 59a905aa..4b9fcfd1 100644 --- a/server/user.jobengine.osgi.db/sql/5-CreateFTIndex.db2 +++ b/server/user.jobengine.osgi.db/sql/5-CreateFTIndex.db2 @@ -27,3 +27,13 @@ db2ts "update index media_desc for text connect to mc" db2ts "update index media_houseid for text connect to mc" db2ts "update index mediafile_houseid for text connect to mc" + +db2ts "create index media_title1 for text on clobmedia(title) UPDATE FREQUENCY D(*) H(*) M(0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31,32,33,34,35,36,37,38,39,40,41,42,43,44,45,46,47,48,49,50,51,52,53,54,55,56,57,58,59) UPDATE MINIMUM 1 connect to mc" +db2ts "create index media_desc1 for text on clobmedia(description) UPDATE FREQUENCY D(*) H(*) M(0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31,32,33,34,35,36,37,38,39,40,41,42,43,44,45,46,47,48,49,50,51,52,53,54,55,56,57,58,59) UPDATE MINIMUM 1 connect to mc" +db2ts "create index media_houseid1 for text on clobmedia(houseid) UPDATE FREQUENCY D(*) H(*) M(0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31,32,33,34,35,36,37,38,39,40,41,42,43,44,45,46,47,48,49,50,51,52,53,54,55,56,57,58,59) UPDATE MINIMUM 1 connect to mc" +db2ts "update index media_title1 for text connect to mc" +db2ts "update index media_desc1 for text connect to mc" +db2ts "update index media_houseid1 for text connect to mc" +db2ts "drop index media_title1 for text connect to mc" +db2ts "drop index media_desc1 for text connect to mc" +db2ts "drop index media_houseid1 for text connect to mc" diff --git a/server/user.jobengine.osgi.db/sql/stat.sql b/server/user.jobengine.osgi.db/sql/stat.sql index ac732e9a..f0995e31 100644 --- a/server/user.jobengine.osgi.db/sql/stat.sql +++ b/server/user.jobengine.osgi.db/sql/stat.sql @@ -2,11 +2,14 @@ select * from vw_items where mediacreated like '2018-01%' select distinct(mediafilehouseid) from vw_items where mediacreated like '2018-01%' select count(distinct(mediafilehouseid)) from vw_items where mediacreated like '2018-01%' -select count(distinct(mediafilehouseid)) from vw_items + --Octopus ID-hez rendelt klip select * from vw_items where itemtitle like 'Echo TV%' and mediacreated like '2018-01%' -select count(distinct(mediahouseid)) from vw_items where itemtitle like 'Echo TV%' and mediacreated like '2018-01%' +select count(distinct(mediafilehouseid)) from vw_items where itemtitle like 'Echo TV%' and mediacreated like '2018-01%' + + +select * from vw_items where mediacreated like '2018-01%' --Automatikusan archivált bejátszók select * from vw_items where itemtitle like '%Echo TV%' and itemtitle like '2%' and mediatitle not like '%CLN%' and mediatitle not like '%PGM%' and mediacreated like '2018-01%' @@ -20,6 +23,13 @@ select count(distinct(mediafilehouseid)) from vw_items where itemtitle like '%Ec select * from vw_items where itemtitle like 'MC-%' and mediahouseid like 'MC-%'and mediacreated like '2018-01%' select count(distinct(mediafilehouseid)) from vw_items where itemtitle like 'MC-%' and mediahouseid like 'MC-%'and mediacreated like '2018-01%' + +select archived, mediahouseid, mediatitle, mediafilehouseid from vw_items where MEDIAFILEHOUSEID in ( + select mediafilehouseid from vw_items where mediahouseid like 'MC-%' and mediatitle not like 'MC-%' + group by mediafilehouseid having count(*) > 1 +) + + --Migrált archívum select i.created, i.houseid as ihouse, i.title as ititle, m.houseid as mhouse, m.title as mtitle from item i left outer join media m on (m.itemid = i.id) where i.houseid not like 'MC-%' and m.houseid like 'MC-%' @@ -40,8 +50,28 @@ select * from vw_items where mediafilehouseid like 'M%' and mediafilehouseid not select count(distinct(mediafilehouseid)) from vw_items where mediafilehouseid like 'M%' and mediafilehouseid not like 'MC%' and mediacreated like '2018-01%' -- Egyéb ? -select * from vw_items where mediafilehouseid not like 'M%' and mediahouseid not like 'P%' and mediahouseid not like 'R%' and mediafilehouseid not like 'MC%' and mediacreated like '2018-01%' + +select mediafilehouseid, count(*) as c from vw_items where mediafilehouseid not like 'M%' and mediahouseid not like 'P%' and mediahouseid not like 'R%' and mediafilehouseid not like 'MC%' and mediacreated like '2018-01%' and itemtitle not like '%Echo TV%' and itemtitle not like '2%' and mediatitle not like '%CLN%' and mediatitle not like '%PGM%' +group by mediafilehouseid having count(*) > 1 order by c desc + +select archived, mediahouseid, mediatitle, mediafilehouseid from vw_items where MEDIAFILEHOUSEID in ( + select mediafilehouseid from vw_items where mediafilehouseid not like 'M%' and mediahouseid not like 'P%' and mediahouseid not like 'R%' and mediafilehouseid not like 'MC%' + and itemtitle not like '%Echo TV%' and itemtitle not like '2%' and mediatitle not like '%CLN%' and mediatitle not like '%PGM%' + group by mediafilehouseid having count(*) > 1 +) + +--ID-k +select x.archived, x.mediahouseid, x.mediatitle, x.mediafilehouseid, h.duplicates from vw_items x +inner join ( + select mediafilehouseid, count(*) as duplicates from vw_items group by mediafilehouseid order by mediafilehouseid +) h on x.mediafilehouseid=h.mediafilehouseid and +x.mediafilehouseid not like 'M%' and x.mediahouseid not like 'P%' and x.mediahouseid not like 'R%' and x.mediafilehouseid not like 'MC%' and x.mediacreated like '2018-01%' +and x.itemtitle not like '%Echo TV%' and x.itemtitle not like '2%' and x.mediatitle not like '%CLN%' and x.mediatitle not like '%PGM%' + +select * from WORKFLOWACTION where started like '2018-01%' + +select min(started) from WORKFLOWACTION where USERNAME!='echotest' and started like '2018-02%' select count(distinct(mediafilehouseid)) from vw_items where mediafilehouseid not like 'M%' and mediahouseid not like 'P%' and mediahouseid not like 'R%' and mediafilehouseid not like 'MC%' and mediacreated like '2018-01%' and itemtitle not like '%Echo TV%' and itemtitle not like '2%' and mediatitle not like '%CLN%' and mediatitle not like '%PGM%' @@ -49,6 +79,11 @@ and itemtitle not like '%Echo TV%' and itemtitle not like '2%' and mediatitle no ----------------------------------------------------------------- +select archived, mediahouseid, mediatitle, mediafilehouseid from vw_items where MEDIAFILEHOUSEID in ( + select mediafilehouseid, count(*) from vw_items where mediafilehouseid not like 'MC-%' and mediafilehouseid like 'M%' + group by mediafilehouseid having count(*) > 1 +) + select * from vw_items where mediahouseid like 'P%' select count(distinct(mediahouseid)) from vw_items where mediahouseid like 'P%' diff --git a/server/user.jobengine.osgi.db/test/user/jobengine/db/Support.java b/server/user.jobengine.osgi.db/test/user/jobengine/db/Support.java index 01641e04..f5c26fe4 100644 --- a/server/user.jobengine.osgi.db/test/user/jobengine/db/Support.java +++ b/server/user.jobengine.osgi.db/test/user/jobengine/db/Support.java @@ -21,6 +21,7 @@ import org.junit.AfterClass; import org.junit.BeforeClass; import org.junit.Test; +import sqlj.runtime.ref.DefaultContext; import user.commons.IEntityBase; import user.commons.logging.LogUtils; @@ -177,4 +178,42 @@ public class Support { return result; } + @Test + public void testClob() throws Exception { + ResultSet rs = null; + PreparedStatement st = null; + DefaultContext context = ((ItemManager) manager).getDbContext(); + Connection connection = context.getConnection(); + try { + + String query = String.format("select text from ftclob"); + st = connection.prepareStatement(query); + rs = st.executeQuery(); + if (rs.next()) { + String text = rs.getString("text"); + System.out.println(text); + } + + connection.commit(); + } catch (Exception e) { + try { + connection.rollback(); + } catch (Exception e1) { + } + } finally { + try { + if (rs != null) + rs.close(); + } catch (Exception e1) { + } + try { + if (st != null) + st.close(); + } catch (Exception e1) { + } + ((ItemManager) manager).putDbContext(context); + } + + } + } diff --git a/server/user.jobengine.osgi.server/src/user/jobengine/server/steps/JobStep.java b/server/user.jobengine.osgi.server/src/user/jobengine/server/steps/JobStep.java index 4338eeb4..79b8b9e7 100644 --- a/server/user.jobengine.osgi.server/src/user/jobengine/server/steps/JobStep.java +++ b/server/user.jobengine.osgi.server/src/user/jobengine/server/steps/JobStep.java @@ -36,6 +36,20 @@ public class JobStep implements IJobStep { return !jobRuntime.isWaiting(); } + protected T check(T value, String name) { + boolean empty = false; + if (value instanceof Long || value instanceof Integer) { + long i = (long) value; + if (i == 0) + empty = true; + } + if (value == null || empty) { + logger.error(getMarker(), "A folyamat '{}' bemeneti paramétere üres.", name); + throw new NullPointerException(String.format("System is not configured properly, missing '%s' input parameter.", name)); + } + return value; + } + @Override public void cleanup() { }