From: Vásáry Dániel Date: Wed, 6 Nov 2019 12:40:09 +0000 (+0000) Subject: git-tfs-id: [http://tfs.userrendszerhaz.hu:8080/tfs/DefaultCollection]$/MediaCube... X-Git-Url: http://git.useribm.hu/?a=commitdiff_plain;h=0adf5ead7091cde2cb759596db640f460fc47bed;p=mediacube.git git-tfs-id: [tfs.userrendszerhaz.hu:8080/tfs/DefaultCollection]$/MediaCube;C31619 --- diff --git a/server/-configuration/log4j2-test.xml b/server/-configuration/log4j2-test.xml new file mode 100644 index 00000000..dd800b0a --- /dev/null +++ b/server/-configuration/log4j2-test.xml @@ -0,0 +1,16 @@ + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/server/-configuration/mediacube.json b/server/-configuration/mediacube.json index 4f38637d..5e36e7f5 100644 --- a/server/-configuration/mediacube.json +++ b/server/-configuration/mediacube.json @@ -75,9 +75,9 @@ ], "authentication": { "authEnabled": true, - "adHost": "10.10.254.11", + "adHost": "intra.mediavivantis.hu", "adNonSecurePort": 3268, - "adBaseDn": "DC=intra,DC=echotv,DC=hu", + "adBaseDn": "DC=intra,DC=mediavivantis,DC=hu", "adAdminMap": [ "G_ECH_U_INFORMATIKUSOK", "G_ECH_U_MUSZAKVEZETOK", diff --git a/server/hu.user.mediacube.executors.tests/src/hu/user/mediacube/executors/tests/HSMMigrateListBuilderStepTest.java b/server/hu.user.mediacube.executors.tests/src/hu/user/mediacube/executors/tests/HSMMigrateStepTest.java similarity index 82% rename from server/hu.user.mediacube.executors.tests/src/hu/user/mediacube/executors/tests/HSMMigrateListBuilderStepTest.java rename to server/hu.user.mediacube.executors.tests/src/hu/user/mediacube/executors/tests/HSMMigrateStepTest.java index 95577332..44c216d5 100644 --- a/server/hu.user.mediacube.executors.tests/src/hu/user/mediacube/executors/tests/HSMMigrateListBuilderStepTest.java +++ b/server/hu.user.mediacube.executors.tests/src/hu/user/mediacube/executors/tests/HSMMigrateStepTest.java @@ -3,6 +3,8 @@ package hu.user.mediacube.executors.tests; import static org.junit.Assert.assertNotNull; import static org.junit.Assert.assertNull; +import java.nio.file.Paths; + import org.junit.BeforeClass; import org.junit.Test; @@ -14,7 +16,7 @@ import user.mediacube.metadata.interfaces.IMetadataProvider; import user.mediacube.metadata.interfaces.IMetadataProviderFactory; import user.mediacube.metadata.interfaces.MetadataProviderType; -public class HSMMigrateListBuilderStepTest { +public class HSMMigrateStepTest { @BeforeClass static public void setUpConnection() { @@ -68,4 +70,12 @@ public class HSMMigrateListBuilderStepTest { BasicDBObject actual = sut.queryFileHistory("/OMARCHIVE/6.3.4.200-TIV-TSMALL-Linuxx86_64.bin"); assertNotNull(actual); } + + @Test + public void testResumableCopy() throws Exception { + HSMMigrateStep sut = new HSMMigrateStep(); + Paths.get("c:/_video/03c.mp4").toFile().delete(); + sut.copyChunk(Paths.get("c:/_video/1.txt"), Paths.get("c:/_video/2.txt"), 5); + sut.resumeableCopy(Paths.get("c:/_video/1.txt"), Paths.get("c:/_video/2.txt")); + } } diff --git a/server/hu.user.mediacube.executors.tests/src/hu/user/mediacube/executors/tests/SmallTests.java b/server/hu.user.mediacube.executors.tests/src/hu/user/mediacube/executors/tests/SmallTests.java index 0b37d49a..4959df0d 100644 --- a/server/hu.user.mediacube.executors.tests/src/hu/user/mediacube/executors/tests/SmallTests.java +++ b/server/hu.user.mediacube.executors.tests/src/hu/user/mediacube/executors/tests/SmallTests.java @@ -1,8 +1,11 @@ package hu.user.mediacube.executors.tests; +import java.io.File; +import java.nio.file.Files; import java.util.Arrays; import java.util.List; +import org.apache.commons.io.FilenameUtils; import org.junit.Test; import user.jobengine.db.Media; @@ -20,6 +23,16 @@ public class SmallTests { class PojoRoot { } + @Test + public void test0() throws Exception { + File f = new File("/opt/test.txt"); + Files.write(f.toPath(), "AAA".getBytes()); + System.out.println(f.length()); + Files.write(f.toPath(), "AAAAAA".getBytes()); + System.out.println(f.length()); + Files.delete(f.toPath()); + } + @Test public void test1() throws Exception { Object[] sut = { 1, 5, "valami", new Pojo() }; @@ -64,6 +77,12 @@ public class SmallTests { } + @Test + public void test5() throws Exception { + System.out.println(FilenameUtils.removeExtension("valami.txt") + ".mp4"); + + } + void xxx(int x, boolean y) { System.out.println("call"); } diff --git a/server/user.jobengine.executors/jobtemplates/archive-material.xml b/server/user.jobengine.executors/jobtemplates/archive-material.xml index 63ba7c37..bfb5cd28 100644 --- a/server/user.jobengine.executors/jobtemplates/archive-material.xml +++ b/server/user.jobengine.executors/jobtemplates/archive-material.xml @@ -60,5 +60,15 @@ + + + + + + + + + + \ No newline at end of file diff --git a/server/user.jobengine.executors/src/user/jobengine/server/steps/ArchiveListBuilderStep.java b/server/user.jobengine.executors/src/user/jobengine/server/steps/ArchiveListBuilderStep.java index 9f67384b..7774de97 100644 --- a/server/user.jobengine.executors/src/user/jobengine/server/steps/ArchiveListBuilderStep.java +++ b/server/user.jobengine.executors/src/user/jobengine/server/steps/ArchiveListBuilderStep.java @@ -31,8 +31,6 @@ public class ArchiveListBuilderStep extends JobStep { private static final Logger logger = LogManager.getLogger(); // private static final String UTF8 = "utf-8"; private static final String STATUSFOLDER = ".STATUS"; - private static final String MXFEXT = ".mxf"; - private static final String WAVEXT = ".wav"; private static final String JSONEXT = ".json"; private static final String CATCHEDEXT = ".catched"; @@ -137,8 +135,7 @@ public class ArchiveListBuilderStep extends JobStep { // if (mediaFile.length() > 0) // return false; - if (mediaFile.isDirectory() || !mediaFile.getName().toLowerCase().endsWith(MXFEXT.toLowerCase()) - || mediaFile.getName().toLowerCase().endsWith(WAVEXT.toLowerCase())) { + if (mediaFile.isDirectory()) { return false; } diff --git a/server/user.jobengine.executors/src/user/jobengine/server/steps/HSMMigrateStep.java b/server/user.jobengine.executors/src/user/jobengine/server/steps/HSMMigrateStep.java index 5e2d2a14..3dcaeb11 100644 --- a/server/user.jobengine.executors/src/user/jobengine/server/steps/HSMMigrateStep.java +++ b/server/user.jobengine.executors/src/user/jobengine/server/steps/HSMMigrateStep.java @@ -1,10 +1,18 @@ package user.jobengine.server.steps; +import java.io.BufferedInputStream; +import java.io.BufferedOutputStream; import java.io.File; +import java.io.FileInputStream; +import java.io.FileOutputStream; import java.io.IOException; +import java.io.InputStream; +import java.io.OutputStream; import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.Paths; +import java.nio.file.attribute.BasicFileAttributes; +import java.text.SimpleDateFormat; import java.util.Arrays; import java.util.LinkedHashMap; import java.util.List; @@ -26,6 +34,8 @@ import user.mediacube.metadata.interfaces.IMetadata; import user.mediacube.metadata.interfaces.IMetadataProvider; import user.mediacube.metadata.interfaces.IMetadataProviderFactory; import user.mediacube.metadata.interfaces.MetadataProviderType; +import user.mediacube.metadata.interfaces.MetadataType; +import user.mediacube.metadata.interfaces.PlanAirMetadataListOptions; public class HSMMigrateStep extends JobStep { private static final Logger logger = LogManager.getLogger(); @@ -39,6 +49,7 @@ public class HSMMigrateStep extends JobStep { private DBCollection volumeHistory; private DB db; private IMetadataProvider planairProvider; + private SimpleDateFormat df = new SimpleDateFormat("yyyy.MM.dd"); private void cleanupHistory() { if (db == null) @@ -51,28 +62,56 @@ public class HSMMigrateStep extends JobStep { volumeHistory.drop(); } - private BasicDBObject createMetadata(String fileName) { + public void copyChunk(Path source, Path target, long chunk) throws IOException { - Path filePath = Paths.get(fileName); - String mediaHouseId = FilenameUtils.removeExtension(filePath.getFileName().toString()); + File sourceFile = source.toFile(); + File targetFile = target.toFile(); + + try (InputStream in = new BufferedInputStream(new FileInputStream(sourceFile)); + OutputStream out = new BufferedOutputStream(new FileOutputStream(targetFile))) { + + byte[] buffer = new byte[1]; + long copied = 0; + int lengthRead; + while ((lengthRead = in.read(buffer)) > 0) { + out.write(buffer, 0, lengthRead); + out.flush(); - //get meta + copied += lengthRead; + if (copied > chunk) { + out.close(); + in.close(); + return; + } + + } + } - String itemHouseId = "itemHouseId"; - String itemTitle = "itemTitle"; - String itemDesc = "itemDesc"; - String mediaTitle = "mediaTitle"; - String mediaDesc = "mediaDesc"; + } + + private BasicDBObject createMetadata(String volumeName, String fileName) throws Exception { + + Path filePath = Paths.get(fileName); + String mediaHouseId = FilenameUtils.removeExtension(filePath.getFileName().toString()); + BasicDBObject result = null; + try { + result = getPlanAirMetadata(mediaHouseId); + } catch (Exception e) { + logger.error("PlanAir metadata error", e); + throw e; + } - BasicDBObject result = new BasicDBObject(); - result.put("itemHouseId", itemHouseId); - result.put("itemTitle", itemTitle); - result.put("itemDescription", itemDesc); + if (result == null) { + result = new BasicDBObject(); + BasicFileAttributes attr = Files.readAttributes(filePath, BasicFileAttributes.class); + result.put("itemHouseId", df.format(attr.lastModifiedTime().toMillis())); + result.put("itemTitle", filePath.getParent().toString()); + result.put("mediaHouseId", mediaHouseId); + result.put("mediaTitle", fileName); + result.put("mediaDescription", volumeName); + result.put("mediaType", "Generic"); + } result.put("userName", "mediacube"); - result.put("mediaHouseId", fileName); - result.put("mediaTitle", mediaTitle); - result.put("mediaDescription", mediaDesc); - result.put("mediaType", "Generic"); return result; } @@ -82,7 +121,7 @@ public class HSMMigrateStep extends JobStep { if (jobRuntime != null) marker = jobRuntime.getMarker(); //remove from prod - cleanupHistory(); + //cleanupHistory(); hsmProvider = getMetadataProvider(MetadataProviderType.HSM); if (hsmProvider == null) throw new NullPointerException("No HSM metadata provider available"); @@ -103,35 +142,42 @@ public class HSMMigrateStep extends JobStep { logger.info(marker, "A kazetta feldolgozása elindul: {}", volumeName); List contents = getContents(volumeName); int p = 0; - boolean success = false; + boolean oneSuccess = false; + boolean hasError = false; for (IMetadata content : contents) { BasicDBObject c = content.asJSON(); - String contentFileName = c.getString("fileName"); + String hsmFileName = c.getString("fileName"); long contentFileSize = NoSQLUtils.asLong(c, "fileSize"); - if (!tapeContents.containsKey(contentFileName)) { - tapeContents.put(contentFileName, c); - Path sourceFilePath = Paths.get(sourceLocation, contentFileName); + if (!tapeContents.containsKey(hsmFileName)) { + tapeContents.put(hsmFileName, c); + Path sourceFilePath = Paths.get(sourceLocation, hsmFileName); if (contentFileSize < Files.getFileStore(targetPath).getUsableSpace()) { Path targetFilePath = Paths.get(targetLocation, sourceFilePath.getFileName().toString()); - if (processPathItem(contentFileName, sourceFilePath, targetFilePath)) { - success = true; - //throw new Exception("Teszt"); + + try { + if (processHSMFile(volumeName, hsmFileName, sourceFilePath, targetFilePath)) { + oneSuccess = true; + } + } catch (Exception e) { + hasError = true; } } } setProgress(p++ * 100 / contents.size()); } - saveVolumeHistory(volumeName); - String subject = "A kazetta eltávolítható a HSM rendszerből: " + volumeName; - logger.info(new MediaCubeMarker() { - { - setSubject(subject); - } - }, subject); + if (oneSuccess && !hasError) { + saveVolumeHistory(volumeName); + String subject = "A kazetta eltávolítható a HSM rendszerből: " + volumeName; + logger.info(new MediaCubeMarker() { + { + setSubject(subject); + } + }, subject); + } //ha mar sikerult valamit archivalni kilepunk - if (success) + if (oneSuccess) break; } } catch (Exception e) { @@ -155,61 +201,111 @@ public class HSMMigrateStep extends JobStep { return factory.getProvider(type); } - private BasicDBObject getPlanAirMetadata() { - // planairProvider - // List result = null; - // opt.setType(MetadataType.Material); - // result = sut.list(opt); - // if (result.size() != 0) { - // System.out.println("Found as MATERIAL"); - // return; - // } - // opt.setType(MetadataType.Promo); - // result = sut.list(opt); - // if (result.size() != 0) { - // System.out.println("Found as PROMO"); - // return; - // } - // opt.setType(MetadataType.AD); - // result = sut.list(opt); - // if (result.size() != 0) { - // System.out.println("Found as AD"); - // return; - // } + private BasicDBObject getPlanAirMetadata(String mediaHouseId) throws Exception { + PlanAirMetadataListOptions opt = new PlanAirMetadataListOptions(); + opt.setSearch(mediaHouseId); + + List result = null; + opt.setType(MetadataType.Material); + result = planairProvider.list(opt); + if (result.size() != 0) + return result.get(0).asJSON(); + + opt.setType(MetadataType.Promo); + result = planairProvider.list(opt); + if (result.size() != 0) + return result.get(0).asJSON(); + + opt.setType(MetadataType.AD); + result = planairProvider.list(opt); + if (result.size() != 0) + return result.get(0).asJSON(); + return null; } - private boolean processPathItem(String contentFileName, Path sourceFilePath, Path targetFilePath) throws IOException { - BasicDBObject excludeResult = queryExclude(contentFileName); + //true if need copy + public boolean prepareCopy(String hsmFileName, Path source, Path target) throws IOException { + boolean result = true; + File sourceFile = source.toFile(); + File targetFile = target.toFile(); + + BasicDBObject excludeResult = queryExclude(hsmFileName); if (excludeResult != null) { - logger.warn(marker, "Kivételként megjelölt: {}", sourceFilePath, excludeResult.get("name")); + logger.warn(marker, "Kivételként megjelölt: {}", hsmFileName, excludeResult.get("name")); return false; } - File mediaFile = sourceFilePath.toFile(); - if (!mediaFile.exists()) { - logger.warn(marker, "A fájl nem elérhető: {}", sourceFilePath); + if (!sourceFile.exists()) { + logger.warn(marker, "A forrás nem elérhető: {}", source); return false; } - BasicDBObject historyResult = queryFileHistory(contentFileName); - if (historyResult != null) - return false; + // BasicDBObject historyResult = queryFileHistory(contentFileName); + // if (historyResult != null) + // return false; // if (!mediaFile.getName().toLowerCase().endsWith(MOVEXT.toLowerCase()) && !mediaFile.getName().toLowerCase().endsWith(MXFEXT.toLowerCase())) // return; // logger.info("Start copy from {} to {}", sourceFilePath, targetFilePath); - logger.warn(marker, "Migrálás: {}", sourceFilePath); - if (!targetFilePath.toFile().exists()) - Files.copy(sourceFilePath, targetFilePath); - else - logger.warn(marker, "A fájl már létezik: {}", targetFilePath); + boolean targetExists = targetFile.exists(); + + long targetLength = targetFile.length(); + long sourceLength = sourceFile.length(); + + if (targetLength == sourceLength) { + logger.warn(marker, "A fájl már fel van dolgozva: {}, {} -> {}", source, sourceLength, targetLength); + return false; + } + + if (targetLength > sourceLength) { + logger.warn(marker, "A célfájl nagyobb, törlöm: {}", target); + Files.delete(target); + targetLength = 0; + targetExists = false; + } + + if (targetExists) { + logger.warn(marker, "A fájl már létezik, a másolás folytatódik: {}, {} -> {}", target, sourceLength, targetLength); + } else + logger.warn(marker, "Migrálás: {}, {} -> {}", source, sourceLength, targetLength); - String metadata = createMetadata(contentFileName).toString(); - EscortFiles.createMetadata(targetFilePath.getParent().toString(), targetFilePath.getFileName().toString(), metadata, marker); - saveFileHistory(contentFileName); - return true; + return result; + } + + private boolean processHSMFile(String volumeName, String hsmFileName, Path sourceFilePath, Path targetFilePath) throws Exception { + int repeat = 4; + boolean successCopy = true; + if (prepareCopy(hsmFileName, sourceFilePath, targetFilePath)) { + successCopy = false; + while (repeat > 0) { + try { + resumeableCopy(sourceFilePath, targetFilePath); + repeat = 0; + successCopy = true; + } catch (Exception e) { + //logger.warn(marker, "Hiba a másolás során: {} ({})", sourceFilePath, e.getMessage()); + repeat--; + } + } + } + if (successCopy) { + String metadata = null; + try { + metadata = createMetadata(volumeName, hsmFileName).toPrettyString(""); + EscortFiles.createMetadata(targetFilePath.getParent().toString(), targetFilePath.getFileName().toString(), metadata, marker); + //saveFileHistory(contentFileName); + + } catch (Exception e) { + logger.error(marker, "Metadata error", e); + return false; + } + return true; + } + + logger.error(marker, "A fájl másolása nem lehetséges: {}", sourceFilePath); + return false; } public BasicDBObject queryExclude(String fileName) { @@ -244,6 +340,40 @@ public class HSMMigrateStep extends JobStep { return historyResult; } + public void resumeableCopy(Path source, Path target) throws Exception { + File sourceFile = source.toFile(); + File targetFile = target.toFile(); + boolean targetExists = targetFile.exists(); + + long targetLength = targetFile.length(); + long sourceLength = sourceFile.length(); + + try (InputStream in = new BufferedInputStream(new FileInputStream(sourceFile)); + OutputStream out = new BufferedOutputStream(new FileOutputStream(targetFile, targetExists))) { + + byte[] buffer = new byte[256 * 1024 * 4 * 100]; + int lengthRead; + + if (targetExists) + in.skip(targetLength); + + while ((lengthRead = in.read(buffer)) > 0) { + out.write(buffer, 0, lengthRead); + out.flush(); + targetLength = targetFile.length(); + if (targetLength > sourceLength) { + throw new Exception("Hiba! A fájl túl nagy lett."); + } + } + + targetLength = targetFile.length(); + sourceLength = sourceFile.length(); + if (targetLength != sourceLength) { + throw new Exception("Hiba! A fájl mérete nem egyezik."); + } + } + } + private void saveFileHistory(String fileName) { if (db == null) db = NoSQLUtils.getNoSQLDB(); diff --git a/server/user.jobengine.executors/src/user/jobengine/server/steps/MediaToolStep.java b/server/user.jobengine.executors/src/user/jobengine/server/steps/MediaToolStep.java index 69810a4b..8dc0ba4f 100644 --- a/server/user.jobengine.executors/src/user/jobengine/server/steps/MediaToolStep.java +++ b/server/user.jobengine.executors/src/user/jobengine/server/steps/MediaToolStep.java @@ -9,15 +9,13 @@ import org.apache.logging.log4j.Marker; import user.commons.mediatool.MediaInfo; import user.jobengine.db.Media; -import user.jobengine.server.IJobEngine; -import user.jobengine.server.IJobRuntime; public class MediaToolStep extends JobStep { private static final Logger logger = LogManager.getLogger(); private Marker marker; @StepEntry - public Object[] execute(ArchiveItem archiveItem, Media mediaCubeMedia, IJobEngine jobEngine, IJobRuntime jobRuntime) throws Exception { + public Object[] execute(ArchiveItem archiveItem, Media mediaCubeMedia) throws Exception { marker = jobRuntime.getMarker(); if (mediaCubeMedia.getLength() > 0) diff --git a/server/user.jobengine.executors/src/user/jobengine/server/steps/TSMBackupStep.java b/server/user.jobengine.executors/src/user/jobengine/server/steps/TSMBackupStep.java index b00e23e9..562ad8f2 100644 --- a/server/user.jobengine.executors/src/user/jobengine/server/steps/TSMBackupStep.java +++ b/server/user.jobengine.executors/src/user/jobengine/server/steps/TSMBackupStep.java @@ -8,6 +8,7 @@ import java.sql.Timestamp; import java.util.Date; import java.util.List; +import org.apache.commons.lang.RandomStringUtils; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.apache.logging.log4j.Marker; @@ -64,6 +65,10 @@ public class TSMBackupStep extends JobStep { if (existingMediaId == 0) existingMediaId = ItemManagerExtensions.getExistingRundownMedia(manager, sourceFileName.replace(MXFEXT, "")); + // a-z, A-Z, 0-9. For example: WRMcpIk7, s57JwCVA + //veletlenszeru neveket adunk! + String targetFileName = String.format("%s-%s", RandomStringUtils.randomAlphanumeric(8), sourceFileName); + if (existingMediaId == 0) { StoreUri sourceUri = manager.createStoreUri(RemoteStoreProtocol.LOCAL, sourceMediaFile.getParent().toString()); @@ -80,7 +85,8 @@ public class TSMBackupStep extends JobStep { evt.setCancel(!canContinue()); } }); - RemoteFile remoteFile = sourceUri.transferFrom(targetUri, sourceFileName, sourceFileName); + + RemoteFile remoteFile = sourceUri.transferFrom(targetUri, sourceFileName, targetFileName); } if (existingMediaId > 0) @@ -95,9 +101,9 @@ public class TSMBackupStep extends JobStep { sourceMediaFile.getName()); mediaCubeMedia.remove(); } else { - saveMetadata(mediaCubeMedia, sourceMediaFile, existingMediaId); + saveMetadata(mediaCubeMedia, sourceMediaFile, targetFileName, existingMediaId); logger.info(marker, "Az '{}' archiválása sikeres.", sourceFileName); - if (killDateDays > 0) + if (killDateDays != 0) EscortFiles.createUNCKillDate(sourceMediaFile.getParent(), sourceFileName, killDateDays, marker); } @@ -114,10 +120,10 @@ public class TSMBackupStep extends JobStep { return null; } - private void saveMetadata(Media mediaCubeMedia, File sourceFile, long existingMediaId) { + private void saveMetadata(Media mediaCubeMedia, File sourceFile, String targetFileName, long existingMediaId) { if (existingMediaId == 0) { - MediaFile mediaFile = manager.createMediaFile(sourceFile.getName(), fileType, tsmStore, mediaCubeMedia); + MediaFile mediaFile = manager.createMediaFile(targetFileName, fileType, tsmStore, mediaCubeMedia); mediaFile.setHouseId(sourceFile.getName()); mediaFile.add(); } else { diff --git a/server/user.jobengine.executors/src/user/jobengine/server/steps/TranscodeFFAStranStep.java b/server/user.jobengine.executors/src/user/jobengine/server/steps/TranscodeFFAStranStep.java index e6b454c7..c4feeada 100644 --- a/server/user.jobengine.executors/src/user/jobengine/server/steps/TranscodeFFAStranStep.java +++ b/server/user.jobengine.executors/src/user/jobengine/server/steps/TranscodeFFAStranStep.java @@ -6,6 +6,7 @@ import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.Paths; +import org.apache.commons.io.FilenameUtils; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.apache.logging.log4j.Marker; @@ -22,7 +23,8 @@ import user.jobengine.server.IJobEngine; import user.jobengine.server.IJobRuntime; public class TranscodeFFAStranStep extends JobStep { - private static final String MP4EXT = ".mp4"; + private static final int POLL_INTERVALL = 3000; + private static final String MP4EXT = ".MP4"; private static final String MXFEXT = ".MXF"; private static final String LOWRES_FILETYPE = "Low-res"; private static final Logger logger = LogManager.getLogger(); @@ -55,9 +57,11 @@ public class TranscodeFFAStranStep extends JobStep { String details = String.format("%s (%d bytes)", sourceFileName, sourceMediaFile.length()); Path targetPath = null; try { - targetPath = Paths.get(localLowresTargetPath, sourceFileName.replace(MXFEXT, MP4EXT)); + String targetFileName = FilenameUtils.removeExtension(sourceFileName) + MP4EXT; + targetPath = Paths.get(localLowresTargetPath, targetFileName); if (!targetPath.toFile().exists()) { - jobRuntime.setDescription(String.format("%s: %s", jobRuntime.getDescription(), details)); + // jobRuntime.setDescription(String.format("%s: %s", jobRuntime.getDescription(), details)); + jobRuntime.setDescription(String.format("%s transzkódolása", details)); String sourceFile = Paths.get(globalHiresSourcePath, sourceFileName).toString(); IFFAStransAPI api = new FFAStransAPI(transcoderAddress, p -> { if (p <= 100) @@ -65,17 +69,24 @@ public class TranscodeFFAStranStep extends JobStep { }); api.submit(transcoderTemplateName, sourceFile); - api.monitor(3000); + api.monitor(POLL_INTERVALL); } //a sikeres transzkod utan nem mindig van ott egybol a fajl - Thread.sleep(3000); + long started = System.currentTimeMillis(); + while (!targetPath.toFile().exists()) { + long current = System.currentTimeMillis(); + //max 5 perc varakozas + if (current - started > 5 * 60 * 1000) + throw new Exception("Transcode job target file access timed out"); + Thread.sleep(POLL_INTERVALL); + } + postprocess(targetPath, webPath); } catch (Exception e) { logger.catching(e); - Message m = new ParameterizedMessage("Az '{}' állomány átkódolása sikertelen. A rendszer hibaüzenete: {}, {}", details, - e.getClass().getSimpleName(), e.getMessage()); + Message m = new ParameterizedMessage("{} átkódolás hiba: {}", e.getClass().getSimpleName(), e.getMessage()); logger.error(marker, m); throw new Exception(m.getFormattedMessage()); } finally { diff --git a/server/user.jobengine.osgi.commons/src/user/commons/FFAStransAPI.java b/server/user.jobengine.osgi.commons/src/user/commons/FFAStransAPI.java index 4159c030..1badc5cf 100644 --- a/server/user.jobengine.osgi.commons/src/user/commons/FFAStransAPI.java +++ b/server/user.jobengine.osgi.commons/src/user/commons/FFAStransAPI.java @@ -1,12 +1,21 @@ package user.commons; import java.io.File; +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.LinkOption; +import java.nio.file.Path; +import java.nio.file.Paths; import java.util.List; +import java.util.concurrent.Executors; +import java.util.concurrent.ThreadPoolExecutor; +import java.util.concurrent.TimeUnit; import javax.ws.rs.client.Entity; import javax.ws.rs.core.MediaType; import javax.ws.rs.core.Response; +import org.apache.commons.io.FilenameUtils; import org.apache.commons.lang.StringUtils; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; @@ -20,22 +29,55 @@ import user.commons.nosql.NoSQLUtils; public class FFAStransAPI implements IFFAStransAPI { + static { + try { + Path config = Paths.get(System.getProperty("user.dir"), "../-configuration/log4j2-test.xml"); + String configLocation = config.toRealPath(LinkOption.NOFOLLOW_LINKS).toString(); + System.setProperty("log4j.configurationFile", configLocation); + } catch (IOException e) { + // TODO Auto-generated catch block + e.printStackTrace(); + } + } + private static final Logger logger = LogManager.getLogger(); public static void main(String[] args) throws Exception { + final ThreadPoolExecutor executor = (ThreadPoolExecutor) Executors.newFixedThreadPool(10); + + String inDir = "\\\\10.11.1.90\\data\\"; + String outDir = "\\\\10.11.1.100\\Promise\\TRANSCODER\\FFASTRANSCODER\\Out\\"; + + String inputs[] = { "xxx.mxf", "17-03362-0000-1.mov", "17-03194-0000-1.mov", "17-03181-0000-1.mov", "17-03107-0000-1.mov", "09-00082-0002-3.mov" }; + for (int i = 0; i < 6; i++) { + final String inputi = inDir + inputs[i]; + final String outputi = outDir + FilenameUtils.removeExtension(inputs[i]) + ".MP4"; + //Task task = new Task(inDir + input, outDir + output); + + Runnable task = new Runnable() { + private String input = inputi; + private String output = outputi; + + @Override + public void run() { + try { + IFFAStransAPI api = new FFAStransAPI("http://10.11.1.111:65445/api/json/v1/", p -> { + //System.out.println(output + " progress: " + p); + }); + + api.submit("MP4", input); + api.monitor(1000); + Thread.sleep(2000); + if (!Files.exists(Paths.get(output))) + throw new Exception("Missing " + output); + } catch (Exception e) { + e.printStackTrace(); + } + } - for (int i = 0; i < 1; i++) { - IFFAStransAPI api = new FFAStransAPI("http://10.11.1.111:65445/api/json/v1/", p -> { - System.out.println("Progress: " + p); - }); - - String inputFile = "\\\\10.11.1.100\\Promise\\ARCHIVE\\20.MXF"; - api.submit("MP4", inputFile); - try { - api.monitor(1000); - } catch (Exception e) { - System.out.println(e.getMessage()); - } + }; + + executor.execute(task); // try { // // String outputFile = "\\\\10.10.1.74\\MAM-Proxy_output\\20180201-0700_hirado_TEST-_CS.mp4"; @@ -47,8 +89,9 @@ public class FFAStransAPI implements IFFAStransAPI { // } catch (Exception e) { // System.out.println(e.getMessage()); // } - } + executor.shutdown(); + executor.awaitTermination(1, TimeUnit.HOURS); } private ResteasyWebTarget webTarget; @@ -56,6 +99,7 @@ public class FFAStransAPI implements IFFAStransAPI { private String jobId; private String inputFile; private String workflowName; + private String inputFileName; public FFAStransAPI(String apiAddress, IProgressChangedListener listener) { this.listener = listener; @@ -68,13 +112,14 @@ public class FFAStransAPI implements IFFAStransAPI { if (apiResponse.getStatus() != 202) throw new Exception("Can not submit, response status is: " + apiResponse.getStatus()); String json = apiResponse.readEntity(String.class); - logger.info("Transoder response: {}", json); + //logger.info("Transoder response: {}", json); if (StringUtils.isBlank(json)) throw new Exception("Can not submit, response JSON is empty"); BasicDBObject resultObject = (BasicDBObject) JSONUtil.jsonToDbObject(json); if (resultObject == null) throw new Exception("Can not submit, response object is null"); jobId = resultObject.getString("job_id"); + logger.info("Job submitted: {}", jobId); } @Override @@ -83,12 +128,19 @@ public class FFAStransAPI implements IFFAStransAPI { BasicDBObject result = null; try { Response apiResponse = target.request().get(); - if (apiResponse.getStatus() != 200) + if (apiResponse.getStatus() != 200) { + logger.info("{} | Invalid response {}", inputFileName, apiResponse.getStatus()); return null; + } String json = apiResponse.readEntity(String.class); BasicDBObject resultObject = (BasicDBObject) JSONUtil.jsonToDbObject(json); List jobs = NoSQLUtils.asList(resultObject, "history"); + if (jobs == null || jobs.size() == 0) { + logger.info("{} | No jobs in response", inputFileName); + return null; + } + File f = new File(inputFile); for (BasicDBObject job : jobs) { @@ -104,18 +156,18 @@ public class FFAStransAPI implements IFFAStransAPI { continue; if (file.equals(f.getName())) { - logger.info("Found success history"); + //logger.info("Found success history {}", f.getName()); result = job; break; } if (file.equals(inputFile)) { - logger.info("Found error history"); + //logger.info("Found error history {}", inputFile); result = job; break; } } } catch (Exception e) { - System.out.println(e.getMessage()); + logger.error(e.getClass() + " " + e.getMessage()); } return result; @@ -128,18 +180,25 @@ public class FFAStransAPI implements IFFAStransAPI { BasicDBObject result = null; try { Response apiResponse = target.request().get(); - logger.info("Transoder response code: {}", apiResponse.getStatus()); + //logger.info("Transoder response code: {}", apiResponse.getStatus()); - if (apiResponse.getStatus() != 200) + if (apiResponse.getStatus() != 200) { + logger.info("{} | Invalid response {}", inputFileName, apiResponse.getStatus()); return null; + } String json = apiResponse.readEntity(String.class); - logger.info("Transoder response: {}", json); + //logger.info("Transoder response: {}", json); + //System.out.println(json); BasicDBObject resultObject = (BasicDBObject) JSONUtil.jsonToDbObject(json); List jobs = NoSQLUtils.asList(resultObject, "jobs"); - if (jobs == null) + + if (jobs == null || jobs.size() == 0) { + logger.info("{} | No jobs in response", inputFileName); return null; + } for (BasicDBObject job : jobs) { if (jobID.equals(job.getString("job_id"))) { + //logger.info("Found job {}", jobID); result = job; break; } @@ -175,19 +234,28 @@ public class FFAStransAPI implements IFFAStransAPI { @Override public void monitor(int pollIntervall) throws InterruptedException, Exception { int progress = 0; + + long started = System.currentTimeMillis(); + + while (true) { + long current = System.currentTimeMillis(); + if (current - started > 30 * 60 * 1000) + throw new Exception("Transcode job submit timed out"); + + BasicDBObject status = getStatus(jobId); + if (status != null) + break; + Thread.sleep(pollIntervall); + } + while (true) { Thread.sleep(pollIntervall); - //Status: {"job_id":"20180226-162821-217-A7E91DC625BD","job_start":"2018/02/26 16:28:21","file":"\\\\PROXY-TRANSCODER-01\\MAM-Proxy_input\\20180201-0700_hirado_TEST-_CS.MXF","wf_name":"MAM_proxy","splits":[{"steps":"4 / 5","processor":"Folder","status":"Waiting for next processor resources...","node":"PROXY-TRANSCODE","progress":"78.5"}]} BasicDBObject status = getStatus(jobId); if (status != null) { - logger.info("Status: " + status.toPrettyString(null)); List splits = NoSQLUtils.asList(status, "splits"); int current = 0; if (splits != null && splits.size() > 0) { - // String processor = NoSQLUtils.asString(splits.get(0), "processor"); - // if (StringUtils.isNotBlank(processor) && "Generate text file".equals(processor)) - // continue; for (BasicDBObject split : splits) { String prg = NoSQLUtils.asString(split, "progress"); current += (int) Float.parseFloat(prg); @@ -198,30 +266,23 @@ public class FFAStransAPI implements IFFAStransAPI { if (current != progress) { progress = current; listener.onProgressChanged(progress); + logger.info("{} | {}%", inputFileName, progress); } } else { listener.onProgressChanged(100); BasicDBObject history = getHistory(); - - // if (history == null && lastJobToSubmit != null) { - // //plusz 1 proba - // doSubmit(); - // lastJobToSubmit = null; - // monitor(pollIntervall); - // return; - // } - - //System.out.println("History: " + history.toPrettyString(null)); - if (history == null || NoSQLUtils.asLong(history, "state") != 1) { + long state = NoSQLUtils.asLong(history, "state"); + if (history == null || state != 1) { String error = NoSQLUtils.asString(history, "outcome"); - throw new Exception("Transcode error. " + error); + if (error == null) + error = "Unknown error"; + throw new Exception("Transcode error. State: " + state + ", error: " + error); } else { - //System.out.println("Transcode completed"); + logger.info("{} | completed", inputFileName); break; } - //History: {"wf_name":"MAM_proxy","job_start":"2018/02/26 16:28:21","job_end":"2018/02/26 16:28:36","file":"20180201-0700_hirado_TEST-_CS.MXF","outcome":"Success","state":1} } } @@ -231,6 +292,7 @@ public class FFAStransAPI implements IFFAStransAPI { public void submit(String workflowName, String inputFile) throws Exception { this.workflowName = workflowName; this.inputFile = inputFile; + this.inputFileName = Paths.get(inputFile).getFileName().toString(); List workflows = getWorkflows(); if (workflows == null) throw new Exception("No workflows"); diff --git a/server/user.jobengine.osgi.server/pages/joblist.zul b/server/user.jobengine.osgi.server/pages/joblist.zul index d2f43bef..c23084ee 100644 --- a/server/user.jobengine.osgi.server/pages/joblist.zul +++ b/server/user.jobengine.osgi.server/pages/joblist.zul @@ -23,6 +23,7 @@