--- /dev/null
+<?xml version="1.0" encoding="UTF-8"?>\r
+<Configuration status="ERROR" monitorInterval="10" packages="user.commons.log4j2.appender">\r
+ <Appenders>\r
+ <Console name="Console" target="SYSTEM_OUT">\r
+ <Filters>\r
+ <MarkerFilter marker="MEDIACUBE" onMatch="DENY" onMismatch="NEUTRAL" />\r
+ </Filters>\r
+ <PatternLayout pattern="%d{HH:mm:ss.SSS} %-5level %logger{1}.%M - %msg (%F:%L) %n" />\r
+ </Console>\r
+ </Appenders>\r
+ <Loggers>\r
+ <Root level="INFO">\r
+ <AppenderRef ref="Console" />\r
+ </Root>\r
+ </Loggers>\r
+</Configuration>
\ No newline at end of file
],\r
"authentication": {\r
"authEnabled": true,\r
- "adHost": "10.10.254.11",\r
+ "adHost": "intra.mediavivantis.hu",\r
"adNonSecurePort": 3268,\r
- "adBaseDn": "DC=intra,DC=echotv,DC=hu",\r
+ "adBaseDn": "DC=intra,DC=mediavivantis,DC=hu",\r
"adAdminMap": [\r
"G_ECH_U_INFORMATIKUSOK",\r
"G_ECH_U_MUSZAKVEZETOK",\r
import static org.junit.Assert.assertNotNull;\r
import static org.junit.Assert.assertNull;\r
\r
+import java.nio.file.Paths;\r
+\r
import org.junit.BeforeClass;\r
import org.junit.Test;\r
\r
import user.mediacube.metadata.interfaces.IMetadataProviderFactory;\r
import user.mediacube.metadata.interfaces.MetadataProviderType;\r
\r
-public class HSMMigrateListBuilderStepTest {\r
+public class HSMMigrateStepTest {\r
\r
@BeforeClass\r
static public void setUpConnection() {\r
BasicDBObject actual = sut.queryFileHistory("/OMARCHIVE/6.3.4.200-TIV-TSMALL-Linuxx86_64.bin");\r
assertNotNull(actual);\r
}\r
+\r
+ @Test\r
+ public void testResumableCopy() throws Exception {\r
+ HSMMigrateStep sut = new HSMMigrateStep();\r
+ Paths.get("c:/_video/03c.mp4").toFile().delete();\r
+ sut.copyChunk(Paths.get("c:/_video/1.txt"), Paths.get("c:/_video/2.txt"), 5);\r
+ sut.resumeableCopy(Paths.get("c:/_video/1.txt"), Paths.get("c:/_video/2.txt"));\r
+ }\r
}\r
package hu.user.mediacube.executors.tests;\r
\r
+import java.io.File;\r
+import java.nio.file.Files;\r
import java.util.Arrays;\r
import java.util.List;\r
\r
+import org.apache.commons.io.FilenameUtils;\r
import org.junit.Test;\r
\r
import user.jobengine.db.Media;\r
class PojoRoot {\r
}\r
\r
+ @Test\r
+ public void test0() throws Exception {\r
+ File f = new File("/opt/test.txt");\r
+ Files.write(f.toPath(), "AAA".getBytes());\r
+ System.out.println(f.length());\r
+ Files.write(f.toPath(), "AAAAAA".getBytes());\r
+ System.out.println(f.length());\r
+ Files.delete(f.toPath());\r
+ }\r
+\r
@Test\r
public void test1() throws Exception {\r
Object[] sut = { 1, 5, "valami", new Pojo() };\r
\r
}\r
\r
+ @Test\r
+ public void test5() throws Exception {\r
+ System.out.println(FilenameUtils.removeExtension("valami.txt") + ".mp4");\r
+\r
+ }\r
+\r
void xxx(int x, boolean y) {\r
System.out.println("call");\r
}\r
\r
</commands>\r
</calljobstep>\r
+ <calljobstep type="user.jobengine.server.steps.MediaToolStep" weight="1">\r
+ <inputs>\r
+ <input>\r
+ <variable name="archiveItem" />\r
+ </input>\r
+ <input>\r
+ <variable name="mediaCubeMedia" />\r
+ </input>\r
+ </inputs>\r
+ </calljobstep> \r
</commands>\r
</jobtemplate>
\ No newline at end of file
private static final Logger logger = LogManager.getLogger();\r
// private static final String UTF8 = "utf-8";\r
private static final String STATUSFOLDER = ".STATUS";\r
- private static final String MXFEXT = ".mxf";\r
- private static final String WAVEXT = ".wav";\r
private static final String JSONEXT = ".json";\r
private static final String CATCHEDEXT = ".catched";\r
\r
// if (mediaFile.length() > 0)\r
// return false;\r
\r
- if (mediaFile.isDirectory() || !mediaFile.getName().toLowerCase().endsWith(MXFEXT.toLowerCase())\r
- || mediaFile.getName().toLowerCase().endsWith(WAVEXT.toLowerCase())) {\r
+ if (mediaFile.isDirectory()) {\r
return false;\r
}\r
\r
package user.jobengine.server.steps;\r
\r
+import java.io.BufferedInputStream;\r
+import java.io.BufferedOutputStream;\r
import java.io.File;\r
+import java.io.FileInputStream;\r
+import java.io.FileOutputStream;\r
import java.io.IOException;\r
+import java.io.InputStream;\r
+import java.io.OutputStream;\r
import java.nio.file.Files;\r
import java.nio.file.Path;\r
import java.nio.file.Paths;\r
+import java.nio.file.attribute.BasicFileAttributes;\r
+import java.text.SimpleDateFormat;\r
import java.util.Arrays;\r
import java.util.LinkedHashMap;\r
import java.util.List;\r
import user.mediacube.metadata.interfaces.IMetadataProvider;\r
import user.mediacube.metadata.interfaces.IMetadataProviderFactory;\r
import user.mediacube.metadata.interfaces.MetadataProviderType;\r
+import user.mediacube.metadata.interfaces.MetadataType;\r
+import user.mediacube.metadata.interfaces.PlanAirMetadataListOptions;\r
\r
public class HSMMigrateStep extends JobStep {\r
private static final Logger logger = LogManager.getLogger();\r
private DBCollection volumeHistory;\r
private DB db;\r
private IMetadataProvider planairProvider;\r
+ private SimpleDateFormat df = new SimpleDateFormat("yyyy.MM.dd");\r
\r
private void cleanupHistory() {\r
if (db == null)\r
volumeHistory.drop();\r
}\r
\r
- private BasicDBObject createMetadata(String fileName) {\r
+ public void copyChunk(Path source, Path target, long chunk) throws IOException {\r
\r
- Path filePath = Paths.get(fileName);\r
- String mediaHouseId = FilenameUtils.removeExtension(filePath.getFileName().toString());\r
+ File sourceFile = source.toFile();\r
+ File targetFile = target.toFile();\r
+\r
+ try (InputStream in = new BufferedInputStream(new FileInputStream(sourceFile));\r
+ OutputStream out = new BufferedOutputStream(new FileOutputStream(targetFile))) {\r
+\r
+ byte[] buffer = new byte[1];\r
+ long copied = 0;\r
+ int lengthRead;\r
+ while ((lengthRead = in.read(buffer)) > 0) {\r
+ out.write(buffer, 0, lengthRead);\r
+ out.flush();\r
\r
- //get meta\r
+ copied += lengthRead;\r
+ if (copied > chunk) {\r
+ out.close();\r
+ in.close();\r
+ return;\r
+ }\r
+\r
+ }\r
+ }\r
\r
- String itemHouseId = "itemHouseId";\r
- String itemTitle = "itemTitle";\r
- String itemDesc = "itemDesc";\r
- String mediaTitle = "mediaTitle";\r
- String mediaDesc = "mediaDesc";\r
+ }\r
+\r
+ private BasicDBObject createMetadata(String volumeName, String fileName) throws Exception {\r
+\r
+ Path filePath = Paths.get(fileName);\r
+ String mediaHouseId = FilenameUtils.removeExtension(filePath.getFileName().toString());\r
+ BasicDBObject result = null;\r
+ try {\r
+ result = getPlanAirMetadata(mediaHouseId);\r
+ } catch (Exception e) {\r
+ logger.error("PlanAir metadata error", e);\r
+ throw e;\r
+ }\r
\r
- BasicDBObject result = new BasicDBObject();\r
- result.put("itemHouseId", itemHouseId);\r
- result.put("itemTitle", itemTitle);\r
- result.put("itemDescription", itemDesc);\r
+ if (result == null) {\r
+ result = new BasicDBObject();\r
+ BasicFileAttributes attr = Files.readAttributes(filePath, BasicFileAttributes.class);\r
+ result.put("itemHouseId", df.format(attr.lastModifiedTime().toMillis()));\r
+ result.put("itemTitle", filePath.getParent().toString());\r
+ result.put("mediaHouseId", mediaHouseId);\r
+ result.put("mediaTitle", fileName);\r
+ result.put("mediaDescription", volumeName);\r
+ result.put("mediaType", "Generic");\r
+ }\r
result.put("userName", "mediacube");\r
- result.put("mediaHouseId", fileName);\r
- result.put("mediaTitle", mediaTitle);\r
- result.put("mediaDescription", mediaDesc);\r
- result.put("mediaType", "Generic");\r
return result;\r
}\r
\r
if (jobRuntime != null)\r
marker = jobRuntime.getMarker();\r
//remove from prod\r
- cleanupHistory();\r
+ //cleanupHistory();\r
hsmProvider = getMetadataProvider(MetadataProviderType.HSM);\r
if (hsmProvider == null)\r
throw new NullPointerException("No HSM metadata provider available");\r
logger.info(marker, "A kazetta feldolgozása elindul: {}", volumeName);\r
List<IMetadata> contents = getContents(volumeName);\r
int p = 0;\r
- boolean success = false;\r
+ boolean oneSuccess = false;\r
+ boolean hasError = false;\r
for (IMetadata content : contents) {\r
BasicDBObject c = content.asJSON();\r
- String contentFileName = c.getString("fileName");\r
+ String hsmFileName = c.getString("fileName");\r
long contentFileSize = NoSQLUtils.asLong(c, "fileSize");\r
- if (!tapeContents.containsKey(contentFileName)) {\r
- tapeContents.put(contentFileName, c);\r
- Path sourceFilePath = Paths.get(sourceLocation, contentFileName);\r
+ if (!tapeContents.containsKey(hsmFileName)) {\r
+ tapeContents.put(hsmFileName, c);\r
+ Path sourceFilePath = Paths.get(sourceLocation, hsmFileName);\r
if (contentFileSize < Files.getFileStore(targetPath).getUsableSpace()) {\r
Path targetFilePath = Paths.get(targetLocation, sourceFilePath.getFileName().toString());\r
- if (processPathItem(contentFileName, sourceFilePath, targetFilePath)) {\r
- success = true;\r
- //throw new Exception("Teszt");\r
+\r
+ try {\r
+ if (processHSMFile(volumeName, hsmFileName, sourceFilePath, targetFilePath)) {\r
+ oneSuccess = true;\r
+ }\r
+ } catch (Exception e) {\r
+ hasError = true;\r
}\r
}\r
}\r
setProgress(p++ * 100 / contents.size());\r
}\r
\r
- saveVolumeHistory(volumeName);\r
- String subject = "A kazetta eltávolítható a HSM rendszerből: " + volumeName;\r
- logger.info(new MediaCubeMarker() {\r
- {\r
- setSubject(subject);\r
- }\r
- }, subject);\r
+ if (oneSuccess && !hasError) {\r
+ saveVolumeHistory(volumeName);\r
+ String subject = "A kazetta eltávolítható a HSM rendszerből: " + volumeName;\r
+ logger.info(new MediaCubeMarker() {\r
+ {\r
+ setSubject(subject);\r
+ }\r
+ }, subject);\r
+ }\r
\r
//ha mar sikerult valamit archivalni kilepunk\r
- if (success)\r
+ if (oneSuccess)\r
break;\r
}\r
} catch (Exception e) {\r
return factory.getProvider(type);\r
}\r
\r
- private BasicDBObject getPlanAirMetadata() {\r
- // planairProvider\r
- // List<IMetadata> result = null;\r
- // opt.setType(MetadataType.Material);\r
- // result = sut.list(opt);\r
- // if (result.size() != 0) {\r
- // System.out.println("Found as MATERIAL");\r
- // return;\r
- // }\r
- // opt.setType(MetadataType.Promo);\r
- // result = sut.list(opt);\r
- // if (result.size() != 0) {\r
- // System.out.println("Found as PROMO");\r
- // return;\r
- // }\r
- // opt.setType(MetadataType.AD);\r
- // result = sut.list(opt);\r
- // if (result.size() != 0) {\r
- // System.out.println("Found as AD");\r
- // return;\r
- // }\r
+ private BasicDBObject getPlanAirMetadata(String mediaHouseId) throws Exception {\r
+ PlanAirMetadataListOptions opt = new PlanAirMetadataListOptions();\r
+ opt.setSearch(mediaHouseId);\r
+\r
+ List<IMetadata> result = null;\r
+ opt.setType(MetadataType.Material);\r
+ result = planairProvider.list(opt);\r
+ if (result.size() != 0)\r
+ return result.get(0).asJSON();\r
+\r
+ opt.setType(MetadataType.Promo);\r
+ result = planairProvider.list(opt);\r
+ if (result.size() != 0)\r
+ return result.get(0).asJSON();\r
+\r
+ opt.setType(MetadataType.AD);\r
+ result = planairProvider.list(opt);\r
+ if (result.size() != 0)\r
+ return result.get(0).asJSON();\r
+\r
return null;\r
}\r
\r
- private boolean processPathItem(String contentFileName, Path sourceFilePath, Path targetFilePath) throws IOException {\r
- BasicDBObject excludeResult = queryExclude(contentFileName);\r
+ //true if need copy\r
+ public boolean prepareCopy(String hsmFileName, Path source, Path target) throws IOException {\r
+ boolean result = true;\r
+ File sourceFile = source.toFile();\r
+ File targetFile = target.toFile();\r
+\r
+ BasicDBObject excludeResult = queryExclude(hsmFileName);\r
if (excludeResult != null) {\r
- logger.warn(marker, "Kivételként megjelölt: {}", sourceFilePath, excludeResult.get("name"));\r
+ logger.warn(marker, "Kivételként megjelölt: {}", hsmFileName, excludeResult.get("name"));\r
return false;\r
}\r
\r
- File mediaFile = sourceFilePath.toFile();\r
- if (!mediaFile.exists()) {\r
- logger.warn(marker, "A fájl nem elérhető: {}", sourceFilePath);\r
+ if (!sourceFile.exists()) {\r
+ logger.warn(marker, "A forrás nem elérhető: {}", source);\r
return false;\r
}\r
\r
- BasicDBObject historyResult = queryFileHistory(contentFileName);\r
- if (historyResult != null)\r
- return false;\r
+ // BasicDBObject historyResult = queryFileHistory(contentFileName);\r
+ // if (historyResult != null)\r
+ // return false;\r
\r
// if (!mediaFile.getName().toLowerCase().endsWith(MOVEXT.toLowerCase()) && !mediaFile.getName().toLowerCase().endsWith(MXFEXT.toLowerCase()))\r
// return;\r
// logger.info("Start copy from {} to {}", sourceFilePath, targetFilePath);\r
- logger.warn(marker, "Migrálás: {}", sourceFilePath);\r
\r
- if (!targetFilePath.toFile().exists())\r
- Files.copy(sourceFilePath, targetFilePath);\r
- else\r
- logger.warn(marker, "A fájl már létezik: {}", targetFilePath);\r
+ boolean targetExists = targetFile.exists();\r
+\r
+ long targetLength = targetFile.length();\r
+ long sourceLength = sourceFile.length();\r
+\r
+ if (targetLength == sourceLength) {\r
+ logger.warn(marker, "A fájl már fel van dolgozva: {}, {} -> {}", source, sourceLength, targetLength);\r
+ return false;\r
+ }\r
+\r
+ if (targetLength > sourceLength) {\r
+ logger.warn(marker, "A célfájl nagyobb, törlöm: {}", target);\r
+ Files.delete(target);\r
+ targetLength = 0;\r
+ targetExists = false;\r
+ }\r
+\r
+ if (targetExists) {\r
+ logger.warn(marker, "A fájl már létezik, a másolás folytatódik: {}, {} -> {}", target, sourceLength, targetLength);\r
+ } else\r
+ logger.warn(marker, "Migrálás: {}, {} -> {}", source, sourceLength, targetLength);\r
\r
- String metadata = createMetadata(contentFileName).toString();\r
- EscortFiles.createMetadata(targetFilePath.getParent().toString(), targetFilePath.getFileName().toString(), metadata, marker);\r
- saveFileHistory(contentFileName);\r
- return true;\r
+ return result;\r
+ }\r
+\r
+ private boolean processHSMFile(String volumeName, String hsmFileName, Path sourceFilePath, Path targetFilePath) throws Exception {\r
+ int repeat = 4;\r
+ boolean successCopy = true;\r
+ if (prepareCopy(hsmFileName, sourceFilePath, targetFilePath)) {\r
+ successCopy = false;\r
+ while (repeat > 0) {\r
+ try {\r
+ resumeableCopy(sourceFilePath, targetFilePath);\r
+ repeat = 0;\r
+ successCopy = true;\r
+ } catch (Exception e) {\r
+ //logger.warn(marker, "Hiba a másolás során: {} ({})", sourceFilePath, e.getMessage());\r
+ repeat--;\r
+ }\r
+ }\r
+ }\r
+ if (successCopy) {\r
+ String metadata = null;\r
+ try {\r
+ metadata = createMetadata(volumeName, hsmFileName).toPrettyString("");\r
+ EscortFiles.createMetadata(targetFilePath.getParent().toString(), targetFilePath.getFileName().toString(), metadata, marker);\r
+ //saveFileHistory(contentFileName);\r
+\r
+ } catch (Exception e) {\r
+ logger.error(marker, "Metadata error", e);\r
+ return false;\r
+ }\r
+ return true;\r
+ }\r
+\r
+ logger.error(marker, "A fájl másolása nem lehetséges: {}", sourceFilePath);\r
+ return false;\r
}\r
\r
public BasicDBObject queryExclude(String fileName) {\r
return historyResult;\r
}\r
\r
+ public void resumeableCopy(Path source, Path target) throws Exception {\r
+ File sourceFile = source.toFile();\r
+ File targetFile = target.toFile();\r
+ boolean targetExists = targetFile.exists();\r
+\r
+ long targetLength = targetFile.length();\r
+ long sourceLength = sourceFile.length();\r
+\r
+ try (InputStream in = new BufferedInputStream(new FileInputStream(sourceFile));\r
+ OutputStream out = new BufferedOutputStream(new FileOutputStream(targetFile, targetExists))) {\r
+\r
+ byte[] buffer = new byte[256 * 1024 * 4 * 100];\r
+ int lengthRead;\r
+\r
+ if (targetExists)\r
+ in.skip(targetLength);\r
+\r
+ while ((lengthRead = in.read(buffer)) > 0) {\r
+ out.write(buffer, 0, lengthRead);\r
+ out.flush();\r
+ targetLength = targetFile.length();\r
+ if (targetLength > sourceLength) {\r
+ throw new Exception("Hiba! A fájl túl nagy lett.");\r
+ }\r
+ }\r
+\r
+ targetLength = targetFile.length();\r
+ sourceLength = sourceFile.length();\r
+ if (targetLength != sourceLength) {\r
+ throw new Exception("Hiba! A fájl mérete nem egyezik.");\r
+ }\r
+ }\r
+ }\r
+\r
private void saveFileHistory(String fileName) {\r
if (db == null)\r
db = NoSQLUtils.getNoSQLDB();\r
\r
import user.commons.mediatool.MediaInfo;\r
import user.jobengine.db.Media;\r
-import user.jobengine.server.IJobEngine;\r
-import user.jobengine.server.IJobRuntime;\r
\r
public class MediaToolStep extends JobStep {\r
private static final Logger logger = LogManager.getLogger();\r
private Marker marker;\r
\r
@StepEntry\r
- public Object[] execute(ArchiveItem archiveItem, Media mediaCubeMedia, IJobEngine jobEngine, IJobRuntime jobRuntime) throws Exception {\r
+ public Object[] execute(ArchiveItem archiveItem, Media mediaCubeMedia) throws Exception {\r
marker = jobRuntime.getMarker();\r
\r
if (mediaCubeMedia.getLength() > 0)\r
import java.util.Date;\r
import java.util.List;\r
\r
+import org.apache.commons.lang.RandomStringUtils;\r
import org.apache.logging.log4j.LogManager;\r
import org.apache.logging.log4j.Logger;\r
import org.apache.logging.log4j.Marker;\r
if (existingMediaId == 0)\r
existingMediaId = ItemManagerExtensions.getExistingRundownMedia(manager, sourceFileName.replace(MXFEXT, ""));\r
\r
+ // a-z, A-Z, 0-9. For example: WRMcpIk7, s57JwCVA\r
+ //veletlenszeru neveket adunk!\r
+ String targetFileName = String.format("%s-%s", RandomStringUtils.randomAlphanumeric(8), sourceFileName);\r
+\r
if (existingMediaId == 0) {\r
StoreUri sourceUri = manager.createStoreUri(RemoteStoreProtocol.LOCAL, sourceMediaFile.getParent().toString());\r
\r
evt.setCancel(!canContinue());\r
}\r
});\r
- RemoteFile remoteFile = sourceUri.transferFrom(targetUri, sourceFileName, sourceFileName);\r
+\r
+ RemoteFile remoteFile = sourceUri.transferFrom(targetUri, sourceFileName, targetFileName);\r
}\r
\r
if (existingMediaId > 0)\r
sourceMediaFile.getName());\r
mediaCubeMedia.remove();\r
} else {\r
- saveMetadata(mediaCubeMedia, sourceMediaFile, existingMediaId);\r
+ saveMetadata(mediaCubeMedia, sourceMediaFile, targetFileName, existingMediaId);\r
logger.info(marker, "Az '{}' archiválása sikeres.", sourceFileName);\r
- if (killDateDays > 0)\r
+ if (killDateDays != 0)\r
EscortFiles.createUNCKillDate(sourceMediaFile.getParent(), sourceFileName, killDateDays, marker);\r
}\r
\r
return null;\r
}\r
\r
- private void saveMetadata(Media mediaCubeMedia, File sourceFile, long existingMediaId) {\r
+ private void saveMetadata(Media mediaCubeMedia, File sourceFile, String targetFileName, long existingMediaId) {\r
\r
if (existingMediaId == 0) {\r
- MediaFile mediaFile = manager.createMediaFile(sourceFile.getName(), fileType, tsmStore, mediaCubeMedia);\r
+ MediaFile mediaFile = manager.createMediaFile(targetFileName, fileType, tsmStore, mediaCubeMedia);\r
mediaFile.setHouseId(sourceFile.getName());\r
mediaFile.add();\r
} else {\r
import java.nio.file.Path;\r
import java.nio.file.Paths;\r
\r
+import org.apache.commons.io.FilenameUtils;\r
import org.apache.logging.log4j.LogManager;\r
import org.apache.logging.log4j.Logger;\r
import org.apache.logging.log4j.Marker;\r
import user.jobengine.server.IJobRuntime;\r
\r
public class TranscodeFFAStranStep extends JobStep {\r
- private static final String MP4EXT = ".mp4";\r
+ private static final int POLL_INTERVALL = 3000;\r
+ private static final String MP4EXT = ".MP4";\r
private static final String MXFEXT = ".MXF";\r
private static final String LOWRES_FILETYPE = "Low-res";\r
private static final Logger logger = LogManager.getLogger();\r
String details = String.format("%s (%d bytes)", sourceFileName, sourceMediaFile.length());\r
Path targetPath = null;\r
try {\r
- targetPath = Paths.get(localLowresTargetPath, sourceFileName.replace(MXFEXT, MP4EXT));\r
+ String targetFileName = FilenameUtils.removeExtension(sourceFileName) + MP4EXT;\r
+ targetPath = Paths.get(localLowresTargetPath, targetFileName);\r
if (!targetPath.toFile().exists()) {\r
- jobRuntime.setDescription(String.format("%s: %s", jobRuntime.getDescription(), details));\r
+ // jobRuntime.setDescription(String.format("%s: %s", jobRuntime.getDescription(), details));\r
+ jobRuntime.setDescription(String.format("%s transzkódolása", details));\r
String sourceFile = Paths.get(globalHiresSourcePath, sourceFileName).toString();\r
IFFAStransAPI api = new FFAStransAPI(transcoderAddress, p -> {\r
if (p <= 100)\r
});\r
\r
api.submit(transcoderTemplateName, sourceFile);\r
- api.monitor(3000);\r
+ api.monitor(POLL_INTERVALL);\r
}\r
\r
//a sikeres transzkod utan nem mindig van ott egybol a fajl\r
- Thread.sleep(3000);\r
+ long started = System.currentTimeMillis();\r
+ while (!targetPath.toFile().exists()) {\r
+ long current = System.currentTimeMillis();\r
+ //max 5 perc varakozas\r
+ if (current - started > 5 * 60 * 1000)\r
+ throw new Exception("Transcode job target file access timed out");\r
+ Thread.sleep(POLL_INTERVALL);\r
+ }\r
+\r
postprocess(targetPath, webPath);\r
\r
} catch (Exception e) {\r
logger.catching(e);\r
- Message m = new ParameterizedMessage("Az '{}' állomány átkódolása sikertelen. A rendszer hibaüzenete: {}, {}", details,\r
- e.getClass().getSimpleName(), e.getMessage());\r
+ Message m = new ParameterizedMessage("{} átkódolás hiba: {}", e.getClass().getSimpleName(), e.getMessage());\r
logger.error(marker, m);\r
throw new Exception(m.getFormattedMessage());\r
} finally {\r
package user.commons;\r
\r
import java.io.File;\r
+import java.io.IOException;\r
+import java.nio.file.Files;\r
+import java.nio.file.LinkOption;\r
+import java.nio.file.Path;\r
+import java.nio.file.Paths;\r
import java.util.List;\r
+import java.util.concurrent.Executors;\r
+import java.util.concurrent.ThreadPoolExecutor;\r
+import java.util.concurrent.TimeUnit;\r
\r
import javax.ws.rs.client.Entity;\r
import javax.ws.rs.core.MediaType;\r
import javax.ws.rs.core.Response;\r
\r
+import org.apache.commons.io.FilenameUtils;\r
import org.apache.commons.lang.StringUtils;\r
import org.apache.logging.log4j.LogManager;\r
import org.apache.logging.log4j.Logger;\r
\r
public class FFAStransAPI implements IFFAStransAPI {\r
\r
+ static {\r
+ try {\r
+ Path config = Paths.get(System.getProperty("user.dir"), "../-configuration/log4j2-test.xml");\r
+ String configLocation = config.toRealPath(LinkOption.NOFOLLOW_LINKS).toString();\r
+ System.setProperty("log4j.configurationFile", configLocation);\r
+ } catch (IOException e) {\r
+ // TODO Auto-generated catch block\r
+ e.printStackTrace();\r
+ }\r
+ }\r
+\r
private static final Logger logger = LogManager.getLogger();\r
\r
public static void main(String[] args) throws Exception {\r
+ final ThreadPoolExecutor executor = (ThreadPoolExecutor) Executors.newFixedThreadPool(10);\r
+\r
+ String inDir = "\\\\10.11.1.90\\data\\";\r
+ String outDir = "\\\\10.11.1.100\\Promise\\TRANSCODER\\FFASTRANSCODER\\Out\\";\r
+\r
+ String inputs[] = { "xxx.mxf", "17-03362-0000-1.mov", "17-03194-0000-1.mov", "17-03181-0000-1.mov", "17-03107-0000-1.mov", "09-00082-0002-3.mov" };\r
+ for (int i = 0; i < 6; i++) {\r
+ final String inputi = inDir + inputs[i];\r
+ final String outputi = outDir + FilenameUtils.removeExtension(inputs[i]) + ".MP4";\r
+ //Task task = new Task(inDir + input, outDir + output);\r
+\r
+ Runnable task = new Runnable() {\r
+ private String input = inputi;\r
+ private String output = outputi;\r
+\r
+ @Override\r
+ public void run() {\r
+ try {\r
+ IFFAStransAPI api = new FFAStransAPI("http://10.11.1.111:65445/api/json/v1/", p -> {\r
+ //System.out.println(output + " progress: " + p);\r
+ });\r
+\r
+ api.submit("MP4", input);\r
+ api.monitor(1000);\r
+ Thread.sleep(2000);\r
+ if (!Files.exists(Paths.get(output)))\r
+ throw new Exception("Missing " + output);\r
+ } catch (Exception e) {\r
+ e.printStackTrace();\r
+ }\r
+ }\r
\r
- for (int i = 0; i < 1; i++) {\r
- IFFAStransAPI api = new FFAStransAPI("http://10.11.1.111:65445/api/json/v1/", p -> {\r
- System.out.println("Progress: " + p);\r
- });\r
-\r
- String inputFile = "\\\\10.11.1.100\\Promise\\ARCHIVE\\20.MXF";\r
- api.submit("MP4", inputFile);\r
- try {\r
- api.monitor(1000);\r
- } catch (Exception e) {\r
- System.out.println(e.getMessage());\r
- }\r
+ };\r
+\r
+ executor.execute(task);\r
// try {\r
//\r
// String outputFile = "\\\\10.10.1.74\\MAM-Proxy_output\\20180201-0700_hirado_TEST-_CS.mp4";\r
// } catch (Exception e) {\r
// System.out.println(e.getMessage());\r
// }\r
-\r
}\r
+ executor.shutdown();\r
+ executor.awaitTermination(1, TimeUnit.HOURS);\r
}\r
\r
private ResteasyWebTarget webTarget;\r
private String jobId;\r
private String inputFile;\r
private String workflowName;\r
+ private String inputFileName;\r
\r
public FFAStransAPI(String apiAddress, IProgressChangedListener listener) {\r
this.listener = listener;\r
if (apiResponse.getStatus() != 202)\r
throw new Exception("Can not submit, response status is: " + apiResponse.getStatus());\r
String json = apiResponse.readEntity(String.class);\r
- logger.info("Transoder response: {}", json);\r
+ //logger.info("Transoder response: {}", json);\r
if (StringUtils.isBlank(json))\r
throw new Exception("Can not submit, response JSON is empty");\r
BasicDBObject resultObject = (BasicDBObject) JSONUtil.jsonToDbObject(json);\r
if (resultObject == null)\r
throw new Exception("Can not submit, response object is null");\r
jobId = resultObject.getString("job_id");\r
+ logger.info("Job submitted: {}", jobId);\r
}\r
\r
@Override\r
BasicDBObject result = null;\r
try {\r
Response apiResponse = target.request().get();\r
- if (apiResponse.getStatus() != 200)\r
+ if (apiResponse.getStatus() != 200) {\r
+ logger.info("{} | Invalid response {}", inputFileName, apiResponse.getStatus());\r
return null;\r
+ }\r
String json = apiResponse.readEntity(String.class);\r
BasicDBObject resultObject = (BasicDBObject) JSONUtil.jsonToDbObject(json);\r
List<BasicDBObject> jobs = NoSQLUtils.asList(resultObject, "history");\r
\r
+ if (jobs == null || jobs.size() == 0) {\r
+ logger.info("{} | No jobs in response", inputFileName);\r
+ return null;\r
+ }\r
+\r
File f = new File(inputFile);\r
\r
for (BasicDBObject job : jobs) {\r
continue;\r
\r
if (file.equals(f.getName())) {\r
- logger.info("Found success history");\r
+ //logger.info("Found success history {}", f.getName());\r
result = job;\r
break;\r
}\r
if (file.equals(inputFile)) {\r
- logger.info("Found error history");\r
+ //logger.info("Found error history {}", inputFile);\r
result = job;\r
break;\r
}\r
}\r
} catch (Exception e) {\r
- System.out.println(e.getMessage());\r
+ logger.error(e.getClass() + " " + e.getMessage());\r
}\r
\r
return result;\r
BasicDBObject result = null;\r
try {\r
Response apiResponse = target.request().get();\r
- logger.info("Transoder response code: {}", apiResponse.getStatus());\r
+ //logger.info("Transoder response code: {}", apiResponse.getStatus());\r
\r
- if (apiResponse.getStatus() != 200)\r
+ if (apiResponse.getStatus() != 200) {\r
+ logger.info("{} | Invalid response {}", inputFileName, apiResponse.getStatus());\r
return null;\r
+ }\r
String json = apiResponse.readEntity(String.class);\r
- logger.info("Transoder response: {}", json);\r
+ //logger.info("Transoder response: {}", json);\r
+ //System.out.println(json);\r
BasicDBObject resultObject = (BasicDBObject) JSONUtil.jsonToDbObject(json);\r
List<BasicDBObject> jobs = NoSQLUtils.asList(resultObject, "jobs");\r
- if (jobs == null)\r
+\r
+ if (jobs == null || jobs.size() == 0) {\r
+ logger.info("{} | No jobs in response", inputFileName);\r
return null;\r
+ }\r
for (BasicDBObject job : jobs) {\r
if (jobID.equals(job.getString("job_id"))) {\r
+ //logger.info("Found job {}", jobID);\r
result = job;\r
break;\r
}\r
@Override\r
public void monitor(int pollIntervall) throws InterruptedException, Exception {\r
int progress = 0;\r
+\r
+ long started = System.currentTimeMillis();\r
+\r
+ while (true) {\r
+ long current = System.currentTimeMillis();\r
+ if (current - started > 30 * 60 * 1000)\r
+ throw new Exception("Transcode job submit timed out");\r
+\r
+ BasicDBObject status = getStatus(jobId);\r
+ if (status != null)\r
+ break;\r
+ Thread.sleep(pollIntervall);\r
+ }\r
+\r
while (true) {\r
Thread.sleep(pollIntervall);\r
- //Status: {"job_id":"20180226-162821-217-A7E91DC625BD","job_start":"2018/02/26 16:28:21","file":"\\\\PROXY-TRANSCODER-01\\MAM-Proxy_input\\20180201-0700_hirado_TEST-_CS.MXF","wf_name":"MAM_proxy","splits":[{"steps":"4 / 5","processor":"Folder","status":"Waiting for next processor resources...","node":"PROXY-TRANSCODE","progress":"78.5"}]}\r
BasicDBObject status = getStatus(jobId);\r
if (status != null) {\r
- logger.info("Status: " + status.toPrettyString(null));\r
List<BasicDBObject> splits = NoSQLUtils.asList(status, "splits");\r
\r
int current = 0;\r
if (splits != null && splits.size() > 0) {\r
- // String processor = NoSQLUtils.asString(splits.get(0), "processor");\r
- // if (StringUtils.isNotBlank(processor) && "Generate text file".equals(processor))\r
- // continue;\r
for (BasicDBObject split : splits) {\r
String prg = NoSQLUtils.asString(split, "progress");\r
current += (int) Float.parseFloat(prg);\r
if (current != progress) {\r
progress = current;\r
listener.onProgressChanged(progress);\r
+ logger.info("{} | {}%", inputFileName, progress);\r
}\r
\r
} else {\r
listener.onProgressChanged(100);\r
BasicDBObject history = getHistory();\r
-\r
- // if (history == null && lastJobToSubmit != null) {\r
- // //plusz 1 proba\r
- // doSubmit();\r
- // lastJobToSubmit = null;\r
- // monitor(pollIntervall);\r
- // return;\r
- // }\r
-\r
- //System.out.println("History: " + history.toPrettyString(null));\r
- if (history == null || NoSQLUtils.asLong(history, "state") != 1) {\r
+ long state = NoSQLUtils.asLong(history, "state");\r
+ if (history == null || state != 1) {\r
String error = NoSQLUtils.asString(history, "outcome");\r
- throw new Exception("Transcode error. " + error);\r
+ if (error == null)\r
+ error = "Unknown error";\r
+ throw new Exception("Transcode error. State: " + state + ", error: " + error);\r
} else {\r
- //System.out.println("Transcode completed");\r
+ logger.info("{} | completed", inputFileName);\r
break;\r
}\r
\r
- //History: {"wf_name":"MAM_proxy","job_start":"2018/02/26 16:28:21","job_end":"2018/02/26 16:28:36","file":"20180201-0700_hirado_TEST-_CS.MXF","outcome":"Success","state":1}\r
}\r
\r
}\r
public void submit(String workflowName, String inputFile) throws Exception {\r
this.workflowName = workflowName;\r
this.inputFile = inputFile;\r
+ this.inputFileName = Paths.get(inputFile).getFileName().toString();\r
List<BasicDBObject> workflows = getWorkflows();\r
if (workflows == null)\r
throw new Exception("No workflows");\r
<button label="Futtatás" onClick="@command('executeJob')" />\r
<button sclass="buttonboldfont" label="Leállítás" onClick="@command('shutdown')" autodisable="self" disabled="@load(jlm.shutdownDisabled)" />\r
<button sclass="buttonboldfont" label="Elindítás" onClick="@command('startup')" autodisable="self" disabled="@load(jlm.startupDisabled)" />\r
+ <button sclass="buttonboldfont" label="Takarítás" onClick="@command('cleanupSuspended')" autodisable="self" />\r
<button sclass="buttonboldfont" label="NEXIO szinkronizálás" onClick="@command('restartNexio')" visible="${not sessionScope.nexioDisable}" />\r
</div>\r
\r
controls : true,\r
preload : "auto"\r
});\r
- player.on('ended', function() {\r
- this.dispose();\r
- console.log("ended", "Player disposed");\r
- });\r
\r
function setSource(src) {\r
console.log("setSource", player, src);\r
import user.jobengine.server.scheduler.SchedulerService;
public interface IJobEngine {
- public static final int QUEUE_POLL_INTERVALL = 10;
void addJobChangedEventListener(IJobChangedListener listener);
void removeJobChangedEventListener(IJobChangedListener listener);
+ void removeSuspended();
+
void rePrioritization(IJobRuntime jobRuntime);
void sendMessage(IJobMessage jobMessage);
import user.jobengine.server.scheduler.ScheduledJob;
import user.jobengine.server.scheduler.SchedulerService;
import user.jobengine.server.steps.IJobStep;
+import user.jobengine.zk.util.SessionUtil;
import user.tsm.client.TSMClient;
import user.tsm.client.TSMException;
while (!shutdown) {
try {
- IJobRuntime jobRuntime = runQueue.poll(IJobEngine.QUEUE_POLL_INTERVALL, TimeUnit.MILLISECONDS);
+ IJobRuntime jobRuntime = runQueue.poll(SessionUtil.getMediaCubeConfig().getJobQueuePollInterval(), TimeUnit.MILLISECONDS);
if (jobRuntime != null) {
while (jobRuntime.hasNextInstruction() && jobRuntime.isRunable()) {
ir = jobRuntime.getNextInstruction();
}
}
- private void removeSuspended() {
+ @Override
+ public void removeSuspended() {
List<Long> removeId = new ArrayList<>();
for (Long id : submittedJobs.keySet()) {
if (submittedJobs.get(id).getStatus() == JobStatus.SUSPENDED)
import user.jobengine.server.messages.JobStepCompletedMessage;
import user.jobengine.server.messages.JobStepSkippedMessage;
import user.jobengine.server.steps.IJobStep;
+import user.jobengine.zk.util.SessionUtil;
public class JobStepExecutor implements IJobStepExecutor {
private class Worker extends Thread {
IJobStep step = null;
while (true) {
try {
- jobRuntime = queue.poll(IJobEngine.QUEUE_POLL_INTERVALL, TimeUnit.MILLISECONDS);
+ jobRuntime = queue.poll(SessionUtil.getMediaCubeConfig().getJobQueuePollInterval(), TimeUnit.MILLISECONDS);
if (jobRuntime == null && shutdown) {
logger.trace("Shutting down");
break;
}
public JobStepExecutor(Class<IJobStep> clazz, int maxConcurrent) throws Exception {
- create(clazz.getName(), maxConcurrent);
+ this(clazz.getName(), maxConcurrent);
}
public JobStepExecutor(String className, int maxConcurrent) throws Exception {
}\r
}\r
\r
+ @Command\r
+ public void cleanupSuspended() {\r
+ jobEngine.removeSuspended();\r
+ initializeList();\r
+ }\r
+\r
private int compare(IJobRuntime o1, IJobRuntime o2) {\r
// return o1.getStatus().compareTo(o2.getStatus());\r
int ret = (o2.getPriority() - o1.getPriority());\r
import java.util.List;\r
\r
public class MediaCubeConfig {\r
+ private int jobQueuePollInterval = 10;\r
private MediaCubeAuthentication authentication;\r
private List<TypeFilter> topTypeFilters;\r
private List<TypeFilter> bottomTypeFilters;\r
return bottomTypeFilters;\r
}\r
\r
+ public int getJobQueuePollInterval() {\r
+ return jobQueuePollInterval;\r
+ }\r
+\r
public List<TypeFilter> getTopTypeFilters() {\r
return topTypeFilters;\r
}\r
this.bottomTypeFilters = bottomTypeFilters;\r
}\r
\r
+ public void setJobQueuePollInterval(int jobQueuePollInterval) {\r
+ this.jobQueuePollInterval = jobQueuePollInterval;\r
+ }\r
+\r
public void setTopTypeFilters(List<TypeFilter> topTypeFilters) {\r
this.topTypeFilters = topTypeFilters;\r
}\r
private static final String ARG = "arg";\r
private static final String MEDIA_ID = "showMediaID";\r
private static final Logger logger = LogManager.getLogger();\r
- private static MediaCubeConfig mediaCubeConfig = SessionUtil.getMediaCubeConfig();\r
private List<TypeFilter> topTypeFilters;\r
private List<TypeFilter> bottomTypeFilters;\r
private Map<String, TypeFilter> typeFilters;\r
public SearchModel() {\r
this.itemManager = ItemManager.getInstance();\r
typeFilters = new LinkedHashMap<>();\r
+ MediaCubeConfig mediaCubeConfig = SessionUtil.getMediaCubeConfig();\r
setTopTypeFilters(mediaCubeConfig.getTopTypeFilters());\r
setBottomTypeFilters(mediaCubeConfig.getBottomTypeFilters());\r
\r
import user.jobengine.zk.model.MediaCubeConfig;\r
\r
public class SessionUtil {\r
+ private static Logger logger = LogManager.getLogger();\r
private static final String JOBENGINE_MEDIACUBE_CONFIG = "jobengine.mediacube.config";\r
private static final String JOBENGINE_MAESTRO_CONFIG = "jobengine.maestro.config";\r
private static final String SEARCH_FILTER = "searchFilter";\r
private static final String SEARCH_HISTORY = "searchHistory";\r
private static final String USER_NAME = "userName";\r
private static final String USER_SETTINGS = "userSettings";\r
- private static Logger logger = LogManager.getLogger();\r
- private static final String AUTH_ENABLED = "auth_enabled";\r
private static final String USERPRINCIPAL = "userPrincipal";\r
private static final String AUTH_LOCATION = "mediacube.auth.location";\r
public static final String JOBLISTENER = "joblistener";\r
\r
static {\r
try {\r
- mediaCubeConfig = JSONConfig.read("jobengine.mediacube.config", MediaCubeConfig.class);\r
+ mediaCubeConfig = JSONConfig.read(JOBENGINE_MEDIACUBE_CONFIG, MediaCubeConfig.class);\r
} catch (Exception e) {\r
logger.error("Critical error", e);\r
}\r
}\r
\r
public static MediaCubeConfig getMediaCubeConfig() {\r
- MediaCubeConfig result = null;\r
- try {\r
- result = JSONConfig.read(JOBENGINE_MEDIACUBE_CONFIG, MediaCubeConfig.class);\r
- } catch (Exception e) {\r
- logger.catching(e);\r
- }\r
- return result;\r
+ return mediaCubeConfig;\r
}\r
\r
static public String getQueryParameter(String key) {\r
\r
@Override\r
protected DataSource createDataSource() {\r
- return new PooledDataSource(JDBC_DRIVER, JDBC_URL, JDBC_USER, JDBC_PWD);\r
+ PooledDataSource dataSource = new PooledDataSource(JDBC_DRIVER, JDBC_URL, JDBC_USER, JDBC_PWD);\r
+ dataSource.setPoolPingEnabled(true);\r
+ dataSource.setPoolPingQuery("SELECT 1 FROM SYSIBM.SYSDUMMY1");\r
+ dataSource.setPoolPingConnectionsNotUsedFor(5000);\r
+ return dataSource;\r
}\r
\r
@Override\r
@Result(property = "epTitle", column = "v_EpTitle"), @Result(property = "episode", column = "v_Episode"),\r
@Result(property = "variantNrSegments", column = "v_VariantNrSegments"), @Result(property = "firstBroadcastDate", column = "v_FirstBroadcastDate"),\r
@Result(property = "nextBroadcastDate", column = "v_NextBroadcastDate"), @Result(property = "okForAir", column = "v_OkForAir"),\r
- @Result(property = "segNr", column = "v_SegNr"), @Result(property = "segID", column = "v_SegID") })\r
+ @Result(property = "segNr", column = "v_SegNr"), @Result(property = "segID", column = "v_SegID"),\r
+ @Result(property = "programmeID", column = "v_ProgrammeID") })\r
public List<PlanAirMaterial> getMaterials(@Param("options") Integer options, @Param("search") String search, @Param("broadcastDay") Date broadcastDay);\r
}\r
@Override\r
public BasicDBObject asJSON() {\r
BasicDBObject result = new BasicDBObject();\r
- result.put("mediaID", mediaID);\r
- result.put("title", title);\r
- result.put("spotID", spotID);\r
- result.put("okForAirs", okForAirs);\r
+ result.put("itemHouseId", mediaID);\r
+ result.put("itemTitle", title);\r
+ result.put("mediaHouseId", mediaID);\r
+ result.put("mediaTitle", title);\r
+ result.put("mediaType", "Reklám");\r
return result;\r
}\r
\r
private short episode;\r
private boolean okForAir;\r
private int variantNrSegments;\r
+ private int programmeID;\r
private Date nextBroadcastDate;\r
private Date firstBroadcastDate;\r
private int segNr;\r
@Override\r
public BasicDBObject asJSON() {\r
BasicDBObject result = new BasicDBObject();\r
- result.put("episodeID", episodeID);\r
- result.put("variantID", variantID);\r
- result.put("mediaID", mediaID);\r
- result.put("epTitle", epTitle);\r
- result.put("progTitle", progTitle);\r
- result.put("episode", episode);\r
- result.put("okForAir", okForAir);\r
- result.put("variantNrSegments", variantNrSegments);\r
- result.put("nextBroadcastDate", nextBroadcastDate);\r
- result.put("firstBroadcastDate", firstBroadcastDate);\r
- result.put("segNr", segNr);\r
- result.put("segID", segID);\r
+ result.put("itemHouseId", programmeID);\r
+ result.put("itemTitle", progTitle);\r
+ result.put("mediaHouseId", episodeID);\r
+ result.put("mediaTitle", epTitle == null || epTitle.trim().length() == 0 ? progTitle : epTitle);\r
+ result.put("mediaDescription", episode > 0 ? "Epizód: " + episode : "");\r
+ result.put("mediaType", "Műsor");\r
return result;\r
}\r
\r
return nextBroadcastDate;\r
}\r
\r
+ public int getProgrammeID() {\r
+ return programmeID;\r
+ }\r
+\r
public String getProgTitle() {\r
return progTitle;\r
}\r
this.okForAir = okForAir;\r
}\r
\r
+ public void setProgrammeID(int programmeID) {\r
+ this.programmeID = programmeID;\r
+ }\r
+\r
public void setProgTitle(String progTitle) {\r
this.progTitle = progTitle;\r
}\r
@Override\r
public BasicDBObject asJSON() {\r
BasicDBObject result = new BasicDBObject();\r
- result.put("mediaID", mediaID);\r
- result.put("title", title);\r
- result.put("progTitle", progTitle);\r
- result.put("episode", episode);\r
- result.put("psStart", psStart);\r
- result.put("spotID", spotID);\r
- result.put("okForAirs", okForAirs);\r
+ result.put("itemHouseId", mediaID);\r
+ result.put("itemTitle", title);\r
+ result.put("mediaHouseId", mediaID);\r
+ result.put("mediaTitle", title);\r
+ result.put("mediaType", "Promo");\r
return result;\r
}\r
\r
https://git.useribm.hu / mediacube.git / commitdiff