From: vasary.daniel Date: Fri, 5 Feb 2021 12:22:45 +0000 (+0000) Subject: git-tfs-id: [http://tfs.userrendszerhaz.hu:8080/tfs/DefaultCollection]$/MediaCube... X-Git-Url: http://git.useribm.hu/?a=commitdiff_plain;h=87de8ee8c36790f0830886a4b6de37c1dcc76296;p=mediacube.git git-tfs-id: [tfs.userrendszerhaz.hu:8080/tfs/DefaultCollection]$/MediaCube;C32130 --- diff --git a/server/-configuration/build-mediacube-plugins.launch b/server/-configuration/build-mediacube-plugins.launch new file mode 100644 index 00000000..544a7670 --- /dev/null +++ b/server/-configuration/build-mediacube-plugins.launch @@ -0,0 +1,19 @@ + + + + + + + + + + + + + + + + + + + diff --git a/server/-configuration/run-mediacube-server-hirtv.launch b/server/-configuration/run-mediacube-server-hirtv.launch index 689f6fb2..f92465a3 100644 --- a/server/-configuration/run-mediacube-server-hirtv.launch +++ b/server/-configuration/run-mediacube-server-hirtv.launch @@ -22,7 +22,7 @@ - + diff --git a/server/-configuration/run-mediacube-server-mv.launch b/server/-configuration/run-mediacube-server-mv.launch index 68ece584..c56d7cff 100644 --- a/server/-configuration/run-mediacube-server-mv.launch +++ b/server/-configuration/run-mediacube-server-mv.launch @@ -22,7 +22,7 @@ - + diff --git a/server/-configuration/run-mediacube-server-test.launch b/server/-configuration/run-mediacube-server-test.launch index a17e092c..ed4ce421 100644 --- a/server/-configuration/run-mediacube-server-test.launch +++ b/server/-configuration/run-mediacube-server-test.launch @@ -22,7 +22,7 @@ - + diff --git a/server/-configuration/run-mediacube-server-user.launch b/server/-configuration/run-mediacube-server-user.launch index 3f71cb6e..4b40742d 100644 --- a/server/-configuration/run-mediacube-server-user.launch +++ b/server/-configuration/run-mediacube-server-user.launch @@ -22,7 +22,7 @@ - + diff --git a/server/-configuration/run-mediacube-server.launch b/server/-configuration/run-mediacube-server.launch index f5ff4bb2..eee244c3 100644 --- a/server/-configuration/run-mediacube-server.launch +++ b/server/-configuration/run-mediacube-server.launch @@ -21,7 +21,7 @@ - + diff --git a/server/-dependencies/jobengine.target b/server/-dependencies/jobengine.target index 62660c20..ba125a69 100644 --- a/server/-dependencies/jobengine.target +++ b/server/-dependencies/jobengine.target @@ -1,5 +1,5 @@ - + @@ -15,10 +15,11 @@ + - - - + + + @@ -34,9 +35,11 @@ + + - - + + @@ -47,6 +50,7 @@ + diff --git a/server/-dependencies/libs/org.apache.felix.gogo.command_0.10.0.v201209301215.jar b/server/-dependencies/libs/org.apache.felix.gogo.command_0.10.0.v201209301215.jar new file mode 100644 index 00000000..6c7c9d75 Binary files /dev/null and b/server/-dependencies/libs/org.apache.felix.gogo.command_0.10.0.v201209301215.jar differ diff --git a/server/-dependencies/libs/org.eclipse.equinox.launcher_1.3.200.v20160318-1642.jar b/server/-dependencies/libs/org.eclipse.equinox.launcher_1.3.200.v20160318-1642.jar new file mode 100644 index 00000000..2aabbf4b Binary files /dev/null and b/server/-dependencies/libs/org.eclipse.equinox.launcher_1.3.200.v20160318-1642.jar differ diff --git a/server/-dependencies/pom.xml b/server/-dependencies/pom.xml index c3263272..c1e79f10 100644 --- a/server/-dependencies/pom.xml +++ b/server/-dependencies/pom.xml @@ -59,10 +59,10 @@ install-file - libs/humble-video-arch-x86_64-pc-linux-gnu6-0.2.1.jar - humble.video - linux - 0.2.1 + libs/sqljdbc42.jar + com.microsoft.sqlserver + sqljdbc + 6.0.8112-100 jar @@ -73,10 +73,10 @@ install-file - libs/humble-video-arch-x86_64-w64-mingw32-0.2.1.jar - humble.video - windows - 0.2.1 + libs/org.apache.felix.gogo.command_0.10.0.v201209301215.jar + org.apache.felix + org.apache.felix.gogo.command + 0.10.0.v201209301215 jar @@ -87,24 +87,10 @@ install-file - libs/humble-video-noarch-0.2.1.jar - humble.video - noarch - 0.2.1 - jar - - - - install5 - package - - install-file - - - libs/sqljdbc42.jar - com.microsoft.sqlserver - sqljdbc - 6.0.8112-100 + libs/org.eclipse.equinox.launcher_1.3.200.v20160318-1642.jar + org.eclipse.platform + org.eclipse.equinox.launcher + 1.3.200.v20160318-1642 jar @@ -119,6 +105,25 @@ default-cli + + org.eclipse.platform:org.eclipse.equinox.launcher:1.3.200.v20160318-1642 + + + org.apache.felix:org.apache.felix.fileinstall:3.6.8 + true + + + net.java.dev.jna:jna:4.2.0 + + + io.humble:humble-video-all:0.2.1 + + + io.humble:humble-video-arch-x86_64-pc-linux-gnu6:0.2.1 + + + io.humble:humble-video-arch-x86_64-w64-mingw32:0.2.1 + org.ow2.asm:asm:5.0.1 @@ -159,11 +164,12 @@ org.apache.felix:org.apache.felix.gogo.shell:0.10.0 - org.apache.felix:org.apache.felix.gogo.command:0.10.0 + org.apache.felix:org.apache.felix.gogo.runtime:0.10.0 - org.apache.felix:org.apache.felix.gogo.runtime:0.10.0 + org.apache.felix:org.apache.felix.gogo.command:0.10.0.v201209301215 + commons-io:commons-io:2.2 @@ -203,15 +209,6 @@ org.mybatis:mybatis:3.5.2 - - humble.video:linux:0.2.1 - - - humble.video:windows:0.2.1 - - - humble.video:noarch:0.2.1 - com.fasterxml.jackson.core:jackson-core:${fasterxml-jackson-core} diff --git a/server/-modules/plugins/pom.xml b/server/-modules/plugins/pom.xml new file mode 100644 index 00000000..30cf7f03 --- /dev/null +++ b/server/-modules/plugins/pom.xml @@ -0,0 +1,130 @@ + + + 4.0.0 + user.jobengine + MediaCube-plugins + 1.0.0 + pom + + ../../user.mediacube.gui + ../../user.mediacube.metadata + ../../user.jobengine.osgi.commons + ../../user.jobengine.osgi.db + ../../user.jobengine.osgi.server + ../../user.jobengine.osgi.services + ../../user.commons.log4j2 + ../../user.commons.zk + ../../user.tsm.client + + + + 1.0.0 + 1.0.0 + 1.8 + 1.8 + UTF-8 + UTF-8 + + + + + dependencies + p2 + file:${project.basedir}/../-dependencies/target/repository/ + + + + + + + org.eclipse.tycho + tycho-packaging-plugin + ${tycho.version} + + ${project.artifactId}_${project.version} + + + + org.apache.maven.plugins + maven-deploy-plugin + 2.8.2 + + true + + + + org.eclipse.tycho + tycho-maven-plugin + ${tycho.version} + true + + + + + + org.eclipse.tycho + target-platform-configuration + ${tycho.version} + + + + org.eclipse.m2e + lifecycle-mapping + 1.0.0 + + + + + + + org.eclipse.tycho + + + tycho-packaging-plugin + + + [1.0.0,) + + + validate-id + + validate-version + + build-qualifier + + build-qualifier-aggregator + + + + + + + + + + + org.apache.maven.plugins + + + maven-clean-plugin + + + [2.5,) + + + clean + + + + + + + + + + + + + + diff --git a/server/-modules/pom.xml b/server/-modules/pom.xml index 668b8789..6fe687f7 100644 --- a/server/-modules/pom.xml +++ b/server/-modules/pom.xml @@ -35,7 +35,6 @@ eclipse-neon http://download.eclipse.org/releases/neon - p2 diff --git a/server/-product/mediacube.product b/server/-product/mediacube.product index 3f116596..314bd6eb 100644 --- a/server/-product/mediacube.product +++ b/server/-product/mediacube.product @@ -107,10 +107,11 @@ + - - - + + + @@ -126,9 +127,11 @@ - - - + + + + + @@ -138,6 +141,8 @@ + + @@ -157,7 +162,7 @@ - + diff --git a/server/hu.user.mediacube.executors.tests/src/hu/user/mediacube/executors/tests/SmallTests.java b/server/hu.user.mediacube.executors.tests/src/hu/user/mediacube/executors/tests/SmallTests.java index f6f0feb4..cccf6a20 100644 --- a/server/hu.user.mediacube.executors.tests/src/hu/user/mediacube/executors/tests/SmallTests.java +++ b/server/hu.user.mediacube.executors.tests/src/hu/user/mediacube/executors/tests/SmallTests.java @@ -6,6 +6,9 @@ import static org.junit.Assert.assertTrue; import java.io.File; import java.io.IOException; +import java.lang.management.ManagementFactory; +import java.lang.management.ThreadInfo; +import java.lang.management.ThreadMXBean; import java.lang.reflect.Method; import java.nio.file.DirectoryStream; import java.nio.file.FileStore; @@ -14,6 +17,7 @@ import java.nio.file.Path; import java.nio.file.Paths; import java.sql.Timestamp; import java.text.DecimalFormat; +import java.text.Normalizer; import java.text.NumberFormat; import java.text.SimpleDateFormat; import java.time.Duration; @@ -36,6 +40,7 @@ import java.util.Map; import java.util.Set; import java.util.TreeMap; import java.util.TreeSet; +import java.util.regex.Pattern; import org.apache.commons.io.FileUtils; import org.apache.commons.io.FilenameUtils; @@ -49,6 +54,7 @@ import com.ibm.nosql.json.api.BasicDBObject; import groovy.lang.GroovyClassLoader; import user.commons.RemoteFile; import user.commons.StoreUri; +import user.commons.mediaarea.StreamKind; import user.commons.mediatool.Timecode; import user.commons.mediatool.Timecode.Type; import user.commons.remotestore.RemoteStoreProtocol; @@ -70,6 +76,8 @@ public class SmallTests { class PojoRoot { } + private static final String DOT = "."; + public static String readableFileSize(long size) { if (size <= 0) return "0"; @@ -268,7 +276,7 @@ public class SmallTests { System.out.println(value); } String name = "valammi.mxf"; - System.out.println(name.substring(0, name.lastIndexOf("."))); + System.out.println(name.substring(0, name.lastIndexOf(DOT))); boolean create = false; create |= false; @@ -568,7 +576,7 @@ public class SmallTests { System.out.println("Start"); for (String fileName : missing) { String onlyId = fileName.trim(); - onlyId = onlyId.substring(0, onlyId.lastIndexOf(".")); + onlyId = onlyId.substring(0, onlyId.lastIndexOf(DOT)); boolean tsmContains = tsm.contains(fileName); @@ -642,7 +650,7 @@ public class SmallTests { omCount++; String fileName = omFilePath.trim().substring(omFilePath.lastIndexOf("/") + 1); - String fileId = fileName.substring(0, fileName.lastIndexOf(".")); + String fileId = fileName.substring(0, fileName.lastIndexOf(DOT)); boolean excludeContains = exclude.contains(fileId); if (excludeContains) { @@ -903,8 +911,8 @@ public class SmallTests { String[] tokens = t.split("\t"); String id = tokens[0].replace(".part", ""); - if (id.contains(".")) - id = id.substring(0, id.lastIndexOf(".")); + if (id.contains(DOT)) + id = id.substring(0, id.lastIndexOf(DOT)); if (tsmmap.containsKey(id)) { String tsmtime = tsmmap.get(id); if (tsmtime.compareTo(tokens[1]) > 0) { @@ -1138,4 +1146,72 @@ public class SmallTests { System.out.println(FileUtils.byteCountToDisplaySize(size / diff) + "/sec"); } -} \ No newline at end of file + + @Test + public void test99995() throws Exception { + Pattern DIACRITICS_AND_FRIENDS = Pattern.compile("[\\p{InCombiningDiacriticalMarks}\\p{IsLm}\\p{IsSk}]+"); + String name = "kedves_kiskacsa_furdik_a_toban_anyjahoz_keszul_lengyelorszagba.mxf"; + String type = "Muszter DEMO"; + int limit = 20; + String extension = ""; + if (name.contains(DOT)) { + extension = DOT + name.substring(name.lastIndexOf(DOT) + 1); + name = name.substring(0, name.lastIndexOf(DOT)); + } + String typeName = Normalizer.normalize(type, Normalizer.Form.NFD); + typeName = DIACRITICS_AND_FRIENDS.matcher(typeName).replaceAll(""); + typeName = typeName.replace(" ", "_"); + int allowedSize = limit - typeName.length() - 1 - extension.length(); + if (name.length() > allowedSize) + name = name.substring(0, allowedSize); + String formatatted = String.format("%s_%s%s", name, typeName, extension); + System.out.println(formatatted + " " + formatatted.length()); + } + + @Test + public void test99996() throws Exception { + System.out.println(System.getProperty("user.home")); + System.setProperty("java.library.path", "/users/elgekko"); + Path filePath = Paths.get("/opt/AMC/2021/ISSUES/#203/IFL000000071_43.mxf"); + user.commons.mediatool.MediaInfo mi = new user.commons.mediatool.MediaInfo(filePath); + mi.process(); + System.out.println(String.format("%s %d %s", mi.getCodecProfileName(), mi.getHeight(), mi.getDisplayAspect())); + } + + @Test + public void test99997() throws Exception { + + System.setProperty("jna.library.path", "/users/elgekko"); + List files = Arrays.asList("c:\\opt\\AMC\\2021\\ISSUES\\#203\\IFL000000071_43.mxf", "c:\\opt\\AMC\\2021\\ISSUES\\#203\\MX_12290041.mxf", + "c:\\opt\\AMC\\2021\\ISSUES\\#203\\DK_20344100_02_high_16_9.mxf", "c:\\opt\\AMC\\2021\\ISSUES\\#203\\IPR000011027_169.mxf"); + for (String file : files) { + try (user.commons.mediaarea.MediaInfo mi = new user.commons.mediaarea.MediaInfo(file)) { + //Display aspect ratio + //Format profile + //String inform = mi.inform(); + System.out.println("---------------------------------------"); + System.out.println(file); + System.out.println(mi.get(StreamKind.Video, 0, "Width")); + System.out.println(mi.get(StreamKind.Video, 0, "Height")); + System.out.println(mi.get(StreamKind.Video, 0, "DisplayAspectRatio/String")); + System.out.println(mi.get(StreamKind.Video, 0, "Format_Profile")); + } catch (Exception ex) { + throw new RuntimeException(ex); + } + } + + } + + @Test + public void test99998() throws Exception { + ThreadMXBean threadMXBean = ManagementFactory.getThreadMXBean(); + + for (Long threadID : threadMXBean.getAllThreadIds()) { + ThreadInfo info = threadMXBean.getThreadInfo(threadID); + System.out.println("Thread name: " + info.getThreadName()); + System.out.println("Thread State: " + info.getThreadState()); + long cpuTime = threadMXBean.getThreadCpuTime(threadID); + System.out.println(String.format("CPU time: %.2f %%", cpuTime == 0 ? cpuTime : cpuTime * 100 / 1000000000d)); + } + } +} diff --git a/server/user.commons.log4j2/pom.xml b/server/user.commons.log4j2/pom.xml index daf78393..e401fb54 100644 --- a/server/user.commons.log4j2/pom.xml +++ b/server/user.commons.log4j2/pom.xml @@ -12,4 +12,36 @@ user.commons.log4j2 eclipse-plugin 1.0.0 + + + + + + org.apache.maven.plugins + maven-resources-plugin + 2.7 + + + copy-jar + install + + copy-resources + + + ${project.basedir}}/../-product/target/deploy + + + target + + ${project.artifactId}_${project.version}.jar + + + + + + + + + + \ No newline at end of file diff --git a/server/user.jobengine.executors/amc/user/jobengine/server/steps/ForkDownloadStep.java b/server/user.jobengine.executors/amc/user/jobengine/server/steps/ForkDownloadStep.java index 8b72db9d..cc386f64 100644 --- a/server/user.jobengine.executors/amc/user/jobengine/server/steps/ForkDownloadStep.java +++ b/server/user.jobengine.executors/amc/user/jobengine/server/steps/ForkDownloadStep.java @@ -9,14 +9,12 @@ import java.util.ArrayList; import java.util.HashMap; import java.util.List; import java.util.Map; -import java.util.Set; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import user.commons.DownloadableMedia; import user.commons.JobStatus; -import user.commons.MediaCubeMarker; import user.commons.StoreUri; import user.commons.remotestore.RemoteStoreProtocol; import user.jobengine.db.Store; @@ -28,9 +26,6 @@ public class ForkDownloadStep extends JobStep { @StepEntry public Object[] execute(String tempStoreName, String template, String expectedColorSpace, int limit) throws Exception { - - sendStatusReport(); - //return null; DirectoryStream directoryStream = null; int count = limit; int allCount = 0; @@ -164,25 +159,6 @@ public class ForkDownloadStep extends JobStep { return true; } - private void sendStatusReport() { - IJobEngine engine = getEngine(); - int jobCount = 0; - StringBuilder sb = new StringBuilder(); - Map jobs = engine.getJobs(); - if (jobs != null) { - Set keys = jobs.keySet(); - jobCount = keys.size(); - for (Long key : keys) { - IJobRuntime runtime = jobs.get(key); - if (JobStatus.SUSPENDED.equals(runtime.getStatus())) - sb.append(runtime.getRelated() + " SUSPENDED: " + runtime.getDescription() + "
"); - } - } - MediaCubeMarker marker = new MediaCubeMarker(); - marker.setSubject("AMC MediaCube feldolgozási sor " + jobCount); - logger.info(marker, sb.toString()); - } - private boolean targetExists(StoreUri storeUri, String fileName) { try { Path targetFile = Paths.get(storeUri.toString(true), fileName); diff --git a/server/user.jobengine.executors/amc/user/jobengine/server/steps/PASAPOOLTransferToStep.java b/server/user.jobengine.executors/amc/user/jobengine/server/steps/PASAPOOLTransferToStep.java index 52c90280..b5106b6e 100644 --- a/server/user.jobengine.executors/amc/user/jobengine/server/steps/PASAPOOLTransferToStep.java +++ b/server/user.jobengine.executors/amc/user/jobengine/server/steps/PASAPOOLTransferToStep.java @@ -3,7 +3,6 @@ package user.jobengine.server.steps; import java.nio.file.Path; import java.nio.file.Paths; -import org.apache.commons.io.FileUtils; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.apache.logging.log4j.Marker; @@ -11,7 +10,8 @@ import org.apache.logging.log4j.MarkerManager; import user.commons.RemoteFile; import user.commons.StoreUri; -import user.commons.mediatool.MediaInfo; +import user.commons.mediaarea.MediaArea; +import user.commons.strings.FileSizeUtils; public class PASAPOOLTransferToStep extends TransferStep { private static final Logger logger = LogManager.getLogger(); @@ -32,11 +32,13 @@ public class PASAPOOLTransferToStep extends TransferStep { long start = System.currentTimeMillis(); Object[] result = super.execute(sourceStoreUri, sourceFileName, targetStoreUri, targetFileName); RemoteFile remoteFile = sourceStoreUri.getRemoteFile(targetFileName); - if (remoteFile != null) { + if (remoteFile != null && remoteFile.getSize() > 0) { long diff = (System.currentTimeMillis() - start) / 1000; - long bytesSpeed = remoteFile.getSize() / diff; - String speed = FileUtils.byteCountToDisplaySize(bytesSpeed); - logger.info(getMarker(), "{} size is {}, upload speed was {}/sec", targetFileName, FileUtils.byteCountToDisplaySize(remoteFile.getSize()), speed); + if (diff > 0) { + long bytesSpeed = remoteFile.getSize() / diff; + String speed = FileSizeUtils.sizeAsString(bytesSpeed); + logger.info(getMarker(), "{} size is {}, upload speed was {}/sec", targetFileName, FileSizeUtils.sizeAsString(remoteFile.getSize()), speed); + } } return result; } @@ -44,7 +46,7 @@ public class PASAPOOLTransferToStep extends TransferStep { // HD // height = 1080 // >PEABLEBEACH | \\10.170.100.21\media\BeachPool - private boolean isHD(MediaInfo mi) { + private boolean isHD(MediaArea mi) { return mi.getHeight() == 1080; } @@ -87,8 +89,9 @@ public class PASAPOOLTransferToStep extends TransferStep { // profile=High // aspect=16:9 // >SELENIOPOOL_16_9 | \\10.170.100.21\media\SelenioPool\16_9 - private boolean isSD_HIGH_16_9(MediaInfo mi) { - return mi.getHeight() < 650 && CODEC_PROFILE_HIGH.equals(mi.getCodecProfileName()) && DISPLAY_ASPECT_16_9.equals(mi.getDisplayAspect()); + private boolean isSD_HIGH_16_9(MediaArea mi) { + String formatProfileName = mi.getFormatProfileName(); + return mi.getHeight() < 650 && formatProfileName.contains(CODEC_PROFILE_HIGH) && DISPLAY_ASPECT_16_9.equals(mi.getDisplayAspect()); } // SD - HIGH@HIGH / 4:3 @@ -96,29 +99,25 @@ public class PASAPOOLTransferToStep extends TransferStep { // profile=High // aspect=4:3 // >SELENIOPOOL_4_3 | \\10.170.100.21\media\SelenioPool\4_3 - private boolean isSD_HIGH_4_3(MediaInfo mi) { - String codecProfileName = mi.getCodecProfileName(); - if (codecProfileName != null) - codecProfileName = codecProfileName.toUpperCase(); - return mi.getHeight() < 650 && CODEC_PROFILE_HIGH.equals(codecProfileName) && DISPLAY_ASPECT_4_3.equals(mi.getDisplayAspect()); + private boolean isSD_HIGH_4_3(MediaArea mi) { + String formatProfileName = mi.getFormatProfileName(); + return mi.getHeight() < 650 && formatProfileName.contains(CODEC_PROFILE_HIGH) && DISPLAY_ASPECT_4_3.equals(mi.getDisplayAspect()); } // SD - MAIN/422@HIGH // height < 650 // profile=Main || profile=4:2:2 // >PEABLEBEACH | \\10.170.100.21\media\BeachPool - private boolean isSD_MAIN_422(MediaInfo mi) { - String codecProfileName = mi.getCodecProfileName(); - if (codecProfileName != null) - codecProfileName = codecProfileName.toUpperCase(); - return mi.getHeight() < 650 && (CODEC_PROFILE_MAIN.equals(codecProfileName) || CODEC_PROFILE_422.equals(mi.getCodecProfileName())); + private boolean isSD_MAIN_422(MediaArea mi) { + String formatProfileName = mi.getFormatProfileName(); + return mi.getHeight() < 650 && (formatProfileName.contains(CODEC_PROFILE_MAIN) || formatProfileName.contains(CODEC_PROFILE_422)); } protected void logMediaProfile() { Marker marker = MarkerManager.getMarker("MEDIAPROFILE"); try { Path filePath = Paths.get(sourceStoreUri.toString(true), sourceFileName); - MediaInfo mi = new MediaInfo(filePath); + MediaArea mi = new MediaArea(filePath); mi.process(); if (isHD(mi)) { diff --git a/server/user.jobengine.executors/amc/user/jobengine/server/steps/ReportServerStatusStep.java b/server/user.jobengine.executors/amc/user/jobengine/server/steps/ReportServerStatusStep.java new file mode 100644 index 00000000..3c0026b4 --- /dev/null +++ b/server/user.jobengine.executors/amc/user/jobengine/server/steps/ReportServerStatusStep.java @@ -0,0 +1,64 @@ +package user.jobengine.server.steps; + +import java.lang.management.ManagementFactory; +import java.lang.management.ThreadInfo; +import java.lang.management.ThreadMXBean; +import java.text.SimpleDateFormat; +import java.util.Date; +import java.util.Map; +import java.util.Set; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + +import user.commons.JobStatus; +import user.commons.MediaCubeMarker; +import user.commons.strings.FileSizeUtils; +import user.jobengine.server.IJobRuntime; + +public class ReportServerStatusStep extends JobStep { + private static final Logger logger = LogManager.getLogger(); + private static SimpleDateFormat df = new SimpleDateFormat("yyyy.MM.dd hh:mm"); + + @StepEntry + public Object[] execute() throws Exception { + MediaCubeMarker marker = new MediaCubeMarker(); + + int jobCount = 0; + StringBuilder sb = new StringBuilder("Állapotjelentés:
"); + Map jobs = getEngine().getJobs(); + Set keys = jobs.keySet(); + jobCount = keys.size() - 1; + marker.setSubject(String.format("%s [%d futó folyamat] AMC MediaCube report", df.format(new Date()), jobCount)); + for (Long key : keys) { + IJobRuntime runtime = jobs.get(key); + if (JobStatus.SUSPENDED.equals(runtime.getStatus())) + sb.append(String.format("Felfüggesztve: %s %s %s %s %s
", df.format(runtime.getSubmitted()), df.format(runtime.getFinished()), + runtime.getStatus(), runtime.getRelated(), runtime.getDescription())); + } + + // sb.append("
"); + // float cpuTime = getCPUTime(); + // sb.append(String.format("Processzor használat: %.2f %%
", cpuTime == 0 ? cpuTime : cpuTime * 100 / 1000000000d)); + sb.append("
"); + sb.append(String.format("Szabad JVM memória: %s
", FileSizeUtils.sizeAsString(Runtime.getRuntime().freeMemory()))); + sb.append(String.format("Használt JVM memória: %s
", + FileSizeUtils.sizeAsString(Runtime.getRuntime().maxMemory() - Runtime.getRuntime().freeMemory()))); + sb.append(String.format("Maximum JVM memória: %s
", FileSizeUtils.sizeAsString(Runtime.getRuntime().maxMemory()))); + logger.info(marker, sb.toString()); + + return null; + } + + private long getCPUTime() { + ThreadMXBean threadMXBean = ManagementFactory.getThreadMXBean(); + long result = 0; + for (Long threadID : threadMXBean.getAllThreadIds()) { + ThreadInfo info = threadMXBean.getThreadInfo(threadID); + System.out.println("Thread name: " + info.getThreadName()); + System.out.println("Thread State: " + info.getThreadState()); + result += threadMXBean.getThreadCpuTime(threadID); + } + return result; + } +} diff --git a/server/user.jobengine.executors/amc/user/jobengine/server/steps/TransferStep.java b/server/user.jobengine.executors/amc/user/jobengine/server/steps/TransferStep.java index 39c338c3..d285f70d 100644 --- a/server/user.jobengine.executors/amc/user/jobengine/server/steps/TransferStep.java +++ b/server/user.jobengine.executors/amc/user/jobengine/server/steps/TransferStep.java @@ -3,10 +3,12 @@ package user.jobengine.server.steps; import java.nio.file.Path; import java.nio.file.Paths; +import org.apache.commons.net.ftp.FTPClient; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import user.commons.StoreUri; +import user.commons.remotestore.FtpDirectoryLister; import user.commons.remotestore.RemoteStoreProtocol; import user.jobengine.db.Store; @@ -77,18 +79,46 @@ public class TransferStep extends JobStep { String currentTargetFileName = targetFileName; boolean renameAfterCopy = false; - if (getTmpExtension() != null && RemoteStoreProtocol.LOCAL.equals(targetStoreUri.getProtocol())) { - currentTargetFileName += getTmpExtension(); - renameAfterCopy = true; + boolean renameAfterFTP = false; + if (getTmpExtension() != null) { + if (RemoteStoreProtocol.LOCAL.equals(targetStoreUri.getProtocol())) { + currentTargetFileName += getTmpExtension(); + renameAfterCopy = true; + } + Store targetStore = getManager().getStore(targetStoreUri.getStoreId()); + + if (RemoteStoreProtocol.FTP.equals(targetStoreUri.getProtocol()) && !"NEXIO1".equals(targetStore.getName()) + && !"NEXIO2".equals(targetStore.getName())) { + currentTargetFileName += getTmpExtension(); + renameAfterFTP = true; + } } sourceStoreUri.transferFrom(targetStoreUri, sourceFileName, currentTargetFileName); - logger.info(getMarker(), "Transfer of {} completed from {} to {}", sourceFileName, sourceStoreUri, targetStoreUri); + + logger.info(getMarker(), "Transfer of {} completed from {} to {}", sourceFileName, sourceStoreUri, currentTargetFileName); if (renameAfterCopy) { Path tmpTargetFile = Paths.get(targetStoreUri.toString(true), currentTargetFileName); Path targetFile = Paths.get(targetStoreUri.toString(true), targetFileName); - tmpTargetFile.toFile().renameTo(targetFile.toFile()); + try { + logger.info(getMarker(), "Renaming LOCAL file from {} to {} on {}", currentTargetFileName, targetFileName, sourceStoreUri); + tmpTargetFile.toFile().renameTo(targetFile.toFile()); + } catch (Exception e) { + logger.error(getMarker(), e.getMessage()); + } + } + if (renameAfterFTP) { + try { + FtpDirectoryLister lister = (FtpDirectoryLister) targetStoreUri.getLister(); + FTPClient client = lister.connect(); + logger.info(getMarker(), "Renaming FTP file from {} to {} on {}", currentTargetFileName, targetFileName, sourceStoreUri); + client.rename(currentTargetFileName, targetFileName); + } catch (Exception e) { + logger.error(getMarker(), e.getMessage()); + } finally { + targetStoreUri.cleanUp(); + } } } } diff --git a/server/user.jobengine.executors/config/config-mv.xml b/server/user.jobengine.executors/config/config-mv.xml index e0f67e2e..18c392dd 100644 --- a/server/user.jobengine.executors/config/config-mv.xml +++ b/server/user.jobengine.executors/config/config-mv.xml @@ -1,5 +1,5 @@ - - + + \ No newline at end of file diff --git a/server/user.jobengine.executors/src/user/jobengine/server/steps/CancelableStep.java b/server/user.jobengine.executors/src/user/jobengine/server/steps/CancelableStep.java index b01bf0a2..9979d22e 100644 --- a/server/user.jobengine.executors/src/user/jobengine/server/steps/CancelableStep.java +++ b/server/user.jobengine.executors/src/user/jobengine/server/steps/CancelableStep.java @@ -21,17 +21,17 @@ public class CancelableStep extends JobStep { // ftpTest(); - if (param == 0) - Thread.sleep(5000); - if (param == 1) - throw new Exception("Error teszt"); + // if (param == 0) + // Thread.sleep(5000); + // if (param == 1) + // throw new Exception("Error teszt"); Marker marker = MarkerManager.getMarker("MEDIAPROFILE"); for (int i = 0; i < count; i++) { if (getJobRuntime().isWaitingCancel()) break; - Thread.sleep(500); + Thread.sleep(200); int progress = (i + 1) * 100 / count; setProgress(progress); logger.info(marker, "{}", i); diff --git a/server/user.jobengine.executors/src/user/jobengine/server/steps/CleanupMountedLocationStep.java b/server/user.jobengine.executors/src/user/jobengine/server/steps/CleanupMountedLocationStep.java index 4e6f10d1..0cd2e364 100644 --- a/server/user.jobengine.executors/src/user/jobengine/server/steps/CleanupMountedLocationStep.java +++ b/server/user.jobengine.executors/src/user/jobengine/server/steps/CleanupMountedLocationStep.java @@ -246,9 +246,11 @@ public class CleanupMountedLocationStep extends JobStep implements FileVisitor

0) { + if (!isArchived(filePath)) { + logger.error(marker, "A(z) {} anyag törlésre van kijelölve, de nem található az archívumban.", filePath); + return; + } } if (removeFiles(filePath, killDateFiles)) diff --git a/server/user.jobengine.executors/src/user/jobengine/server/steps/TSMRestoreStep.java b/server/user.jobengine.executors/src/user/jobengine/server/steps/TSMRestoreStep.java index 5b410055..6b40e3b7 100644 --- a/server/user.jobengine.executors/src/user/jobengine/server/steps/TSMRestoreStep.java +++ b/server/user.jobengine.executors/src/user/jobengine/server/steps/TSMRestoreStep.java @@ -2,7 +2,9 @@ package user.jobengine.server.steps; import java.io.IOException; import java.nio.file.Paths; +import java.text.Normalizer; import java.util.List; +import java.util.regex.Pattern; import org.apache.commons.lang.StringUtils; import org.apache.logging.log4j.LogManager; @@ -28,6 +30,8 @@ import user.jobengine.server.IJobEngine; import user.jobengine.server.IJobRuntime; public class TSMRestoreStep extends JobStep { + private static final String DOT = "."; + public static final Pattern DIACRITICS_AND_FRIENDS = Pattern.compile("[\\p{InCombiningDiacriticalMarks}\\p{IsLm}\\p{IsSk}]+"); private static final Logger logger = LogManager.getLogger(); private IItemManager manager; private StoreUri targetUri; @@ -60,6 +64,9 @@ public class TSMRestoreStep extends JobStep { marker = jobRuntime.getSessionMarker(); setAndCheck(mediaCubeMedia, targetPath, targetNamePattern, localRetrievePath, globalRetrievePath, jobEngine); String targetFileName = String.format(targetNamePattern, sourceFileName); + //20210129 + //targetFileName = getMaximizedFileName(mediaCubeMedia, targetFileName, 120); + Timecode timecode = new Timecode(mediaCubeMedia.getLength(), Type.PAL); try { String details = String.format("%s (%s)", sourceFileName, timecode.toString()); @@ -99,6 +106,24 @@ public class TSMRestoreStep extends JobStep { return null; } + private String getMaximizedFileName(Media mediaCubeMedia, String targetFileName, int limit) { + String name = targetFileName; + String extension = ""; + if (name.contains(DOT)) { + extension = DOT + name.substring(name.lastIndexOf(DOT) + 1); + name = name.substring(0, name.lastIndexOf(DOT)); + } + String typeName = Normalizer.normalize(mediaCubeMedia.getItemType().getName(), Normalizer.Form.NFD); + typeName = DIACRITICS_AND_FRIENDS.matcher(typeName).replaceAll(""); + typeName = typeName.replace(" ", "_"); + + int allowedSize = limit - typeName.length() - 1 - extension.length(); + if (name.length() > allowedSize) + name = name.substring(0, allowedSize); + + return String.format("%s_%s%s", name, typeName, extension); + } + private String getSourceFileName(Media mediaCubeMedia, Store store) { List mediaFiles = mediaCubeMedia.getMediaFiles(); if (mediaFiles == null) diff --git a/server/user.jobengine.executors/src/user/jobengine/server/steps/TestForkCancelableStep.java b/server/user.jobengine.executors/src/user/jobengine/server/steps/TestForkCancelableStep.java index e622b543..28e66d3f 100644 --- a/server/user.jobengine.executors/src/user/jobengine/server/steps/TestForkCancelableStep.java +++ b/server/user.jobengine.executors/src/user/jobengine/server/steps/TestForkCancelableStep.java @@ -12,7 +12,7 @@ import user.jobengine.server.IJobRuntime; public class TestForkCancelableStep extends JobStep { private static final String CHILD_TEMPLATE = "cancelable.xml"; private static final Logger logger = LogManager.getLogger(); - int count = 10; + int count = 5; @StepEntry public Object[] execute(IJobEngine jobEngine, IJobRuntime jobRuntime) throws Exception { diff --git a/server/user.jobengine.osgi.commons/META-INF/MANIFEST.MF b/server/user.jobengine.osgi.commons/META-INF/MANIFEST.MF index 3ef3e4ac..f59caf2a 100644 --- a/server/user.jobengine.osgi.commons/META-INF/MANIFEST.MF +++ b/server/user.jobengine.osgi.commons/META-INF/MANIFEST.MF @@ -9,6 +9,7 @@ Import-Package: com.fasterxml.jackson.annotation;version="2.4.5", com.fasterxml.jackson.databind.util;version="2.4.5", com.fasterxml.jackson.datatype.joda;version="2.4.5", com.fasterxml.jackson.jaxrs.json;version="2.4.5", + com.sun.jna, io.humble.video, io.humble.video.awt, junit.framework, @@ -29,6 +30,7 @@ Export-Package: user.commons, user.commons.ftp, user.commons.harris, user.commons.logging, + user.commons.mediaarea, user.commons.mediatool, user.commons.morpheus, user.commons.nexio, diff --git a/server/user.jobengine.osgi.commons/pom.xml b/server/user.jobengine.osgi.commons/pom.xml index c02e675f..04d98e2d 100644 --- a/server/user.jobengine.osgi.commons/pom.xml +++ b/server/user.jobengine.osgi.commons/pom.xml @@ -11,4 +11,36 @@ user.jobengine.osgi.commons eclipse-plugin 1.0.0 + + + + + + org.apache.maven.plugins + maven-resources-plugin + 2.7 + + + copy-jar + install + + copy-resources + + + ${project.basedir}}/../-product/target/deploy + + + target + + ${project.artifactId}_${project.version}.jar + + + + + + + + + + \ No newline at end of file diff --git a/server/user.jobengine.osgi.commons/src/user/commons/mediaarea/InfoKind.java b/server/user.jobengine.osgi.commons/src/user/commons/mediaarea/InfoKind.java new file mode 100644 index 00000000..f9d82732 --- /dev/null +++ b/server/user.jobengine.osgi.commons/src/user/commons/mediaarea/InfoKind.java @@ -0,0 +1,39 @@ +package user.commons.mediaarea; + +/** + * + */ +public enum InfoKind { + /** + * Unique name of parameter. + */ + Name, + /** + * Value of parameter. + */ + Text, + /** + * Unique name of measure unit of parameter. + */ + Measure, Options, + /** + * Translated name of parameter. + */ + Name_Text, + /** + * Translated name of measure unit. + */ + Measure_Text, + /** + * More information about the parameter. + */ + Info, + /** + * How this parameter is supported, could be N (No), B (Beta), R (Read only), W (Read/Write). + */ + HowTo, + /** + * Domain of this piece of information. + */ + Domain; +} diff --git a/server/user.jobengine.osgi.commons/src/user/commons/mediaarea/InformParser.java b/server/user.jobengine.osgi.commons/src/user/commons/mediaarea/InformParser.java new file mode 100644 index 00000000..725c1ed5 --- /dev/null +++ b/server/user.jobengine.osgi.commons/src/user/commons/mediaarea/InformParser.java @@ -0,0 +1,88 @@ +package user.commons.mediaarea; + +import java.io.BufferedReader; +import java.io.IOException; +import java.io.StringReader; +import java.util.LinkedHashMap; +import java.util.Map; + +/** + * Simple parser to adapt inform response to meta objects. The default MediaInfo inform call returns data in the form
+ * + * 

+ * General
+Complete name                            : /path/to/input/file.ext
+Format                                   : Windows Media
+...
+
+Video
+ID                                       : 1
+Format                                   : FMT
+Codec ID                                 : CODECID
+...
+
+Audio
+ID                                       : 2
+Format                                   : MP3
+...
+ * 
+ *
+ */ +public class InformParser { + + static final String DELIM = ":"; + + static MediaMetadata createMM(String type, BufferedReader r, Map pairs) throws IOException { + pairs.clear(); + String meta; + while ((meta = r.readLine()) != null && !meta.isEmpty()) { + int idx = meta.indexOf(DELIM); + if (idx == -1) { + throw new IllegalStateException("Parser received unexpected data, should contain ':'. Type=" + type + " line=" + meta); + } + String key = meta.substring(0, idx).trim(); + String value = meta.substring(++idx).trim(); + pairs.put(key, value); + } + return new MediaMetadata(type, pairs); + } + + public static void main(String[] args) throws Exception { + if (args.length == 0) { + System.out.println("Usage: java -jar .jar [FILE]..."); + System.exit(1); + } + for (String arg : args) { + try (MediaInfo mi = new MediaInfo(arg)) { + Map results = InformParser.parse(mi.inform()); + System.out.println("Inform " + arg + "\n" + results + "\n"); + } + } + + } + + /** + * + * @param inform + * A MediaInfo formatted inform string + * @return Map containing parsed meta data. + */ + public static Map parse(String inform) { + LinkedHashMap results = new LinkedHashMap<>(); + try { + BufferedReader reader = new BufferedReader(new StringReader(inform)); + String line; + Map pairs = new LinkedHashMap<>(); + // first read should always be the type + while ((line = reader.readLine()) != null) { + if (!line.isEmpty()) { + results.put(line, createMM(line, reader, pairs)); + } + } + } catch (IOException ex) { + // shouldn't happen since we're reading from a String + // TODO add logging? + } + return results; + } +} diff --git a/server/user.jobengine.osgi.commons/src/user/commons/mediaarea/InformResult.java b/server/user.jobengine.osgi.commons/src/user/commons/mediaarea/InformResult.java new file mode 100644 index 00000000..103b6c9f --- /dev/null +++ b/server/user.jobengine.osgi.commons/src/user/commons/mediaarea/InformResult.java @@ -0,0 +1,56 @@ +package user.commons.mediaarea; + +import java.io.File; +import java.util.LinkedHashMap; +import java.util.Map; + +import javax.xml.bind.JAXBContext; +import javax.xml.bind.JAXBException; +import javax.xml.bind.Marshaller; +import javax.xml.bind.annotation.XmlElement; +import javax.xml.bind.annotation.XmlRootElement; +import javax.xml.bind.annotation.XmlTransient; + +/** + * Wrapper to perform JAXB Serialization + */ +@XmlRootElement +public class InformResult { + @XmlTransient + private static final JAXBContext CTX; + + static { + JAXBContext ctx; + try { + ctx = JAXBContext.newInstance(InformResult.class); + } catch (JAXBException ex) { + // should never happen + ctx = null; + } + CTX = ctx; + } + + public static InformResult getResult(String file) throws JAXBException { + return (InformResult) CTX.createUnmarshaller().unmarshal(new File(file)); + } + + public static void saveResult(InformResult ir, String filename) throws JAXBException { + Marshaller m = CTX.createMarshaller(); + m.marshal(ir, new File(filename)); + } + + @XmlElement + LinkedHashMap results = new LinkedHashMap<>(); + + public InformResult() { + } + + public InformResult(Map map) { + results.putAll(map); + } + + public LinkedHashMap getResults() { + return results; + } + +} diff --git a/server/user.jobengine.osgi.commons/src/user/commons/mediaarea/LibMediaInfo.java b/server/user.jobengine.osgi.commons/src/user/commons/mediaarea/LibMediaInfo.java new file mode 100644 index 00000000..f3c220e9 --- /dev/null +++ b/server/user.jobengine.osgi.commons/src/user/commons/mediaarea/LibMediaInfo.java @@ -0,0 +1,57 @@ +package user.commons.mediaarea; + +import static java.util.Collections.singletonMap; + +import java.lang.reflect.Method; + +import com.sun.jna.FunctionMapper; +import com.sun.jna.Library; +import com.sun.jna.Native; +import com.sun.jna.NativeLibrary; +import com.sun.jna.Pointer; +import com.sun.jna.WString; + +/** + * JNA Wrapper for MediaInfo. + */ +public interface LibMediaInfo extends Library { + + static final String LIB_NAME = "mediainfo"; + + final LibMediaInfo INSTANCE = (LibMediaInfo) Native.loadLibrary(LIB_NAME, LibMediaInfo.class, + singletonMap(OPTION_FUNCTION_MAPPER, (FunctionMapper) (NativeLibrary lib, Method method) -> "MediaInfo_" + method.getName())); + + /* Closes the file upon completion */ + void Close(Pointer Handle); + + int Count_Get(Pointer Handle, int StreamKind, int StreamNumber); + + /* Deconstructor */ + void Delete(Pointer Handle); + + WString Get(Pointer Handle, int StreamKind, int StreamNumber, WString parameter, int infoKind, int searchKind); + + WString GetI(Pointer Handle, int StreamKind, int StreamNumber, int parameterIndex, int infoKind); + + /* return information in various ways */ + WString Inform(Pointer Handle, int Reserved); + + /* Constructor */ + Pointer New(); + + /* Opens a file for inspection */ + int Open(Pointer Handle, WString file); + + int Open_Buffer_Continue(Pointer handle, byte[] buffer, int size); + + long Open_Buffer_Continue_GoTo_Get(Pointer handle); + + /* release buffer resources (close) */ + int Open_Buffer_Finalize(Pointer handle); + + /* Opens a buffered read for the specified length beginning at offset */ + int Open_Buffer_Init(Pointer handle, long length, long offset); + + /* Set options */ + WString Option(Pointer Handle, WString option, WString value); +} diff --git a/server/user.jobengine.osgi.commons/src/user/commons/mediaarea/MediaArea.java b/server/user.jobengine.osgi.commons/src/user/commons/mediaarea/MediaArea.java new file mode 100644 index 00000000..1c8bdd96 --- /dev/null +++ b/server/user.jobengine.osgi.commons/src/user/commons/mediaarea/MediaArea.java @@ -0,0 +1,41 @@ +package user.commons.mediaarea; + +import java.nio.file.Path; + +public class MediaArea { + + private Path filePath; + private int height; + private String displayAspect; + private String formatProfile = ""; + + public MediaArea(Path filePath) { + this.filePath = filePath; + + } + + public String getDisplayAspect() { + return displayAspect; + } + + public String getFormatProfileName() { + return formatProfile; + } + + public int getHeight() { + return height; + } + + public void process() { + //auto closable + try (MediaInfo mi = new MediaInfo(filePath.toAbsolutePath().toString())) { + height = Integer.parseInt(mi.get(StreamKind.Video, 0, "Height")); + displayAspect = mi.get(StreamKind.Video, 0, "DisplayAspectRatio/String"); + formatProfile = mi.get(StreamKind.Video, 0, "Format_Profile").toUpperCase(); + } catch (Exception ex) { + throw new RuntimeException(ex); + } + + } + +} diff --git a/server/user.jobengine.osgi.commons/src/user/commons/mediaarea/MediaInfo.java b/server/user.jobengine.osgi.commons/src/user/commons/mediaarea/MediaInfo.java new file mode 100644 index 00000000..e08e9771 --- /dev/null +++ b/server/user.jobengine.osgi.commons/src/user/commons/mediaarea/MediaInfo.java @@ -0,0 +1,222 @@ +package user.commons.mediaarea; + +import java.io.IOException; +import java.nio.ByteBuffer; +import java.nio.channels.SeekableByteChannel; + +import com.sun.jna.Pointer; +import com.sun.jna.WString; + +/** + * Class to wrap calls to LibMediaInfo via JNA. This class takes either a file path or a SeekableByteChannel and calls to LibMediaInfo to generate meta info. + * This class must be closed to properly release Native resources. It implements AutoCloseable to allow use try with resources. + */ +public class MediaInfo implements AutoCloseable { + + public static void setXMLOutput(MediaInfo m) { + m.option("Inform", "XML"); + } + + private Pointer pointer; + + private MediaInfo() { + pointer = LibMediaInfo.INSTANCE.New(); + } + + public MediaInfo(SeekableByteChannel channel) throws IOException { + this(); + // open, read and finalize the data in the buffer + long len = channel.size(); + final byte[] buff = new byte[64 * 1024]; + final ByteBuffer dst = ByteBuffer.wrap(buff); + int read; + LibMediaInfo.INSTANCE.Open_Buffer_Init(pointer, len, 0L); + while ((read = channel.read(dst)) != -1) { + int status = LibMediaInfo.INSTANCE.Open_Buffer_Continue(pointer, buff, read); + if ((status & Status.Finalized.val) == Status.Finalized.val) { + break; + } + long seekPos = LibMediaInfo.INSTANCE.Open_Buffer_Continue_GoTo_Get(pointer); + if (seekPos != -1) { + channel.position(seekPos); + LibMediaInfo.INSTANCE.Open_Buffer_Init(pointer, len, seekPos); + } + dst.clear(); + } + LibMediaInfo.INSTANCE.Open_Buffer_Finalize(pointer); + } + + public MediaInfo(String fileName) throws IOException { + this(); + int opened = LibMediaInfo.INSTANCE.Open(pointer, new WString(fileName)); + if (opened == 0) { + throw new IOException("Unable to open file with libmediainfo: " + fileName); + } + } + + /** + * Closes the underlying file handle, and releases the native instance. + * + * @throws Exception + */ + @Override + public void close() throws Exception { + if (pointer != null) { + LibMediaInfo.INSTANCE.Close(pointer); + LibMediaInfo.INSTANCE.Delete(pointer); + pointer = null; + } + } + + /** + * Get a piece of information about a file (parameter is an integer). + * + * + * @param streamKind + * Kind of Stream (general, video, audio...) + * @param streamNumber + * Stream number in Kind of Stream (first, second...) + * @param parameterIndex + * Parameter you are looking for in the Stream (Codec, width, bitrate...), in integer format (first parameter, second parameter...) + * @return information requested, empty string if not found + */ + public String get(StreamKind streamKind, int streamNumber, int parameterIndex) { + return get(streamKind, streamNumber, parameterIndex, InfoKind.Text); + } + + /** + * Get a piece of information about a file (parameter is an integer). + * + * + * @param streamKind + * Kind of Stream (general, video, audio...) + * @param streamNumber + * Stream number in Kind of Stream (first, second...) + * @param parameterIndex + * Parameter you are looking for in the Stream (Codec, width, bitrate...), in integer format (first parameter, second parameter...) + * @param infoKind + * Kind of information you want about the parameter (the text, the measure, the help...) + * @return a string about information you search, an empty string if there is a problem + */ + public String get(StreamKind streamKind, int streamNumber, int parameterIndex, InfoKind infoKind) { + return LibMediaInfo.INSTANCE.GetI(pointer, streamKind.ordinal(), streamNumber, parameterIndex, infoKind.ordinal()).toString(); + } + + /** + * Get a piece of information about a file (parameter is a string). + * + * @param streamKind + * Kind of Stream (general, video, audio...) + * @param streamNumber + * Stream number in Kind of Stream (first, second...) + * @param parameter + * Parameter you are looking for in the Stream (Codec, width, bitrate...), in string format ("Codec", "Width"...) + * @return a string about information you search, an empty string if there is a problem + */ + public String get(StreamKind streamKind, int streamNumber, String parameter) { + return get(streamKind, streamNumber, parameter, InfoKind.Text, InfoKind.Name); + } + + /** + * Get a piece of information about a file (parameter is a string). + * + * @param streamKind + * Kind of Stream (general, video, audio...) + * @param streamNumber + * Stream number in Kind of Stream (first, second...) + * @param parameter + * Parameter you are looking for in the Stream (Codec, width, bitrate...), in string format ("Codec", "Width"...) + * @param infoKind + * Kind of information you want about the parameter (the text, the measure, the help...) + * @return information requested, empty string if not found + */ + public String get(StreamKind streamKind, int streamNumber, String parameter, InfoKind infoKind) { + return get(streamKind, streamNumber, parameter, infoKind, InfoKind.Name); + } + + /** + * Get a piece of information about a file (parameter is a string). + * + * @param streamKind + * Kind of Stream (general, video, audio...) + * @param streamNumber + * Stream number in Kind of Stream (first, second...) + * @param parameter + * Parameter you are looking for in the Stream (Codec, width, bitrate...), in string format ("Codec", "Width"...) + * @param infoKind + * Kind of information you want about the parameter (the text, the measure, the help...) + * @param searchKind + * Where to look for the parameter + * @return a string about information you search, an empty string if there is a problem + */ + public String get(StreamKind streamKind, int streamNumber, String parameter, InfoKind infoKind, InfoKind searchKind) { + return LibMediaInfo.INSTANCE.Get(pointer, streamKind.ordinal(), streamNumber, new WString(parameter), infoKind.ordinal(), searchKind.ordinal()) + .toString(); + } + + /** + * Count of Streams of a Stream kind (StreamNumber not filled), or count of piece of information in this Stream. + * + * + * @param streamKind + * Kind of Stream (general, video, audio...) + * @return number of Streams of the given Stream kind + */ + public int getCount(StreamKind streamKind) { + //We should use NativeLong for -1, but it fails on 64-bit + //int Count_Get(Pointer Handle, int StreamKind, NativeLong StreamNumber); + //return MediaInfoDLL_Internal.INSTANCE.Count_Get(Handle, StreamKind.ordinal(), -1); + //so we use slower Get() with a character string + String streamCount = get(streamKind, 0, "StreamCount"); + if (streamCount == null || streamCount.length() == 0) { + return 0; + } + return Integer.parseInt(streamCount); + } + + /** + * Count of Streams of a Stream kind in the Stream Number. + * + * @param streamKind + * Kind of Stream (general, video, audio...) + * @param streamNumber + * Stream number in this kind of Stream (first, second...) + * @return number of Streams of the given Stream kind + */ + public int getCount(StreamKind streamKind, int streamNumber) { + return LibMediaInfo.INSTANCE.Count_Get(pointer, streamKind.ordinal(), streamNumber); + } + + /** + * Get all details about a file. + * + * @return All details about a file in one string + */ + public String inform() { + return LibMediaInfo.INSTANCE.Inform(pointer, 0).toString(); + } + + /** + * Configure or get information about MediaInfo. + * + * @param option + * The name of option + * @return Depends on the option: by default "" (nothing) means No, other means Yes + */ + public String option(String option) { + return option(option, ""); + } + + /** + * Configure or get information about MediaInfo. + * + * @param option + * The name of option + * @param value + * The value of option + * @return Depends on the option: by default "" (nothing) means No, other means Yes + */ + public String option(String option, String value) { + return LibMediaInfo.INSTANCE.Option(pointer, new WString(option), new WString(value)).toString(); + } +} diff --git a/server/user.jobengine.osgi.commons/src/user/commons/mediaarea/MediaMetadata.java b/server/user.jobengine.osgi.commons/src/user/commons/mediaarea/MediaMetadata.java new file mode 100644 index 00000000..fae0580d --- /dev/null +++ b/server/user.jobengine.osgi.commons/src/user/commons/mediaarea/MediaMetadata.java @@ -0,0 +1,64 @@ +package user.commons.mediaarea; + +import java.util.Collections; +import java.util.LinkedHashMap; +import java.util.Map; + +import javax.xml.bind.annotation.XmlElement; +import javax.xml.bind.annotation.XmlType; + +/** + * Simple data object to hold MediaInfo inform results. + */ +@XmlType +public final class MediaMetadata { + @XmlElement + String type; + + @XmlElement + Map metaData; + + /* JAXB */ + MediaMetadata() { + } + + public MediaMetadata(String type, Map metaData) { + this.type = type; + this.metaData = Collections.unmodifiableMap(new LinkedHashMap<>(metaData)); + } + + /** + * The meta data information for this type. The underlying implementation is an ordered map, but order may not be maintained if the object is constructed + * with an unordered Map. + * + * @return Unmodifiable meta map + */ + public Map getMetaData() { + return metaData; + } + + /** + * The meta info type (General, Video, Audio, etc) + * + * @return + */ + public String getType() { + return type; + } + + /** + * Returns a single meta data value from the meta map. + * + * @param key + * @return value if present or null + */ + public String getValue(final String key) { + return metaData.get(key); + } + + @Override + public String toString() { + return "MediaMetadata{" + "type=" + type + ", metaData=" + metaData + '}'; + } + +} diff --git a/server/user.jobengine.osgi.commons/src/user/commons/mediaarea/Status.java b/server/user.jobengine.osgi.commons/src/user/commons/mediaarea/Status.java new file mode 100644 index 00000000..077b074f --- /dev/null +++ b/server/user.jobengine.osgi.commons/src/user/commons/mediaarea/Status.java @@ -0,0 +1,14 @@ +package user.commons.mediaarea; + +/** + * Status codes when reading from a buffer. + */ +public enum Status { + None(0x00), Accepted(0x01), Filled(0x02), Updated(0x04), Finalized(0x08); + + public final int val; + + Status(int val) { + this.val = val; + } +} diff --git a/server/user.jobengine.osgi.commons/src/user/commons/mediaarea/StreamKind.java b/server/user.jobengine.osgi.commons/src/user/commons/mediaarea/StreamKind.java new file mode 100644 index 00000000..4bd1895e --- /dev/null +++ b/server/user.jobengine.osgi.commons/src/user/commons/mediaarea/StreamKind.java @@ -0,0 +1,8 @@ +package user.commons.mediaarea; + +/** + * Enumeration of the StreamKind. + */ +public enum StreamKind { + General, Video, Audio, Text, Other, Image, Menu; +} diff --git a/server/user.jobengine.osgi.commons/src/user/commons/strings/FileSizeUtils.java b/server/user.jobengine.osgi.commons/src/user/commons/strings/FileSizeUtils.java new file mode 100644 index 00000000..f37435f1 --- /dev/null +++ b/server/user.jobengine.osgi.commons/src/user/commons/strings/FileSizeUtils.java @@ -0,0 +1,31 @@ +package user.commons.strings; + +import java.text.DecimalFormat; + +public class FileSizeUtils { + private static final long K = 1024; + private static final long M = K * K; + private static final long G = M * K; + private static final long T = G * K; + + private static String format(final long value, final long divider, final String unit) { + final double result = divider > 1 ? (double) value / (double) divider : (double) value; + return new DecimalFormat("#,##0.#").format(result) + " " + unit; + } + + public static String sizeAsString(final long value) { + final long[] dividers = new long[] { T, G, M, K, 1 }; + final String[] units = new String[] { "TB", "GB", "MB", "KB", "B" }; + if (value < 1) + throw new IllegalArgumentException("Invalid file size: " + value); + String result = null; + for (int i = 0; i < dividers.length; i++) { + final long divider = dividers[i]; + if (value >= divider) { + result = format(value, divider, units[i]); + break; + } + } + return result; + } +} diff --git a/server/user.jobengine.osgi.db/migrations/scripts/026_insert_itemtype_demo_nyers.sql b/server/user.jobengine.osgi.db/migrations/scripts/026_insert_itemtype_demo_nyers.sql deleted file mode 100644 index 66858867..00000000 --- a/server/user.jobengine.osgi.db/migrations/scripts/026_insert_itemtype_demo_nyers.sql +++ /dev/null @@ -1,11 +0,0 @@ --- // Insert DEMO nyers itemtype --- Migration SQL that makes the change goes here. - -INSERT INTO ITEMTYPE (NAME) VALUES ('DEMO nyers') -@ - --- //@UNDO --- SQL to undo the change goes here. - -DELETE FROM ITEMTYPE WHERE NAME='DEMO nyers' -@ \ No newline at end of file diff --git a/server/user.jobengine.osgi.db/migrations/scripts/026_insert_itemtype_muszter_demo.sql b/server/user.jobengine.osgi.db/migrations/scripts/026_insert_itemtype_muszter_demo.sql new file mode 100644 index 00000000..3dbe459a --- /dev/null +++ b/server/user.jobengine.osgi.db/migrations/scripts/026_insert_itemtype_muszter_demo.sql @@ -0,0 +1,11 @@ +-- // Insert Muszetr DEMO itemtype +-- Migration SQL that makes the change goes here. + +INSERT INTO ITEMTYPE (NAME, DESCRIPTION, ISSTATIC) VALUES ('Muszter DEMO', 'Muszter DEMO', 'N') +@ + +-- //@UNDO +-- SQL to undo the change goes here. + +DELETE FROM ITEMTYPE WHERE NAME='Muszter DEMO' +@ diff --git a/server/user.jobengine.osgi.db/migrations/scripts/029_alter_PASA_storeuri_to_FTP.sql b/server/user.jobengine.osgi.db/migrations/scripts/029_alter_PASA_storeuri_to_FTP.sql new file mode 100644 index 00000000..6d51e9c4 --- /dev/null +++ b/server/user.jobengine.osgi.db/migrations/scripts/029_alter_PASA_storeuri_to_FTP.sql @@ -0,0 +1,52 @@ +-- +-- Copyright 2010-2016 the original author or authors. +-- +-- Licensed under the Apache License, Version 2.0 (the "License"); +-- you may not use this file except in compliance with the License. +-- You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, software +-- distributed under the License is distributed on an "AS IS" BASIS, +-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +-- See the License for the specific language governing permissions and +-- limitations under the License. +-- + +-- // Alter PASA STOREURI change to FTP protocol +-- Migration SQL that makes the change goes here. + +update storeuri set protocol='FTP', URI='10.170.100.21', username='10.170.100.21\pbadmin', password='Txhkkmot52', rootpath='MEDIA_FS/PasaPool' where storeid in (select id from store where name = 'PASAPOOL'); +@ +update storeuri set protocol='FTP', URI='10.170.100.21', username='10.170.100.21\pbadmin', password='Txhkkmot52', rootpath='MEDIA_FS/BeachPool' where storeid in (select id from store where name = 'PEABLEBEACH'); +@ +update storeuri set protocol='FTP', URI='10.170.100.21', username='10.170.100.21\pbadmin', password='Txhkkmot52', rootpath='MEDIA_FS/SelenioPool' where storeid in (select id from store where name = 'SELENIOPOOL'); +@ +update storeuri set protocol='FTP', URI='10.170.100.21', username='10.170.100.21\pbadmin', password='Txhkkmot52', rootpath='MEDIA_FS/SelenioPool/16_9' where storeid in (select id from store where name = 'SELENIOPOOL_16_9'); +@ +update storeuri set protocol='FTP', URI='10.170.100.21', username='10.170.100.21\pbadmin', password='Txhkkmot52', rootpath='MEDIA_FS/SelenioPool/16_9/Processiong' where storeid in (select id from store where name = 'SELENIOPOOL_16_9_PROCESSING'); +@ +update storeuri set protocol='FTP', URI='10.170.100.21', username='10.170.100.21\pbadmin', password='Txhkkmot52', rootpath='MEDIA_FS/SelenioPool/4_3' where storeid in (select id from store where name = 'SELENIOPOOL_4_3'); +@ +update storeuri set protocol='FTP', URI='10.170.100.21', username='10.170.100.21\pbadmin', password='Txhkkmot52', rootpath='MEDIA_FS/SelenioPool/4_3/Processiong' where storeid in (select id from store where name = 'SELENIOPOOL_4_3_PROCESSING'); +@ + +-- //@UNDO +-- SQL to undo the change goes here. + +update storeuri set protocol='LOCAL', URI='/mnt/PEABLEBEACH/PASAPOOL', username=null, password=null, rootpath=null where storeid in (select id from store where name = 'PASAPOOL'); +@ +update storeuri set protocol='LOCAL', URI='/mnt/PEABLEBEACH/BEACHPOOL',username=null, password=null, rootpath=null where storeid in (select id from store where name = 'PEABLEBEACH'); +@ +update storeuri set protocol='LOCAL', URI='/mnt/PEABLEBEACH/SELENIOPOOL', username=null, password=null, rootpath=null where storeid in (select id from store where name = 'SELENIOPOOL'); +@ +update storeuri set protocol='LOCAL', URI='/mnt/PEABLEBEACH/SELENIOPOOL/16_9', username=null, password=null, rootpath=null where storeid in (select id from store where name = 'SELENIOPOOL_16_9'); +@ +update storeuri set protocol='LOCAL', URI='/mnt/PEABLEBEACH/SELENIOPOOL/16_9/PROCESSING', username=null, password=null, rootpath=null where storeid in (select id from store where name = 'SELENIOPOOL_16_9_PROCESSING'); +@ +update storeuri set protocol='LOCAL', URI='/mnt/PEABLEBEACH/SELENIOPOOL/4_3', username=null, password=null, rootpath=null where storeid in (select id from store where name = 'SELENIOPOOL_4_3'); +@ +update storeuri set protocol='LOCAL', URI='/mnt/PEABLEBEACH/SELENIOPOOL/4_3/PROCESSING', username=null, password=null, rootpath=null where storeid in (select id from store where name = 'SELENIOPOOL_4_3_PROCESSING'); +@ + diff --git a/server/user.jobengine.osgi.db/migrations/scripts/030_insert_itemtype_musor_clean.sql b/server/user.jobengine.osgi.db/migrations/scripts/030_insert_itemtype_musor_clean.sql new file mode 100644 index 00000000..4ddc872e --- /dev/null +++ b/server/user.jobengine.osgi.db/migrations/scripts/030_insert_itemtype_musor_clean.sql @@ -0,0 +1,11 @@ +-- // Insert Musor CLEAN itemtype +-- Migration SQL that makes the change goes here. + +INSERT INTO ITEMTYPE (NAME, DESCRIPTION, ISSTATIC) VALUES ('Műsor CLEAN', 'Műsor CLEAN', 'N') +@ + +-- //@UNDO +-- SQL to undo the change goes here. + +DELETE FROM ITEMTYPE WHERE NAME='Műsor CLEAN' +@ \ No newline at end of file diff --git a/server/user.jobengine.osgi.db/pom.xml b/server/user.jobengine.osgi.db/pom.xml index d59dcf0e..ea2ba4e7 100644 --- a/server/user.jobengine.osgi.db/pom.xml +++ b/server/user.jobengine.osgi.db/pom.xml @@ -12,4 +12,36 @@ user.jobengine.osgi.db eclipse-plugin 1.0.0 + + + + + + org.apache.maven.plugins + maven-resources-plugin + 2.7 + + + copy-jar + install + + copy-resources + + + ${project.basedir}}/../-product/target/deploy + + + target + + ${project.artifactId}_${project.version}.jar + + + + + + + + + + \ No newline at end of file diff --git a/server/user.jobengine.osgi.server/pom.xml b/server/user.jobengine.osgi.server/pom.xml index 77c5d836..edfb0de3 100644 --- a/server/user.jobengine.osgi.server/pom.xml +++ b/server/user.jobengine.osgi.server/pom.xml @@ -48,4 +48,36 @@ + + + + + + org.apache.maven.plugins + maven-resources-plugin + 2.7 + + + copy-jar + install + + copy-resources + + + ${project.basedir}}/../-product/target/deploy + + + target + + ${project.artifactId}_${project.version}.jar + + + + + + + + + + \ No newline at end of file diff --git a/server/user.jobengine.osgi.server/src/user/jobengine/server/IJobEngine.java b/server/user.jobengine.osgi.server/src/user/jobengine/server/IJobEngine.java index 3f775fab..d6bb6057 100644 --- a/server/user.jobengine.osgi.server/src/user/jobengine/server/IJobEngine.java +++ b/server/user.jobengine.osgi.server/src/user/jobengine/server/IJobEngine.java @@ -112,6 +112,8 @@ public interface IJobEngine { void putOutputsToStack(IJobRuntime jobRuntime, Object[] outputs); + void reloadGracefully() throws Exception; + void removeFromExecutorQueue(IJobRuntime jobRuntime); void removeFromRunQueue(IJobRuntime jobRuntime); diff --git a/server/user.jobengine.osgi.server/src/user/jobengine/server/IJobStepExecutor.java b/server/user.jobengine.osgi.server/src/user/jobengine/server/IJobStepExecutor.java index 7ad2abb1..02dc8dd9 100644 --- a/server/user.jobengine.osgi.server/src/user/jobengine/server/IJobStepExecutor.java +++ b/server/user.jobengine.osgi.server/src/user/jobengine/server/IJobStepExecutor.java @@ -36,6 +36,10 @@ public interface IJobStepExecutor { void revoke(IJobRuntime jobRuntime); + void setJobEngine(IJobEngine jobEngine); + + void setMaxConcurrent(int maxConcurrent); + /** * V�grehajt� le�ll�t�sa. Minden sz�l le�ll�t�sra ker�l. */ @@ -44,7 +48,7 @@ public interface IJobStepExecutor { /** * A v�grehajt� elind�t�sa. */ - void startup(IJobEngine jobEngine); + void startup(); ClusteredJob steelJob() throws InterruptedException; diff --git a/server/user.jobengine.osgi.server/src/user/jobengine/server/JobEngine.java b/server/user.jobengine.osgi.server/src/user/jobengine/server/JobEngine.java index 0a05285e..0a46bbf6 100644 --- a/server/user.jobengine.osgi.server/src/user/jobengine/server/JobEngine.java +++ b/server/user.jobengine.osgi.server/src/user/jobengine/server/JobEngine.java @@ -283,12 +283,21 @@ public class JobEngine implements IJobEngine { @Override public void addStepExecutor(IJobStepExecutor executor) { //Class stepClass = executor.getStepClass(); + executor.setJobEngine(this); String unitName = executor.getStepUnitName(); if (!executors.containsKey(unitName)) { - logger.info("Executor registered {}", unitName); + logger.info("Executor now registered for {}", unitName); executors.put(unitName, executor); - } else - logger.debug("Executor already registered {}", unitName); + } else { + logger.info("Executor already registered for {}", unitName); + IJobStepExecutor stepExecutor = executors.get(unitName); + int currentMaxConcurrent = stepExecutor.getMaxConcurrent(); + int newMaxConcurrent = executor.getMaxConcurrent(); + if (currentMaxConcurrent != newMaxConcurrent) { + stepExecutor.setMaxConcurrent(newMaxConcurrent); + logger.info("Executor maxConcurrent changed from {} to {}", currentMaxConcurrent, newMaxConcurrent); + } + } } @@ -667,7 +676,7 @@ public class JobEngine implements IJobEngine { public void loadExecutors() { // TODO // shutdownExecutors(); - executors.clear(); + //executors.clear(); InputStream stream = null; try { String stepRoot = DirectoryUtils.normalize(System.getProperty(STEPSROOT), File.separator); @@ -884,6 +893,22 @@ public class JobEngine implements IJobEngine { } } + @Override + public void reloadGracefully() throws Exception { + if (schedulerService != null) + schedulerService.shutdown(); + + loadPrograms(); + loadExecutors(); + + //startupExecutors(); + + schedulerService = new SchedulerService(this); + schedulerService.startup(); + logger.info("JobEngine gracefully reloaded"); + isRunning = true; + } + @Override public void removeFromExecutorQueue(IJobRuntime jobRuntime) { for (IJobStepExecutor executor : executors.values()) @@ -1107,7 +1132,7 @@ public class JobEngine implements IJobEngine { private void startupExecutors() { for (IJobStepExecutor executor : executors.values()) - executor.startup(this); + executor.startup(); } @Override diff --git a/server/user.jobengine.osgi.server/src/user/jobengine/server/JobStepExecutor.java b/server/user.jobengine.osgi.server/src/user/jobengine/server/JobStepExecutor.java index a6be04f5..d2dd3a50 100644 --- a/server/user.jobengine.osgi.server/src/user/jobengine/server/JobStepExecutor.java +++ b/server/user.jobengine.osgi.server/src/user/jobengine/server/JobStepExecutor.java @@ -6,6 +6,7 @@ import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.Paths; import java.util.ArrayList; +import java.util.Collections; import java.util.List; import java.util.concurrent.CountDownLatch; import java.util.concurrent.PriorityBlockingQueue; @@ -41,6 +42,9 @@ public class JobStepExecutor implements IJobStepExecutor { try { Thread.sleep(IJobEngine.QUEUE_POLL_INTERVAL_MS); + if (jobEngine == null) + logger.info("JobEngine is null"); + if (jobEngine.isWorker() && isRemote) { //a worker is csak azokat akarja vegrehajtani ClusteredJob job = jobEngine.getRemoteEngine().getRemoteJob(getStepUnitName()); @@ -76,12 +80,11 @@ public class JobStepExecutor implements IJobStepExecutor { } if (jobRuntime != null) { - long submitted = jobRuntime.getSubmitted().getTime(); - long current = System.currentTimeMillis(); - boolean timeout = current - submitted > WAIT_FOR_REMOTE; - //ha remote, de nem jelentkezik senki, akkor helyi vegrehajtas if (isRemote) { + long submitted = jobRuntime.getSubmitted().getTime(); + long current = System.currentTimeMillis(); + boolean timeout = current - submitted > WAIT_FOR_REMOTE; if (timeout) { logger.info("Remote JobStep timed out, processing locally."); } else { @@ -95,7 +98,11 @@ public class JobStepExecutor implements IJobStepExecutor { logger.info("Executing locally {}", jobRuntime.getId()); //jobRuntime.setDescription(PROCESSING_LOCALLY); Object[] inputs = jobEngine.getInputsFromStack(jobRuntime); - runStepObject(jobRuntime, inputs); + IJobStep stepObject = runStepObject(jobRuntime, inputs); + if (stepObject.isTest()) { + logger.trace("Finishing test worker"); + break; + } } if (shutdown) { @@ -117,15 +124,22 @@ public class JobStepExecutor implements IJobStepExecutor { jobRuntime = null; step = null; } + if (retireUnusedWorker(this)) { + break; + + } + } } } - private void runStepObject(IJobRuntime jobRuntime, Object[] inputs) throws Throwable { + private IJobStep runStepObject(IJobRuntime jobRuntime, Object[] inputs) throws Throwable { + IJobStep step = null; + jobRuntime.setStatus(JobStatus.EXECUTING); jobRuntime.NotifyUpdate(); - IJobStep step = createStepObject(); + step = createStepObject(); if (step == null) throw new Exception("Step object is null"); logger.debug("{} executing", jobRuntime); @@ -139,6 +153,7 @@ public class JobStepExecutor implements IJobStepExecutor { // jobEngine.sendMessage(new JobStepCompletedMessage(jobRuntime.getId(), outputs)); // } jobEngine.sendMessage(new JobStepCompletedMessage(jobRuntime.getId(), outputs)); + return step; } public void shutdown() { @@ -156,7 +171,7 @@ public class JobStepExecutor implements IJobStepExecutor { private Logger logger; private PriorityBlockingQueue queue; - private List workers; + private List workers = Collections.synchronizedList(new ArrayList()); protected IJobEngine jobEngine; private CountDownLatch barrier; private Class stepClass; @@ -180,6 +195,21 @@ public class JobStepExecutor implements IJobStepExecutor { create(className, maxConcurrent, false); } + private void addWorkers(int count) { + barrier = new CountDownLatch(count); + + for (int index = 0; index < count; index++) { + Worker worker = new Worker(); + worker.start(); + workers.add(worker); + } + + try { + barrier.await(); + } catch (Exception e) { + } + } + @Override public void changePriority(IJobRuntime runtime) { if (queue != null && runtime != null) { @@ -220,12 +250,6 @@ public class JobStepExecutor implements IJobStepExecutor { queue = new PriorityBlockingQueue(); this.maxConcurrent = maxConcurrent; - if (maxConcurrent > 0) { - barrier = new CountDownLatch(maxConcurrent); - workers = new ArrayList(); - for (int index = 0; index < maxConcurrent; index++) - workers.add(new Worker()); - } } protected IJobStep createStepObject() throws Exception { @@ -259,7 +283,7 @@ public class JobStepExecutor implements IJobStepExecutor { } @Override - public int getMaxConcurrent() { + public synchronized int getMaxConcurrent() { return maxConcurrent; } @@ -312,31 +336,55 @@ public class JobStepExecutor implements IJobStepExecutor { return result; } + protected synchronized boolean retireUnusedWorker(Worker worker) { + boolean result = false; + if (getMaxConcurrent() < workers.size()) { + workers.remove(worker); + logger.info("Worker {} is retired, max {} current {}", getStepUnitName(), getMaxConcurrent(), workers.size()); + result = true; + } + + return result; + } + @Override public void revoke(IJobRuntime jobRuntime) { queue.remove(jobRuntime); } @Override - public void shutdown() { - for (Worker w : workers) - w.shutdown(); + public void setJobEngine(IJobEngine jobEngine) { + this.jobEngine = jobEngine; } @Override - public void startup(IJobEngine jobEngine) { - this.jobEngine = jobEngine; - if (workers != null) { - for (Worker w : workers) - w.start(); - } - if (barrier != null) { - try { - barrier.await(); - } catch (Exception e) { - } - //barrier.reset(); + public synchronized void setMaxConcurrent(int maxConcurrent) { + if (this.maxConcurrent == maxConcurrent) + return; + + if (this.maxConcurrent < maxConcurrent) { + int diff = maxConcurrent - this.maxConcurrent; + addWorkers(diff); } + + this.maxConcurrent = maxConcurrent; + } + + @Override + public void shutdown() { + if (workers == null) + return; + + workers.forEach(worker -> worker.shutdown()); + workers.clear(); + } + + @Override + public void startup() { + if (maxConcurrent == 0) + return; + + addWorkers(maxConcurrent); } @Override @@ -368,7 +416,6 @@ public class JobStepExecutor implements IJobStepExecutor { @Override public void waitShutdown() { - for (Worker w : workers) - w.waitShutdown(); + workers.forEach(worker -> worker.waitShutdown()); } } diff --git a/server/user.jobengine.osgi.server/src/user/jobengine/server/steps/IJobStep.java b/server/user.jobengine.osgi.server/src/user/jobengine/server/steps/IJobStep.java index e84eb5de..f3c8fde3 100644 --- a/server/user.jobengine.osgi.server/src/user/jobengine/server/steps/IJobStep.java +++ b/server/user.jobengine.osgi.server/src/user/jobengine/server/steps/IJobStep.java @@ -9,6 +9,7 @@ public interface IJobStep { void cleanup(); - Object[] run(IJobEngine jobEngine, IJobRuntime jobRuntime, Object[] inputs) throws Throwable; + boolean isTest(); + Object[] run(IJobEngine jobEngine, IJobRuntime jobRuntime, Object[] inputs) throws Throwable; } diff --git a/server/user.jobengine.osgi.server/src/user/jobengine/server/steps/JobStep.java b/server/user.jobengine.osgi.server/src/user/jobengine/server/steps/JobStep.java index 42de2048..79f19de7 100644 --- a/server/user.jobengine.osgi.server/src/user/jobengine/server/steps/JobStep.java +++ b/server/user.jobengine.osgi.server/src/user/jobengine/server/steps/JobStep.java @@ -143,6 +143,11 @@ public class JobStep implements IJobStep { return (MediaCubeMarker) jobRuntime.getSessionMarker(); } + @Override + public boolean isTest() { + return false; + } + @Override public Object[] run(IJobEngine jobEngine, IJobRuntime jobRuntime, Object[] inputs) throws Throwable { this.jobEngine = jobEngine; diff --git a/server/user.jobengine.osgi.server/test/log4j2-test.xml b/server/user.jobengine.osgi.server/test/log4j2-test.xml new file mode 100644 index 00000000..ad428dc7 --- /dev/null +++ b/server/user.jobengine.osgi.server/test/log4j2-test.xml @@ -0,0 +1,13 @@ + + + + + + + + + + + + + \ No newline at end of file diff --git a/server/user.jobengine.osgi.server/test/user/jobengine/server/JobStepExecutorTest.java b/server/user.jobengine.osgi.server/test/user/jobengine/server/JobStepExecutorTest.java index 19efc467..6f3441e8 100644 --- a/server/user.jobengine.osgi.server/test/user/jobengine/server/JobStepExecutorTest.java +++ b/server/user.jobengine.osgi.server/test/user/jobengine/server/JobStepExecutorTest.java @@ -37,20 +37,25 @@ public class JobStepExecutorTest { }; jobStep = new IJobStep() { @Override - public Object[] run(IJobEngine jobEngine, IJobRuntime jobRuntime, Object[] inputs) throws Exception { - called[0] = true; - if (threadCounter != null) - threadCounter.increment(); - return null; + public boolean canContinue() { + return false; } @Override - public boolean canContinue() { + public void cleanup() { + } + + @Override + public boolean isTest() { return false; } @Override - public void cleanup() { + public Object[] run(IJobEngine jobEngine, IJobRuntime jobRuntime, Object[] inputs) throws Exception { + called[0] = true; + if (threadCounter != null) + threadCounter.increment(); + return null; } }; @@ -62,7 +67,7 @@ public class JobStepExecutorTest { } }; - sut.startup(jobEngine); + sut.startup(); jobRuntime1 = new JobRuntime(jobEngine, new Program()); jobRuntime1.pushToStack("var1"); @@ -81,40 +86,15 @@ public class JobStepExecutorTest { } @Test - public void testSendJobCompletedMessage() { + public void testConstructor_Dynamic_ExistingClass() throws Exception { // Fixture - messageCounter = new ThreadCounter(2); + System.setProperty("jobengine.jobsteps.root", "c:\\$Work\\USER\\JobEngine\\plugins"); // Exercise - sut.submit(jobRuntime1); - sut.submit(jobRuntime2); + sut = new JobStepExecutor("user.jobengine.server.steps.JobStepDyn", 1); // Verify - messageCounter.waitFinish(); - assertEquals(2, messages.size()); - assertEquals(JobStepCompletedMessage.class, messages.get(0).getClass()); - assertEquals(1, messages.get(0).getJobId()); - assertEquals(JobStepCompletedMessage.class, messages.get(1).getClass()); - assertEquals(2, messages.get(1).getJobId()); - } - - @Test - public void testRunStep() { - // Fixture - threadCounter = new ThreadCounter(1); - - // Exercise - sut.submit(jobRuntime1); - - // Validate - threadCounter.waitFinish(); - assertTrue(called[0]); - } - - @Test(expected = NullPointerException.class) - public void testSubmit_NullParameter() { - // Exercise - sut.submit(null); + assertTrue(sut.getStepClass() != null); } @Test(expected = RuntimeException.class) @@ -122,18 +102,6 @@ public class JobStepExecutorTest { new JobStepExecutor("user.jobengine.server.steps.JobStepDyn", 1); } - @Test - public void testConstructor_Dynamic_ExistingClass() throws Exception { - // Fixture - System.setProperty("jobengine.jobsteps.root", "c:\\$Work\\USER\\JobEngine\\plugins"); - - // Exercise - sut = new JobStepExecutor("user.jobengine.server.steps.JobStepDyn", 1); - - // Verify - assertTrue(sut.getStepClass() != null); - } - @Test public void testDynamicExecutor() throws Exception { // Fixture @@ -145,7 +113,7 @@ public class JobStepExecutorTest { jobRuntime1.pushToStack(2); System.setProperty("jobengine.jobsteps.root", "c:\\$Work\\USER\\JobEngine\\plugins"); sut = new JobStepExecutor("user.jobengine.server.steps.JobStepDyn", 1); - sut.startup(jobEngine); + sut.startup(); messageCounter = new ThreadCounter(1); // Exercise @@ -159,4 +127,41 @@ public class JobStepExecutorTest { //check stack } + @Test + public void testRunStep() { + // Fixture + threadCounter = new ThreadCounter(1); + + // Exercise + sut.submit(jobRuntime1); + + // Validate + threadCounter.waitFinish(); + assertTrue(called[0]); + } + + @Test + public void testSendJobCompletedMessage() { + // Fixture + messageCounter = new ThreadCounter(2); + + // Exercise + sut.submit(jobRuntime1); + sut.submit(jobRuntime2); + + // Verify + messageCounter.waitFinish(); + assertEquals(2, messages.size()); + assertEquals(JobStepCompletedMessage.class, messages.get(0).getClass()); + assertEquals(1, messages.get(0).getJobId()); + assertEquals(JobStepCompletedMessage.class, messages.get(1).getClass()); + assertEquals(2, messages.get(1).getJobId()); + } + + @Test(expected = NullPointerException.class) + public void testSubmit_NullParameter() { + // Exercise + sut.submit(null); + } + } diff --git a/server/user.jobengine.osgi.server/test/user/jobengine/server/JobStepExecutorTest1.java b/server/user.jobengine.osgi.server/test/user/jobengine/server/JobStepExecutorTest1.java new file mode 100644 index 00000000..b5aea6dd --- /dev/null +++ b/server/user.jobengine.osgi.server/test/user/jobengine/server/JobStepExecutorTest1.java @@ -0,0 +1,142 @@ +package user.jobengine.server; + +import static org.junit.Assert.assertEquals; + +import java.sql.Timestamp; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.atomic.AtomicInteger; + +import org.junit.After; +import org.junit.Before; +import org.junit.BeforeClass; +import org.junit.Test; + +import user.jobengine.server.steps.IJobStep; + +public class JobStepExecutorTest1 { + @BeforeClass + public static void beforeClass() { + //System.setProperty("org.apache.logging.log4j.simplelog.StatusLogger.level", "TRACE"); + } + + @Before + public void setup() throws Exception { + } + + @After + public void tearDown() throws Exception { + } + + @Test + public void testTicketBase() throws Exception { + // Fixture + int expectedJobCount = 3; + IJobEngine jobEngine = new JobEngine(); + final AtomicInteger stepCallCount = new AtomicInteger(); + final CountDownLatch stepStartBarrier = new CountDownLatch(expectedJobCount); + + final IJobStep jobStep = new IJobStep() { + @Override + public boolean canContinue() { + return true; + } + + @Override + public void cleanup() { + } + + @Override + public boolean isTest() { + return true; + } + + @Override + public Object[] run(IJobEngine jobEngine, IJobRuntime jobRuntime, Object[] inputs) throws Exception { + stepCallCount.getAndIncrement(); + stepStartBarrier.countDown(); + return null; + } + }; + + final IJobStepExecutor sut = new JobStepExecutor(IJobStep.class, expectedJobCount) { + @Override + protected IJobStep createStepObject() throws InstantiationException, IllegalAccessException { + return jobStep; + } + + }; + sut.setJobEngine(jobEngine); + sut.startup(); + + // Exercise + for (int i = 0; i < expectedJobCount; i++) { + JobRuntime jobRuntime = new JobRuntime(jobEngine, new Program()); + jobRuntime.setSubmitted(new Timestamp(System.currentTimeMillis())); + jobRuntime.pushToStack(0); + jobRuntime.saveStack(); + sut.submit(jobRuntime); + } + stepStartBarrier.await(); + // Verify + assertEquals(expectedJobCount, stepCallCount.get()); + } + + @Test + public void testTicketIncrement() throws Exception { + // Fixture + int expectedJobCount = 3; + IJobEngine jobEngine = new JobEngine(); + final int[] stepCallCount = { 0 }; + final CountDownLatch stepStartBarrier = new CountDownLatch(expectedJobCount); + final IJobStep jobStep = new IJobStep() { + @Override + public boolean canContinue() { + return true; + } + + @Override + public void cleanup() { + } + + @Override + public boolean isTest() { + return true; + } + + @Override + public Object[] run(IJobEngine jobEngine, IJobRuntime jobRuntime, Object[] inputs) throws Exception { + + stepCallCount[0]++; + stepStartBarrier.countDown(); + + return null; + } + + }; + + final IJobStepExecutor sut = new JobStepExecutor(IJobStep.class, 1) { + @Override + protected IJobStep createStepObject() throws InstantiationException, IllegalAccessException { + return jobStep; + } + + }; + sut.startup(); + + // Exercise + + sut.setMaxConcurrent(sut.getMaxConcurrent() + 1); + + for (int i = 0; i < expectedJobCount; i++) { + JobRuntime jobRuntime = new JobRuntime(jobEngine, new Program()); + jobRuntime.setSubmitted(new Timestamp(System.currentTimeMillis())); + jobRuntime.pushToStack(0); + sut.submit(jobRuntime); + } + stepStartBarrier.await(); + + // Verify + assertEquals(expectedJobCount, stepCallCount[0]); + } + +} diff --git a/server/user.jobengine.osgi.services/pom.xml b/server/user.jobengine.osgi.services/pom.xml index 787546d8..d0b367b5 100644 --- a/server/user.jobengine.osgi.services/pom.xml +++ b/server/user.jobengine.osgi.services/pom.xml @@ -44,29 +44,60 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + - org.codehaus.mojo - exec-maven-plugin - 1.5.0 + org.apache.maven.plugins + maven-resources-plugin + 2.7 - install + copy-jar install - exec + copy-resources - echo - - yyy - + ${project.basedir}}/../-product/target/deploy + + + target + + ${project.artifactId}_${project.version}.jar + + + - diff --git a/server/user.mediacube.gui/pages/joblist.zul b/server/user.mediacube.gui/pages/joblist.zul index 480bbcb4..900880a4 100644 --- a/server/user.mediacube.gui/pages/joblist.zul +++ b/server/user.mediacube.gui/pages/joblist.zul @@ -30,6 +30,7 @@ + diff --git a/server/user.mediacube.gui/pom.xml b/server/user.mediacube.gui/pom.xml index 73ceef6c..4cef5f7d 100644 --- a/server/user.mediacube.gui/pom.xml +++ b/server/user.mediacube.gui/pom.xml @@ -11,4 +11,37 @@ user.mediacube.gui eclipse-plugin 1.0.0 - \ No newline at end of file + + + + + + org.apache.maven.plugins + maven-resources-plugin + 2.7 + + + copy-jar + install + + copy-resources + + + ${project.basedir}}/../-product/target/deploy + + + target + + ${project.artifactId}_${project.version}.jar + + + + + + + + + + + + \ No newline at end of file diff --git a/server/user.mediacube.gui/resources/i3-label_hu.properties b/server/user.mediacube.gui/resources/i3-label_hu.properties index b0e183c6..e54983be 100644 --- a/server/user.mediacube.gui/resources/i3-label_hu.properties +++ b/server/user.mediacube.gui/resources/i3-label_hu.properties @@ -1,4 +1,4 @@ -version=2.7.0 +version=2.7.1 footer=2016-2020 © Copyright User Rendszerház Kft. login_info=Információ diff --git a/server/user.mediacube.gui/src/user/jobengine/zk/model/FileSizeConverter.java b/server/user.mediacube.gui/src/user/jobengine/zk/model/FileSizeConverter.java index 09183ca2..97685bfd 100644 --- a/server/user.mediacube.gui/src/user/jobengine/zk/model/FileSizeConverter.java +++ b/server/user.mediacube.gui/src/user/jobengine/zk/model/FileSizeConverter.java @@ -1,20 +1,21 @@ package user.jobengine.zk.model; -import org.apache.commons.io.FileUtils; import org.zkoss.bind.BindContext; import org.zkoss.bind.Converter; import org.zkoss.zul.Label; -public class FileSizeConverter implements Converter { +import user.commons.strings.FileSizeUtils; - @Override - public String coerceToUi(Long fileSize, Label paramC, BindContext context) { - return FileUtils.byteCountToDisplaySize(fileSize); - } +public class FileSizeConverter implements Converter { @Override public Long coerceToBean(String paramU, Label paramC, BindContext paramBindContext) { // TODO Auto-generated method stub return null; } + + @Override + public String coerceToUi(Long fileSize, Label paramC, BindContext context) { + return FileSizeUtils.sizeAsString(fileSize); + } } diff --git a/server/user.mediacube.gui/src/user/jobengine/zk/model/JobListModel.java b/server/user.mediacube.gui/src/user/jobengine/zk/model/JobListModel.java index d45e5ccf..ef49696f 100644 --- a/server/user.mediacube.gui/src/user/jobengine/zk/model/JobListModel.java +++ b/server/user.mediacube.gui/src/user/jobengine/zk/model/JobListModel.java @@ -333,6 +333,15 @@ public class JobListModel extends AsyncBaseModel implements IJobChangedListener t.start(); } + @Command + public void reload() { + try { + jobEngine.reloadGracefully(); + } catch (Exception e) { + Messagebox.show(e.getMessage(), "Hiba!", Messagebox.OK, Messagebox.ERROR); + } + } + @Command @NotifyChange("searchStatus") public void resetSearchRelated() { diff --git a/server/user.mediacube.metadata/pom.xml b/server/user.mediacube.metadata/pom.xml index e8c132cf..bf09ee71 100644 --- a/server/user.mediacube.metadata/pom.xml +++ b/server/user.mediacube.metadata/pom.xml @@ -11,5 +11,35 @@ user.mediacube.metadata eclipse-plugin 1.0.0 + + + + + org.apache.maven.plugins + maven-resources-plugin + 2.7 + + + copy-jar + install + + copy-resources + + + ${project.basedir}}/../-product/target/deploy + + + target + + ${project.artifactId}_${project.version}.jar + + + + + + + + + \ No newline at end of file diff --git a/server/user.tsm.client/pom.xml b/server/user.tsm.client/pom.xml index e81dac19..e228ea6d 100644 --- a/server/user.tsm.client/pom.xml +++ b/server/user.tsm.client/pom.xml @@ -12,4 +12,35 @@ user.tsm.client eclipse-plugin 1.2.0 + + + + + org.apache.maven.plugins + maven-resources-plugin + 2.7 + + + copy-jar + install + + copy-resources + + + ${project.basedir}}/../-product/target/deploy + + + target + + ${project.artifactId}_${project.version}.jar + + + + + + + + + + \ No newline at end of file