From: vasary.daniel Date: Fri, 10 Sep 2021 15:41:39 +0000 (+0000) Subject: git-tfs-id: [http://tfs.userrendszerhaz.hu:8080/tfs/DefaultCollection]$/MediaCube... X-Git-Url: http://git.useribm.hu/?a=commitdiff_plain;h=c9d899acda9c06f2e8ed1e8782d7c2a6fd2ad61b;p=mediacube.git git-tfs-id: [tfs.userrendszerhaz.hu:8080/tfs/DefaultCollection]$/MediaCube;C32333 --- diff --git a/server/-product/production/HIRTV/jobs/executors.xml b/server/-product/production/HIRTV/jobs/executors.xml index 8cd7c04c..22c80c64 100644 --- a/server/-product/production/HIRTV/jobs/executors.xml +++ b/server/-product/production/HIRTV/jobs/executors.xml @@ -1,19 +1,21 @@ + + + + + - - - @@ -31,7 +33,7 @@ - + diff --git a/server/-product/production/HIRTV/jobs/schedules.json b/server/-product/production/HIRTV/jobs/schedules.json index 825fd64e..f6428f7e 100644 --- a/server/-product/production/HIRTV/jobs/schedules.json +++ b/server/-product/production/HIRTV/jobs/schedules.json @@ -1,21 +1,56 @@ { - "joblist": [{ - "active": true, - "cronexpression": "0/2 * * * * ?", - "template": "cancelable.xml" - }, - { - "active": true, - "executeimmediate": true, - "name" : "OCTOPUS adatok szinkronizálása", - "template": "sync-octopus.xml", - "cronexpression": "0/30 * * * * ?", - "parameters": [ - {"name": "includeArchived", "value": false, "type": "java.lang.Boolean"}, - {"name": "address", "value": "http://10.10.1.11/api/v1", "type": "java.lang.String"}, - {"name": "user", "value": "mam", "type": "java.lang.String"}, - {"name": "pwd", "value": "napocska", "type": "java.lang.String"} - ] - } - ] -} \ No newline at end of file + "joblist": [ + { + "active": false, + "cronexpression": "0/2 * * * * ?", + "template": "cancelable.xml", + "parameters": [ + { + "name": "param", + "value": 100, + "type": "java.lang.Integer" + } + ] + }, + { + "active": false, + "template": "test-fork-cancelable.xml", + "parameters": [ + { + "name": "count", + "value": 2, + "type": "java.lang.Integer" + } + ] + }, + { + "active": true, + "executeimmediate": true, + "name": "OCTOPUS adatok szinkronizálása", + "template": "sync-octopus.xml", + "cronexpression": "0/30 * * * * ?", + "parameters": [ + { + "name": "includeArchived", + "value": false, + "type": "java.lang.Boolean" + }, + { + "name": "address", + "value": "http://10.10.1.11/api/v1", + "type": "java.lang.String" + }, + { + "name": "user", + "value": "mam", + "type": "java.lang.String" + }, + { + "name": "pwd", + "value": "napocska", + "type": "java.lang.String" + } + ] + } + ] +} diff --git a/server/-product/production/HIRTV/jobs/steps/CancelableStep.java b/server/-product/production/HIRTV/jobs/steps/CancelableStep.java new file mode 100644 index 00000000..3368e445 --- /dev/null +++ b/server/-product/production/HIRTV/jobs/steps/CancelableStep.java @@ -0,0 +1,22 @@ +package user.jobengine.server.steps; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + +public class CancelableStep extends JobStep { + private static final Logger logger = LogManager.getLogger(); + + @StepEntry + public Object[] execute(int param) throws Exception { + logger.info(getMarker(), "Executing with param {}", param); + for (int i = 0; i < 10; i++) { + if (getJobRuntime().isWaitingCancel()) + break; + Thread.sleep(1000); + int progress = (i + 1) * 100 / 10; + setProgress(progress); + } + return null; + } + +} diff --git a/server/-product/production/HIRTV/jobs/steps/TestForkCancelableStep.java b/server/-product/production/HIRTV/jobs/steps/TestForkCancelableStep.java new file mode 100644 index 00000000..8083d94f --- /dev/null +++ b/server/-product/production/HIRTV/jobs/steps/TestForkCancelableStep.java @@ -0,0 +1,41 @@ +package user.jobengine.server.steps; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + +import user.commons.IJob; +import user.commons.JobStatus; +import user.commons.ListUtils; +import user.jobengine.server.IJobEngine; +import user.jobengine.server.IJobRuntime; + +public class TestForkCancelableStep extends JobStep { + private static final String CHILD_TEMPLATE = "cancelable.xml"; + private static final Logger logger = LogManager.getLogger(); + + @StepEntry + public Object[] execute(int count) throws Exception { + getJobRuntime().forkPrepare(); + for (int i = 0; i < count; i++) { + if (getJobRuntime().isWaitingCancel()) + break; + IJobRuntime runtime = getEngine().submit(getJobRuntime(), e -> { + IJobRuntime r = (IJobRuntime) e.getSource(); + + if (r.isDisableRetry()) + return; + + if (e.getStatus().equals(JobStatus.CANCELED) || e.getStatus().equals(JobStatus.SUSPENDED)) { + long id = ((IJob) e.getSource()).getId(); + logger.info("Cleanup occured because status is {} {}", e.getStatus(), id); + } + + }, CHILD_TEMPLATE, "JOB " + i, 0, IJobEngine.DEFAULT_OWNER, ListUtils.asMap("param", i)); + + runtime.setRelated("TEST" + runtime.getId()); + } + + getJobRuntime().forkWaitComplete(); + return null; + } +} diff --git a/server/-product/production/HIRTV/jobs/templates/cancelable.xml b/server/-product/production/HIRTV/jobs/templates/cancelable.xml index 4f3b9f86..89b393f2 100644 --- a/server/-product/production/HIRTV/jobs/templates/cancelable.xml +++ b/server/-product/production/HIRTV/jobs/templates/cancelable.xml @@ -1,6 +1,17 @@ - + + + + + + - + + + + + + + \ No newline at end of file diff --git a/server/-product/production/HIRTV/jobs/templates/test-fork-cancelable.xml b/server/-product/production/HIRTV/jobs/templates/test-fork-cancelable.xml new file mode 100644 index 00000000..be028dea --- /dev/null +++ b/server/-product/production/HIRTV/jobs/templates/test-fork-cancelable.xml @@ -0,0 +1,17 @@ + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/server/user.jobengine.executors/src/user/jobengine/server/steps/CancelableStep.java b/server/user.jobengine.executors/src/user/jobengine/server/steps/CancelableStep.java index 164e552c..b830e155 100644 --- a/server/user.jobengine.executors/src/user/jobengine/server/steps/CancelableStep.java +++ b/server/user.jobengine.executors/src/user/jobengine/server/steps/CancelableStep.java @@ -7,13 +7,12 @@ public class CancelableStep extends JobStep { private static final Logger logger = LogManager.getLogger(); @StepEntry - public Object[] execute() throws Exception { - logger.info(getMarker(), "Executing"); + public Object[] execute(int param) throws Exception { + logger.info(getMarker(), "Executing with param {}", param); for (int i = 0; i < 10; i++) { - logger.info(getMarker(), "Executing {}", i); if (getJobRuntime().isWaitingCancel()) break; - Thread.sleep(1000); + Thread.sleep(10); int progress = (i + 1) * 100 / 10; setProgress(progress); } diff --git a/server/user.jobengine.executors/src/user/jobengine/server/steps/TestForkCancelableStep.java b/server/user.jobengine.executors/src/user/jobengine/server/steps/TestForkCancelableStep.java index 358e8827..8083d94f 100644 --- a/server/user.jobengine.executors/src/user/jobengine/server/steps/TestForkCancelableStep.java +++ b/server/user.jobengine.executors/src/user/jobengine/server/steps/TestForkCancelableStep.java @@ -12,29 +12,30 @@ import user.jobengine.server.IJobRuntime; public class TestForkCancelableStep extends JobStep { private static final String CHILD_TEMPLATE = "cancelable.xml"; private static final Logger logger = LogManager.getLogger(); - int count = 5; @StepEntry - public Object[] execute(IJobEngine jobEngine, IJobRuntime jobRuntime) throws Exception { - jobRuntime.forkPrepare(); + public Object[] execute(int count) throws Exception { + getJobRuntime().forkPrepare(); for (int i = 0; i < count; i++) { if (getJobRuntime().isWaitingCancel()) break; - //jobEngine.submit(jobRuntime, null, CHILD_TEMPLATE, CHILD_TITLE, ListUtils.asMap("itemID", i)); - // IJobRuntime runtime = jobEngine.submit(jobRuntime, null, CHILD_TEMPLATE, "JOB " + i, ListUtils.asMap("param", i)); IJobRuntime runtime = getEngine().submit(getJobRuntime(), e -> { IJobRuntime r = (IJobRuntime) e.getSource(); + if (r.isDisableRetry()) return; - if (e.getStatus().equals(JobStatus.CANCELED) || e.getStatus().equals(JobStatus.SUSPENDED)) - logger.info("Cleanup occured because status is {} {}", e.getStatus(), ((IJob) e.getSource()).getId()); + + if (e.getStatus().equals(JobStatus.CANCELED) || e.getStatus().equals(JobStatus.SUSPENDED)) { + long id = ((IJob) e.getSource()).getId(); + logger.info("Cleanup occured because status is {} {}", e.getStatus(), id); + } + }, CHILD_TEMPLATE, "JOB " + i, 0, IJobEngine.DEFAULT_OWNER, ListUtils.asMap("param", i)); runtime.setRelated("TEST" + runtime.getId()); } - jobRuntime.forkWaitComplete(); - logger.info("Done"); + getJobRuntime().forkWaitComplete(); return null; } } diff --git a/server/user.jobengine.osgi.server/src/user/jobengine/server/JobEngineConfiguration.java b/server/user.jobengine.osgi.server/src/user/jobengine/server/JobEngineConfiguration.java index b3fe0976..7f717088 100644 --- a/server/user.jobengine.osgi.server/src/user/jobengine/server/JobEngineConfiguration.java +++ b/server/user.jobengine.osgi.server/src/user/jobengine/server/JobEngineConfiguration.java @@ -58,6 +58,7 @@ public class JobEngineConfiguration implements IJobEngineConfiguration { private final Map schedules = new LinkedHashMap(); private URLClassLoader stepsClassLoader; private boolean resetStepClassLoader; + private GroovyClassLoader groovyClassLoader; public synchronized void bindSystemConfiguration(Object service) { if (service instanceof IConfiguration) { @@ -89,6 +90,28 @@ public class JobEngineConfiguration implements IJobEngineConfiguration { return gcl; } + private GroovyClassLoader createGroovyClassLoader(URLClassLoader stepsClassLoader) throws FileNotFoundException { + String stepsDir = systemConfig.getConfig(DIR_STEPS); + + Set sharedFiles = getJavaFileList(Paths.get(stepsDir, "shared"), null); + Set libFiles = getJavaFileList(Paths.get(stepsDir), null); + libFiles.addAll(sharedFiles); + + GroovyClassLoader gcl = new GroovyClassLoader(stepsClassLoader) { + + @Override + protected CompilationUnit createCompilationUnit(CompilerConfiguration config, CodeSource source) { + CompilationUnit result = new CompilationUnit(config, source, this); + libFiles.forEach(f -> result.addSource(f)); + //loadStepSources(result, Paths.get(stepsDir, "shared")); + //loadStepSources(result, Paths.get(stepsDir)); + return result; + } + + }; + return gcl; + } + @SuppressWarnings("unchecked") @Override public IJobStep createJobStep(String stepUnitName) throws Exception { @@ -96,23 +119,40 @@ public class JobEngineConfiguration implements IJobEngineConfiguration { boolean isGroovyClass = stepUnitName.toLowerCase().endsWith(".java") || stepUnitName.toLowerCase().endsWith(".groovy"); - logger.info("Looking for {} step classloader requirement", stepUnitName); + logger.info("Looking for {} step ClassLoader requirement", stepUnitName); - if (stepsClassLoader == null) - stepsClassLoader = createParentClassLoader(); - else { - if (resetStepClassLoader) { + if (resetStepClassLoader) { + logger.info("Processing ClassLoader reset"); + if (groovyClassLoader != null) { + groovyClassLoader.close(); + groovyClassLoader = null; + } + + if (stepsClassLoader != null) { stepsClassLoader.close(); - stepsClassLoader = createParentClassLoader(); + stepsClassLoader = null; } + + resetStepClassLoader = false; + } + + if (stepsClassLoader == null) { + logger.info("Creating parent ClassLoader"); + stepsClassLoader = createParentClassLoader(); + } + + if (groovyClassLoader == null) { + logger.info("Creating Groovy ClassLoader"); + groovyClassLoader = createGroovyClassLoader(stepsClassLoader); } Class stepClass = null; if (isGroovyClass) { - logger.info("Creating GroovyClassLoader"); - GroovyClassLoader gcl = createGroovyClassLoader(stepsClassLoader, stepUnitName); - stepClass = (Class) loadClassFromSourceCode(gcl, stepUnitName); + //kozos classloader hasznalata + //GroovyClassLoader gcl = createGroovyClassLoader(stepsClassLoader, stepUnitName); + + stepClass = (Class) loadClassFromSourceCode(groovyClassLoader, stepUnitName); } else stepClass = (Class) stepsClassLoader.loadClass(stepUnitName); @@ -238,7 +278,7 @@ public class JobEngineConfiguration implements IJobEngineConfiguration { result = gcl.parseClass(path.toFile()); if (result != null) - logger.info("Successfully loaded {}", className); + logger.info("Class for {} successfully loaded", className); return result; } @@ -316,7 +356,7 @@ public class JobEngineConfiguration implements IJobEngineConfiguration { && file.getFileName().toString().endsWith(".java"); try (Stream stream = Files.list(path)) { stream.filter(filter).forEach(p -> { - if (!excludeFileName.equals(p.getFileName().toString())) + if (excludeFileName != null && !excludeFileName.equals(p.getFileName().toString())) result.add(p.toFile()); }); diff --git a/server/user.jobengine.osgi.server/src/user/jobengine/server/JobStepExecutor.java b/server/user.jobengine.osgi.server/src/user/jobengine/server/JobStepExecutor.java index 4a6e602a..764472f2 100644 --- a/server/user.jobengine.osgi.server/src/user/jobengine/server/JobStepExecutor.java +++ b/server/user.jobengine.osgi.server/src/user/jobengine/server/JobStepExecutor.java @@ -1,6 +1,5 @@ package user.jobengine.server; -import java.net.URLClassLoader; import java.util.ArrayList; import java.util.Collections; import java.util.List; @@ -12,7 +11,6 @@ import org.apache.commons.lang.StringUtils; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; -import groovy.lang.GroovyClassLoader; import user.commons.JobStatus; import user.commons.cluster.ClusteredJob; import user.jobengine.server.messages.JobStepCompletedMessage; @@ -145,21 +143,18 @@ public class JobStepExecutor implements IJobStepExecutor { step = createStep(); outputs = step.run(jobEngine, jobRuntime, inputs); } finally { + + /* + //kozos loader hasznalata! if (step != null) { ClassLoader cl = step.getClass().getClassLoader(); if (cl instanceof GroovyClassLoader) { - + GroovyClassLoader stepClassLoader = (GroovyClassLoader) cl; stepClassLoader.close(); - - // URLClassLoader stepParentClassLoader = (URLClassLoader) cl.getParent(); - // stepParentClassLoader.close(); - } else if (cl instanceof URLClassLoader) { - // URLClassLoader stepClassLoader = (URLClassLoader) cl; - // stepClassLoader.close(); - } } + */ } //TODO itt lekezelni a remote notification-t @@ -288,7 +283,8 @@ public class JobStepExecutor implements IJobStepExecutor { boolean result = false; if (getMaxConcurrent() < workers.size()) { workers.remove(worker); - logger.info("Worker {} is retired, max {} current {}", getStepUnitName(), getMaxConcurrent(), workers.size()); + logger.info("Worker {} is retired, max {} current {}", getStepUnitName(), getMaxConcurrent(), + workers.size()); result = true; } diff --git a/server/user.mediacube.gui/pages/metadatadetails.zul b/server/user.mediacube.gui/pages/metadatadetails.zul index 988ec810..c855d637 100644 --- a/server/user.mediacube.gui/pages/metadatadetails.zul +++ b/server/user.mediacube.gui/pages/metadatadetails.zul @@ -3,6 +3,9 @@ sizable="true" position="left,top" viewModel="@id('vm') @init('user.jobengine.zk.model.MetadataDetailsModel')" forward="onCancel=closeButton.onClick">