From 73fb9ad2d87fb346c3c539b53111d704aee129e0 Mon Sep 17 00:00:00 2001 From: =?utf8?q?V=C3=A1s=C3=A1ry=20D=C3=A1niel?= Date: Wed, 14 Feb 2018 16:16:50 +0000 Subject: [PATCH] git-tfs-id: [http://tfs.userrendszerhaz.hu:8080/tfs/DefaultCollection]$/MediaCube;C30906 --- client/Maestro/MaestroForm.Target.cs | 3 +- client/Maestro/Properties/AssemblyInfo.cs | 4 +- .../debug-mediacube-remote.launch | 4 +- server/-configuration/log4j2.xml | 8 +- server/-configuration/scheduledjobs.json | 12 +- .../user.commons.log4j2/META-INF/MANIFEST.MF | 17 ++- .../log4j2/appender/DB2NoSQLAppender.java | 107 +++++++++++++++++ .../commons/log4j2/appender/SmtpManager.java | 9 +- .../copyforarchive-nexio-recordings.xml | 4 +- .../jobtemplates/import-statistics.xml | 10 ++ .../CopyForArchiveNEXIORecordingsStep.java | 10 +- .../server/steps/FakeNoParamsStep.java | 63 +++++----- .../user/jobengine/server/steps/FakeStep.java | 1 + .../user/jobengine/server/steps/ForkStep.java | 50 ++++++++ .../server/steps/ImportStatisticsStep.java | 112 +++++++++++++++++- .../server/steps/OctopusDataMiner.java | 2 + .../src/user/commons/ListUtils.java | 4 +- .../src/user/commons/octopus/IOctopusAPI.java | 2 + .../octopus/test/OctopusDataMinerTest.java | 13 ++ .../pages/joblist.zul | 104 ++++++++-------- .../pages/jobparameters.zul | 25 ++++ .../resources/i3-label_hu.properties | 4 +- .../user/jobengine/zk/model/JobListModel.java | 22 ++-- .../zk/model/JobParametersModel.java | 45 +++++++ 24 files changed, 521 insertions(+), 114 deletions(-) create mode 100644 server/user.commons.log4j2/src/user/commons/log4j2/appender/DB2NoSQLAppender.java create mode 100644 server/user.jobengine.executors/src/user/jobengine/server/steps/ForkStep.java create mode 100644 server/user.jobengine.osgi.server/pages/jobparameters.zul create mode 100644 server/user.jobengine.osgi.server/src/user/jobengine/zk/model/JobParametersModel.java diff --git a/client/Maestro/MaestroForm.Target.cs b/client/Maestro/MaestroForm.Target.cs index 39a40434..4b74b05f 100644 --- a/client/Maestro/MaestroForm.Target.cs +++ b/client/Maestro/MaestroForm.Target.cs @@ -149,7 +149,8 @@ namespace Maestro { force |= target.SaveArchiveMetadata || target.SaveMorpheusMetadata || target.NexioServer; if (force && (ArchiveMetadata == null || !ArchiveMetadata.IsFilled)) { ArchiveMetadata = GetArchiveMetadata(); - SelectedMetadata.MetadataText = ArchiveMetadata.itemTitle; + if (SelectedMetadata != null && String.IsNullOrEmpty(SelectedMetadata.MetadataText)) + SelectedMetadata.MetadataText = ArchiveMetadata.itemTitle; UpdateEditArchiveMetadataStatus(); if (ArchiveMetadata == null) { MessageBox.Show(String.Format("A {0} folyamat nem futtatható kísérő adatok nélkül.", target.Label)); diff --git a/client/Maestro/Properties/AssemblyInfo.cs b/client/Maestro/Properties/AssemblyInfo.cs index 73085c6a..cf01775d 100644 --- a/client/Maestro/Properties/AssemblyInfo.cs +++ b/client/Maestro/Properties/AssemblyInfo.cs @@ -32,5 +32,5 @@ using System.Runtime.InteropServices; // You can specify all the values or you can default the Build and Revision Numbers // by using the '*' as shown below: // [assembly: AssemblyVersion("1.0.*")] -[assembly: AssemblyVersion("2.0.7.5")] -[assembly: AssemblyFileVersion("2.0.7.5")] +[assembly: AssemblyVersion("2.0.7.6")] +[assembly: AssemblyFileVersion("2.0.7.6")] diff --git a/server/-configuration/debug-mediacube-remote.launch b/server/-configuration/debug-mediacube-remote.launch index 4a99bae5..af15006c 100644 --- a/server/-configuration/debug-mediacube-remote.launch +++ b/server/-configuration/debug-mediacube-remote.launch @@ -10,8 +10,8 @@ - - + + diff --git a/server/-configuration/log4j2.xml b/server/-configuration/log4j2.xml index 498e21f0..d24e9d3b 100644 --- a/server/-configuration/log4j2.xml +++ b/server/-configuration/log4j2.xml @@ -26,8 +26,13 @@ - + + + + + + @@ -50,6 +55,7 @@ + diff --git a/server/-configuration/scheduledjobs.json b/server/-configuration/scheduledjobs.json index 378dd271..54c1b1ed 100644 --- a/server/-configuration/scheduledjobs.json +++ b/server/-configuration/scheduledjobs.json @@ -1,5 +1,15 @@ {"joblist":[ { + "active": false, + "name" : "Statisztika importálása", + "template": "import-statistics.xml", + "executeimmediate": false, + "cronexpression": "0 0 0/1 1/1 * ? *", + "parameters": [ + {"name": "daysBeforeNow", "value": 1, "type": "java.lang.Integer"} + ] + }, + { "active": false, "name" : "MORPHEUS 'missing materials' importálása", "template": "import-morpheus-missing-materials.xml", @@ -67,7 +77,7 @@ {"name": "nexioPort", "value": 2098, "type": "java.lang.Integer"}, {"name": "nexioUserName", "value": "administrator", "type": "java.lang.String"}, {"name": "nexioPassword", "value": "system", "type": "java.lang.String"}, - {"name": "archiveFtp", "value": "ftp://10.10.1.100/ARCHIVE/TEST", "type": "java.lang.String"}, + {"name": "archiveFtp", "value": "ftp://10.10.1.100/ARCHIVE", "type": "java.lang.String"}, {"name": "archiveUserName", "value": "mediacube", "type": "java.lang.String"}, {"name": "archivePassword", "value": "Broadca5T", "type": "java.lang.String"}, {"name": "filterAgencies", "value": "schedule-rec", "type": "java.lang.String"}, diff --git a/server/user.commons.log4j2/META-INF/MANIFEST.MF b/server/user.commons.log4j2/META-INF/MANIFEST.MF index d4355ee3..12057544 100644 --- a/server/user.commons.log4j2/META-INF/MANIFEST.MF +++ b/server/user.commons.log4j2/META-INF/MANIFEST.MF @@ -6,5 +6,18 @@ Bundle-Version: 1.0.0 Fragment-Host: org.apache.logging.log4j.core;bundle-version="2.8.2" Bundle-RequiredExecutionEnvironment: JavaSE-1.8 Export-Package: user.commons.log4j2.appender -Import-Package: javax.mail;version="1.5.0", - user.commons +Import-Package: com.fasterxml.jackson.annotation;version="2.4.5", + com.fasterxml.jackson.core;version="2.4.5", + com.fasterxml.jackson.core.type;version="2.4.5", + com.fasterxml.jackson.core.util;version="2.4.5", + com.fasterxml.jackson.databind;version="2.4.5", + com.fasterxml.jackson.databind.deser.std;version="2.4.5", + com.fasterxml.jackson.databind.module;version="2.4.5", + com.fasterxml.jackson.databind.ser;version="2.4.5", + com.fasterxml.jackson.databind.ser.impl;version="2.4.5", + com.ibm.nosql.bson, + com.ibm.nosql.json, + com.ibm.nosql.json.api, + javax.mail;version="1.5.0", + user.commons, + user.commons.nosql diff --git a/server/user.commons.log4j2/src/user/commons/log4j2/appender/DB2NoSQLAppender.java b/server/user.commons.log4j2/src/user/commons/log4j2/appender/DB2NoSQLAppender.java new file mode 100644 index 00000000..b6e97885 --- /dev/null +++ b/server/user.commons.log4j2/src/user/commons/log4j2/appender/DB2NoSQLAppender.java @@ -0,0 +1,107 @@ +package user.commons.log4j2.appender; + +import java.io.Serializable; +import java.util.Date; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; + +import org.apache.logging.log4j.Marker; +import org.apache.logging.log4j.core.Appender; +import org.apache.logging.log4j.core.Core; +import org.apache.logging.log4j.core.Filter; +import org.apache.logging.log4j.core.Layout; +import org.apache.logging.log4j.core.LogEvent; +import org.apache.logging.log4j.core.appender.AbstractAppender; +import org.apache.logging.log4j.core.config.Configuration; +import org.apache.logging.log4j.core.config.DefaultConfiguration; +import org.apache.logging.log4j.core.config.plugins.Plugin; +import org.apache.logging.log4j.core.config.plugins.PluginAttribute; +import org.apache.logging.log4j.core.config.plugins.PluginConfiguration; +import org.apache.logging.log4j.core.config.plugins.PluginElement; +import org.apache.logging.log4j.core.config.plugins.PluginFactory; +import org.apache.logging.log4j.core.config.plugins.validation.constraints.Required; +import org.apache.logging.log4j.core.filter.ThresholdFilter; +import org.apache.logging.log4j.core.layout.JsonLayout; +import org.apache.logging.log4j.core.util.Booleans; + +import com.ibm.nosql.json.api.BasicDBObject; +import com.ibm.nosql.json.api.DB; +import com.ibm.nosql.json.api.DBCollection; + +import user.commons.MediaCubeMarker; +import user.commons.nosql.NoSQLUtils; + +@Plugin(name = "DB2NoSQLAppender", category = Core.CATEGORY_NAME, elementType = Appender.ELEMENT_TYPE, printObject = true) +public final class DB2NoSQLAppender extends AbstractAppender { + + private static final int CONCURRENT_QUEUES = 5; + private static final ExecutorService executor = Executors.newFixedThreadPool(CONCURRENT_QUEUES); + + @PluginFactory + public static DB2NoSQLAppender createAppender(@PluginConfiguration final Configuration config, @PluginAttribute("name") @Required final String name, + @PluginElement("Layout") Layout layout, @PluginElement("Filter") Filter filter, + @PluginAttribute("ignoreExceptions") final String ignore) { + if (name == null) { + LOGGER.error("No name provided for SmtpAppender"); + return null; + } + + if (layout == null) { + layout = JsonLayout.createDefaultLayout(); + } + if (filter == null) { + filter = ThresholdFilter.createFilter(null, null, null); + } + final boolean ignoreExceptions = Booleans.parseBoolean(ignore, true); + + final Configuration configuration = config != null ? config : new DefaultConfiguration(); + + return new DB2NoSQLAppender(name, filter, layout, ignoreExceptions); + } + + private DBCollection collection; + + private DB2NoSQLAppender(final String name, final Filter filter, final Layout layout, final boolean ignoreExceptions) { + super(name, filter, layout, ignoreExceptions); + } + + @Override + public void append(final LogEvent event) { + Marker m = event.getMarker(); + if (!(m instanceof MediaCubeMarker)) + return; + + final MediaCubeMarker marker = (MediaCubeMarker) m; + final String sessionID = marker.getSessionID(); + if (sessionID == null || sessionID.length() == 0) + return; + + final LogEvent evt = event.toImmutable(); + executor.submit(() -> { + // byte[] bytes = getLayout().toByteArray(evt); + // String json = new String(bytes); + // BasicDBObject eo = (BasicDBObject) JSONUtil.jsonToDbObject(json); + BasicDBObject smartEvent = new BasicDBObject(); + smartEvent.put("dateTime", new Date(evt.getTimeMillis())); + smartEvent.put("thread", evt.getThreadName()); + smartEvent.put("level", evt.getLevel().toString()); + smartEvent.put("loggerName", evt.getLoggerName()); + smartEvent.put("sessionID", Long.parseLong(sessionID)); + smartEvent.put("sessionName", marker.getSessionName()); + smartEvent.put("message", evt.getMessage().getFormattedMessage()); + smartEvent.put("throwable", evt.getMessage().getThrowable()); + + if (collection == null) { + DB db = NoSQLUtils.getNoSQLDB(); + collection = db.getCollection("sessionlogs"); + } + collection.save(smartEvent); + }); + } + + @Override + public boolean isFiltered(final LogEvent event) { + final boolean filtered = super.isFiltered(event); + return filtered; + } +} diff --git a/server/user.commons.log4j2/src/user/commons/log4j2/appender/SmtpManager.java b/server/user.commons.log4j2/src/user/commons/log4j2/appender/SmtpManager.java index 9e402f8e..1b3a43df 100644 --- a/server/user.commons.log4j2/src/user/commons/log4j2/appender/SmtpManager.java +++ b/server/user.commons.log4j2/src/user/commons/log4j2/appender/SmtpManager.java @@ -354,8 +354,15 @@ public class SmtpManager extends AbstractManager { List events = sessionEvents.get(sessionID); if (events == null) return; + + Level mailLevel = level; + for (LogEvent event : events) { + if (event.getLevel().intLevel() < mailLevel.intLevel()) + mailLevel = event.getLevel(); + } + String to = mcm.getTo() == null ? defaultRecipient : mcm.getTo(); - String subject = mcm.getSessionName() == null ? defaultSubject : String.format("%s: %s #%s", level, mcm.getSessionName(), mcm.getSessionID()); + String subject = mcm.getSessionName() == null ? defaultSubject : String.format("%s: %s #%s", mailLevel, mcm.getSessionName(), mcm.getSessionID()); sendEvents(to, subject, layout, events); sessionEvents.remove(sessionID); } diff --git a/server/user.jobengine.executors/jobtemplates/copyforarchive-nexio-recordings.xml b/server/user.jobengine.executors/jobtemplates/copyforarchive-nexio-recordings.xml index 5ad631a4..60ca8b1d 100644 --- a/server/user.jobengine.executors/jobtemplates/copyforarchive-nexio-recordings.xml +++ b/server/user.jobengine.executors/jobtemplates/copyforarchive-nexio-recordings.xml @@ -12,7 +12,7 @@ - + @@ -46,7 +46,7 @@ - + diff --git a/server/user.jobengine.executors/jobtemplates/import-statistics.xml b/server/user.jobengine.executors/jobtemplates/import-statistics.xml index f195723d..44d771cb 100644 --- a/server/user.jobengine.executors/jobtemplates/import-statistics.xml +++ b/server/user.jobengine.executors/jobtemplates/import-statistics.xml @@ -1,7 +1,17 @@ + + + + + + + + + + \ No newline at end of file diff --git a/server/user.jobengine.executors/src/user/jobengine/server/steps/CopyForArchiveNEXIORecordingsStep.java b/server/user.jobengine.executors/src/user/jobengine/server/steps/CopyForArchiveNEXIORecordingsStep.java index d9220945..b1d0bb48 100644 --- a/server/user.jobengine.executors/src/user/jobengine/server/steps/CopyForArchiveNEXIORecordingsStep.java +++ b/server/user.jobengine.executors/src/user/jobengine/server/steps/CopyForArchiveNEXIORecordingsStep.java @@ -179,12 +179,14 @@ public class CopyForArchiveNEXIORecordingsStep extends JobStep { } private String getTargetFileName(String sourceFileName, String[] names) { + String result = sourceFileName + MXFEXT; + if (names == null || names.length == 0) + return result; List remoteNames = new ArrayList<>(); for (String name : names) { if (name != null) remoteNames.add(name.toLowerCase()); } - String result = sourceFileName + MXFEXT; int version = 1; while (remoteNames.contains(result.toLowerCase())) { result = String.format("%s-%d%s", sourceFileName, version, MXFEXT); @@ -313,7 +315,7 @@ public class CopyForArchiveNEXIORecordingsStep extends JobStep { check(nexioUserName, "nexioUserName"); check(nexioPassword, "nexioPassword"); - check(agencies, "agencies"); + check(agencies, "filterAgencies"); filterAgencies = agencies.split(","); check(limit, "limit"); @@ -354,11 +356,11 @@ public class CopyForArchiveNEXIORecordingsStep extends JobStep { sourceFtp = ((FtpDirectoryLister) sourceUri.getLister()).connect(); targetFtp = ((FtpDirectoryLister) targetUri.getLister()).connect(); + result = getTargetFileName(sourceFileName, targetFtp.listNames()); + if (!targetFtp.enterRemotePassiveMode()) throw new Exception("!PASV"); - result = getTargetFileName(sourceFileName, targetFtp.listNames()); - reply = sourceFtp.port(InetAddress.getByName(targetFtp.getPassiveHost()), targetFtp.getPassivePort()); if (!FTPReply.isPositiveCompletion(reply)) throw new Exception("PORT parancs válasza: " + sourceFtp.getReplyString()); diff --git a/server/user.jobengine.executors/src/user/jobengine/server/steps/FakeNoParamsStep.java b/server/user.jobengine.executors/src/user/jobengine/server/steps/FakeNoParamsStep.java index 61bb5b5b..03cadcfb 100644 --- a/server/user.jobengine.executors/src/user/jobengine/server/steps/FakeNoParamsStep.java +++ b/server/user.jobengine.executors/src/user/jobengine/server/steps/FakeNoParamsStep.java @@ -1,57 +1,54 @@ package user.jobengine.server.steps; -import java.util.concurrent.CountDownLatch; +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.concurrent.Semaphore; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import user.commons.ListUtils; +import user.jobengine.db.ItemManagerData.SignalType; +import user.jobengine.server.IJobChangedListener; import user.jobengine.server.IJobEngine; import user.jobengine.server.IJobRuntime; -import user.jobengine.server.JobStatusChangedEvent; public class FakeNoParamsStep extends JobStep { private static final String CHILD_TITLE = "Párhuzamosított alfolyamat"; private static final String CHILD_TEMPLATE = "fake-concurrent.xml"; private static final Logger logger = LogManager.getLogger(); int count = 10; - CountDownLatch latch = new CountDownLatch(count); - - private void childStatusChanged(JobStatusChangedEvent event) { - switch (event.getStatus()) { - case CANCELED: - case FINISHED: - case RUNABLE: - case SUSPENDED: - latch.countDown(); - break; - default: - break; - } - // if (JobStatus.SUSPENDED.equals(event.getStatus()) || JobStatus.FINISHED.equals(event.getStatus())) - } @StepEntry public Object[] execute(IJobEngine jobEngine, IJobRuntime jobRuntime) throws Exception { - // logger.warn(jobRuntime.getMarker(), "Egy WARN {}", "paraméterrel"); - // logger.error(jobRuntime.getMarker(), "Egy ERROR {}", "paraméterrel"); - // for (int i = 0; i < count; i++) { - // if (!canContinue()) - // break; - // jobRuntime.incrementProgress((i + 1) * count); - // - // Thread.sleep(100); - // logger.info(jobRuntime.getMarker(), "Log markered" + i); - // - // if (i == 8) - // throw new Exception("stop"); - // } + final Semaphore semaphore = new Semaphore(1); + List childrenIDs = Collections.synchronizedList(new ArrayList<>()); + + IJobChangedListener listener = event -> { + IJobRuntime child = event.getJob(); + if (event.getSignalType().equals(SignalType.CREATE)) { + if (child.getParentJobId() == jobRuntime.getId()) { + childrenIDs.add(child.getId()); + } - //Thread.sleep(100000); + if (child.getParentJobId() != jobRuntime.getId() && childrenIDs.contains(child.getParentJobId())) { + childrenIDs.add(child.getId()); + } + } + if (event.getSignalType().equals(SignalType.DELETE) && childrenIDs.contains(child.getId())) { + childrenIDs.remove(child.getId()); + if (childrenIDs.size() == 0) + semaphore.release(); + } + }; + jobEngine.addJobChangedEventListener(listener); + semaphore.acquire(); for (int i = 0; i < count; i++) { - jobEngine.submit(jobRuntime, e -> childStatusChanged(e), CHILD_TEMPLATE, CHILD_TITLE, ListUtils.asMap("itemID", i)); + jobEngine.submit(jobRuntime, null, CHILD_TEMPLATE, CHILD_TITLE, ListUtils.asMap("itemID", i)); } - latch.await(); + semaphore.acquire(); + jobEngine.removeJobChangedEventListener(listener); return null; } } diff --git a/server/user.jobengine.executors/src/user/jobengine/server/steps/FakeStep.java b/server/user.jobengine.executors/src/user/jobengine/server/steps/FakeStep.java index 60853110..c3b694d4 100644 --- a/server/user.jobengine.executors/src/user/jobengine/server/steps/FakeStep.java +++ b/server/user.jobengine.executors/src/user/jobengine/server/steps/FakeStep.java @@ -16,6 +16,7 @@ public class FakeStep extends JobStep { Object[] result = { itemID + 1 }; logger.info(jobRuntime.getMarker(), "Starting with {}", itemID); + logger.error(jobRuntime.getMarker(), "Error with {}", itemID); try { for (int i = 0; i < count; i++) { //sb.append("Sikeres iteráció : ") diff --git a/server/user.jobengine.executors/src/user/jobengine/server/steps/ForkStep.java b/server/user.jobengine.executors/src/user/jobengine/server/steps/ForkStep.java new file mode 100644 index 00000000..fe17d209 --- /dev/null +++ b/server/user.jobengine.executors/src/user/jobengine/server/steps/ForkStep.java @@ -0,0 +1,50 @@ +package user.jobengine.server.steps; + +import java.util.ArrayList; +import java.util.Collections; +import java.util.List; +import java.util.concurrent.Semaphore; + +import user.jobengine.db.ItemManagerData.SignalType; +import user.jobengine.server.IJobChangedListener; +import user.jobengine.server.IJobEngine; +import user.jobengine.server.IJobRuntime; + +public abstract class ForkStep extends JobStep { + + @StepEntry + public void execute(IJobEngine jobEngine, IJobRuntime jobRuntime) throws Exception { + final Semaphore semaphore = new Semaphore(1); + List childrenIDs = Collections.synchronizedList(new ArrayList<>()); + + IJobChangedListener listener = event -> { + IJobRuntime child = event.getJob(); + if (event.getSignalType().equals(SignalType.CREATE)) { + if (child.getParentJobId() == jobRuntime.getId()) { + childrenIDs.add(child.getId()); + } + + if (child.getParentJobId() != jobRuntime.getId() && childrenIDs.contains(child.getParentJobId())) { + childrenIDs.add(child.getId()); + } + } + if (event.getSignalType().equals(SignalType.DELETE) && childrenIDs.contains(child.getId())) { + childrenIDs.remove(child.getId()); + if (childrenIDs.size() == 0) + semaphore.release(); + } + }; + jobEngine.addJobChangedEventListener(listener); + semaphore.acquire(); + try { + fork(jobEngine, jobRuntime); + } catch (Exception e) { + semaphore.release(); + throw e; + } + semaphore.acquire(); + jobEngine.removeJobChangedEventListener(listener); + } + + protected abstract void fork(IJobEngine jobEngine, IJobRuntime jobRuntime) throws Exception; +} diff --git a/server/user.jobengine.executors/src/user/jobengine/server/steps/ImportStatisticsStep.java b/server/user.jobengine.executors/src/user/jobengine/server/steps/ImportStatisticsStep.java index c4f01eb2..7268b0ec 100644 --- a/server/user.jobengine.executors/src/user/jobengine/server/steps/ImportStatisticsStep.java +++ b/server/user.jobengine.executors/src/user/jobengine/server/steps/ImportStatisticsStep.java @@ -1,18 +1,128 @@ package user.jobengine.server.steps; +import java.util.Calendar; +import java.util.Date; +import java.util.HashMap; +import java.util.List; +import java.util.Map; + import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; +import org.apache.logging.log4j.Marker; + +import com.ibm.nosql.json.api.BasicDBObject; +import com.ibm.nosql.json.api.DBObject; +import user.commons.CalendarUtils; +import user.commons.nosql.NoSQLUtils; +import user.commons.octopus.IOctopusAPI; +import user.commons.octopus.OctopusAPI; import user.jobengine.server.IJobEngine; import user.jobengine.server.IJobRuntime; public class ImportStatisticsStep extends JobStep { private static final Logger logger = LogManager.getLogger(); + private static final String SCHEDULED_FORMAT = "yyyy.MM.dd HH:mm"; + private IOctopusAPI octopusAPI; + private Marker marker; + + private void createAPI(IJobRuntime jobRuntime) throws Exception { + try { + octopusAPI = new OctopusAPI(); + } catch (Exception e) { + logger.catching(e); + logger.error(jobRuntime.getMarker(), "Az Octopus adatbázis nem érhető el. A rendszer üzenete: {}", e.getMessage()); + throw e; + } + } @StepEntry - public Object[] execute(IJobEngine jobEngine, IJobRuntime jobRuntime) throws Exception { + public Object[] execute(int daysBeforeNow, IJobEngine jobEngine, IJobRuntime jobRuntime) throws Exception { + marker = jobRuntime.getMarker(); + createAPI(jobRuntime); + + Calendar scheduledDate = Calendar.getInstance(); + scheduledDate.add(Calendar.DAY_OF_YEAR, -1 * daysBeforeNow); + List rundowns = octopusAPI.getRundowns(scheduledDate.getTime()); + if (rundowns == null) { + logger.warn(marker, "Nem található adástükör a {} napra.", CalendarUtils.toDateString(scheduledDate)); + return null; + } + Map stories = processRundowns(rundowns); + + List folders = octopusAPI.getStoryFolders(); + Map folderStories = processFolders(folders, scheduledDate); + stories.putAll(folderStories); return null; } + private Map processFolders(List folders, Calendar scheduledDate) { + Map result = new HashMap<>(); + for (DBObject f : folders) { + BasicDBObject folder = (BasicDBObject) f; + String folderName = null; + long folderID = 0; + try { + folderID = NoSQLUtils.asLong(folder, IOctopusAPI.ID); + folderName = folder.getString(IOctopusAPI.NAME); + List stories = octopusAPI.getStoryFolderStories(folderID); + if (stories != null) + result.putAll(processStories(stories, scheduledDate)); + } catch (Exception e) { + logger.catching(e); + logger.error(marker, String.format("A %s %s mappa feldolgozása nem lehetséges. A rendszer üzenete: %s", folderID, folderName, e.getMessage())); + } + } + return result; + } + + private Map processRundowns(List rundowns) { + Map result = new HashMap<>(); + for (DBObject r : rundowns) { + BasicDBObject rundown = (BasicDBObject) r; + String rundownName = null; + long rundownID = 0; + try { + rundownID = NoSQLUtils.asLong(rundown, IOctopusAPI.ID); + String name = NoSQLUtils.asString(NoSQLUtils.asDBObject(rundown, IOctopusAPI.RUNDOWN_TYPE), IOctopusAPI.NAME); + String channel = NoSQLUtils.asString(NoSQLUtils.asDBObject(rundown, IOctopusAPI.CHANNEL), IOctopusAPI.NAME); + Date scheduledStart = rundown.getDate(IOctopusAPI.SCHEDULED_START); + String start = CalendarUtils.toString(CalendarUtils.createCalendar(scheduledStart), SCHEDULED_FORMAT); + //String rundownName = NoSQLUtils.asString(rundown, IOctopusAPI.NAME); + rundownName = String.format("%s %s %s", start, name, channel); + List stories = octopusAPI.getRundownStories(rundownID); + if (stories != null) + result.putAll(processStories(stories, null)); + } catch (Exception e) { + logger.catching(e); + logger.error(marker, + String.format("A %s %s tükör feldolgozása nem lehetséges. A rendszer üzenete: %s", rundownID, rundownName, e.getMessage())); + } + } + return result; + } + + private Map processStories(List stories, Calendar scheduledDate) { + Map result = new HashMap<>(); + for (DBObject storyObject : stories) { + BasicDBObject story = (BasicDBObject) storyObject; + BasicDBObject mosObjects = NoSQLUtils.asDBObject(story, IOctopusAPI.MOS_OBJECTS); + if (mosObjects == null || mosObjects.isEmpty()) + continue; + long storyID = NoSQLUtils.asLong(story, IOctopusAPI.ID); + if (storyID == 0) + continue; + + if (scheduledDate != null) { + //story.get + + } + + if (!result.containsKey(storyID)) + result.put(storyID, story); + } + return result; + } + } diff --git a/server/user.jobengine.executors/src/user/jobengine/server/steps/OctopusDataMiner.java b/server/user.jobengine.executors/src/user/jobengine/server/steps/OctopusDataMiner.java index 9490aa7c..c5e96a9e 100644 --- a/server/user.jobengine.executors/src/user/jobengine/server/steps/OctopusDataMiner.java +++ b/server/user.jobengine.executors/src/user/jobengine/server/steps/OctopusDataMiner.java @@ -588,6 +588,8 @@ public class OctopusDataMiner { long storyID = story.getLong(IOctopusAPI.ID); story.put(IOctopusAPI.MODIFIED, toDate(story, IOctopusAPI.MODIFIED)); + story.put(IOctopusAPI.SCHEDULEFROM, toDate(story, IOctopusAPI.SCHEDULEFROM)); + story.put(IOctopusAPI.SCHEDULETO, toDate(story, IOctopusAPI.SCHEDULETO)); BasicDBList rundownRef = storyRundowns.get(storyID); BasicDBList storyFolderRef = storyStoryFolders.get(storyID); BasicDBList modifiedMOS = extractRelevantMOSObjects(story); diff --git a/server/user.jobengine.osgi.commons/src/user/commons/ListUtils.java b/server/user.jobengine.osgi.commons/src/user/commons/ListUtils.java index fa263bec..9decfe62 100644 --- a/server/user.jobengine.osgi.commons/src/user/commons/ListUtils.java +++ b/server/user.jobengine.osgi.commons/src/user/commons/ListUtils.java @@ -35,10 +35,8 @@ public class ListUtils { return result; } - public static Map asMap(Object... items) throws Exception { + public static Map asMap(Object... items) { Map result = new HashMap<>(); - if (items.length % 2 != 0) - throw new Exception("Even number of parameters is required."); for (int i = 0; i < items.length; i = i + 2) { result.put(String.valueOf(items[i]), items[i + 1]); } diff --git a/server/user.jobengine.osgi.commons/src/user/commons/octopus/IOctopusAPI.java b/server/user.jobengine.osgi.commons/src/user/commons/octopus/IOctopusAPI.java index 9cdbdd58..9b6db5c8 100644 --- a/server/user.jobengine.osgi.commons/src/user/commons/octopus/IOctopusAPI.java +++ b/server/user.jobengine.osgi.commons/src/user/commons/octopus/IOctopusAPI.java @@ -38,6 +38,8 @@ public interface IOctopusAPI { static final String SCHEDULED_START = "scheduledStart"; static final String SLUGS = "slugs"; static final String MODIFIED = "modified"; + static final String SCHEDULEFROM = "scheduleFrom"; + static final String SCHEDULETO = "scheduleTo"; static final String STORIES = "stories"; static final String ID = "id"; static final String _ID = "_id"; diff --git a/server/user.jobengine.osgi.commons/test/user/common/octopus/test/OctopusDataMinerTest.java b/server/user.jobengine.osgi.commons/test/user/common/octopus/test/OctopusDataMinerTest.java index e94eeada..47cb0924 100644 --- a/server/user.jobengine.osgi.commons/test/user/common/octopus/test/OctopusDataMinerTest.java +++ b/server/user.jobengine.osgi.commons/test/user/common/octopus/test/OctopusDataMinerTest.java @@ -117,6 +117,19 @@ public class OctopusDataMinerTest { // assertEquals(x, y); // } + // @Test + // public void test2() throws Exception { + // Map m1 = ListUtils.asMap("1", 1); + // Map m2 = ListUtils.asMap("1", 2); + // Map m3 = new HashMap<>(); + // m3.putAll(m1); + // m3.putAll(m2); + // assertEquals(2, m3.get("1")); + // + // m1.putAll(m2); + // assertEquals(2, m1.get("1")); + // } + @Test public void testClient() { ResteasyWebTarget webTarget = new ResteasyClientBuilder().build().target("http://10.10.1.28/services/rest/octopus/"); diff --git a/server/user.jobengine.osgi.server/pages/joblist.zul b/server/user.jobengine.osgi.server/pages/joblist.zul index e507b0a9..fbf6fb9d 100644 --- a/server/user.jobengine.osgi.server/pages/joblist.zul +++ b/server/user.jobengine.osgi.server/pages/joblist.zul @@ -29,58 +29,60 @@
- - - - - - - - - - - - - - + + + + + + + + + + + + + + - - - - - - - - - - - - - - - - - - - - - + + + + + + + + + + + + + + + + + + + + +
diff --git a/server/user.jobengine.osgi.server/pages/jobparameters.zul b/server/user.jobengine.osgi.server/pages/jobparameters.zul new file mode 100644 index 00000000..36d57dd2 --- /dev/null +++ b/server/user.jobengine.osgi.server/pages/jobparameters.zul @@ -0,0 +1,25 @@ + + + + + + + + + + + +