From: Vásáry Dániel Date: Tue, 23 Jan 2018 15:44:43 +0000 (+0000) Subject: git-tfs-id: [http://tfs.userrendszerhaz.hu:8080/tfs/DefaultCollection]$/MediaCube... X-Git-Url: http://git.useribm.hu/?a=commitdiff_plain;h=904dd9c3e51e595518654d4485a17e945f9f3e11;p=mediacube.git git-tfs-id: [tfs.userrendszerhaz.hu:8080/tfs/DefaultCollection]$/MediaCube;C30845 --- diff --git a/server/-configuration/log4j2.xml b/server/-configuration/log4j2.xml index b576e7da..74ce51be 100644 --- a/server/-configuration/log4j2.xml +++ b/server/-configuration/log4j2.xml @@ -24,7 +24,9 @@ - + + + diff --git a/server/-configuration/mediacube-auth.properties b/server/-configuration/mediacube-auth.properties index ec8d3640..1e070942 100644 --- a/server/-configuration/mediacube-auth.properties +++ b/server/-configuration/mediacube-auth.properties @@ -4,6 +4,6 @@ ad_host=10.10.254.11 ad_nonsecureport=3268 ad_base_dn=DC=intra,DC=echotv,DC=hu ad_admin_map=G_ECH_U_INFORMATIKUSOK,G_ECH_U_MUSZAKVEZETOK,ECH-ISILON-ADMINS -local_users=user:password +local_users=user:password;kuka:kuka local_admins=root:password local_jobsubmitters=lebony:lebony \ No newline at end of file diff --git a/server/-configuration/run-mediacube-server-bsh.launch b/server/-configuration/run-mediacube-server-bsh.launch index 164f27ee..193800bd 100644 --- a/server/-configuration/run-mediacube-server-bsh.launch +++ b/server/-configuration/run-mediacube-server-bsh.launch @@ -19,7 +19,7 @@ - + diff --git a/server/user.commons.log4j2/src/user/commons/log4j2/appender/SmtpManager.java b/server/user.commons.log4j2/src/user/commons/log4j2/appender/SmtpManager.java index 3e747773..666e7b03 100644 --- a/server/user.commons.log4j2/src/user/commons/log4j2/appender/SmtpManager.java +++ b/server/user.commons.log4j2/src/user/commons/log4j2/appender/SmtpManager.java @@ -19,8 +19,13 @@ package user.commons.log4j2.appender; import java.io.ByteArrayOutputStream; import java.io.IOException; import java.io.OutputStream; +import java.util.ArrayList; +import java.util.Arrays; import java.util.Date; +import java.util.List; +import java.util.Map; import java.util.Properties; +import java.util.concurrent.ConcurrentHashMap; import javax.activation.DataSource; import javax.mail.Authenticator; @@ -32,7 +37,6 @@ import javax.mail.Transport; import javax.mail.internet.InternetHeaders; import javax.mail.internet.MimeBodyPart; import javax.mail.internet.MimeMessage; -import javax.mail.internet.MimeMessage.RecipientType; import javax.mail.internet.MimeMultipart; import javax.mail.internet.MimeUtility; import javax.mail.util.ByteArrayDataSource; @@ -54,6 +58,7 @@ import org.apache.logging.log4j.message.ReusableMessage; import org.apache.logging.log4j.util.PropertiesUtil; import org.apache.logging.log4j.util.Strings; +import user.commons.MediaCubeFinishMarker; import user.commons.MediaCubeMarker; /** @@ -147,6 +152,7 @@ public class SmtpManager extends AbstractManager { } private static final SMTPManagerFactory FACTORY = new SMTPManagerFactory(); + private static Map> sessionEvents = new ConcurrentHashMap<>(); private static MimeMessage createMimeMessage(final FactoryData data, final Session session, final LogEvent appendEvent) throws MessagingException { return new MimeMessageBuilder(session).setFrom(data.from).setReplyTo(data.replyto).setRecipients(Message.RecipientType.TO, data.to) @@ -256,9 +262,9 @@ public class SmtpManager extends AbstractManager { return encoded.toByteArray(); } - protected byte[] formatContentToBytes(final LogEvent[] priorEvents, final LogEvent appendEvent, final Layout layout) throws IOException { + protected byte[] formatContentToBytes(final List events, final Layout layout) throws IOException { final ByteArrayOutputStream raw = new ByteArrayOutputStream(); - writeContent(priorEvents, appendEvent, layout, raw); + writeContent(events, layout, raw); return raw.toByteArray(); } @@ -281,6 +287,30 @@ public class SmtpManager extends AbstractManager { return mp; } + private void sendEmail(final String to, byte[] content, String contentType) { + try { + final String encoding = getEncoding(content, contentType); + final byte[] encodedBytes = encodeContentToBytes(content, encoding); + final InternetHeaders headers = getHeaders(contentType, encoding); + final MimeMultipart mp = getMimeMultipart(encodedBytes, headers); + sendMultipartMessage(message, mp); + } catch (final MessagingException | IOException | RuntimeException e) { + logError("Caught exception while sending e-mail notification.", e); + throw new LoggingException("Error occurred while sending email", e); + } + } + + private void sendEvents(final Layout layout, final List events, final String to) { + try { + final byte[] rawBytes = formatContentToBytes(events, layout); + final String contentType = layout.getContentType(); + sendEmail(to, rawBytes, contentType); + } catch (Exception e) { + logError("Caught exception while sending e-mail notification.", e); + throw new LoggingException("Error occurred while sending email", e); + } + } + /** * Send the contents of the cyclic buffer as an e-mail message. * @@ -290,32 +320,31 @@ public class SmtpManager extends AbstractManager { * The event that triggered the send. */ public void sendEvents(final Layout layout, final LogEvent appendEvent) { - if (message == null) { + if (message == null) connect(appendEvent); - } - try { - if (appendEvent.getMarker() instanceof MediaCubeMarker) { - MediaCubeMarker mcm = (MediaCubeMarker) appendEvent.getMarker(); - message.setRecipients(RecipientType.TO, mcm.getTo()); - } else - message.setRecipients(RecipientType.TO, defaultRecipient); - //final LogEvent[] priorEvents = buffer.removeAll(); - // LOG4J-310: log appendEvent even if priorEvents is empty - - final byte[] rawBytes = formatContentToBytes(null, appendEvent, layout); + //System.out.println(appendEvent.getMarker().getClass().getSimpleName() + " " + new String(layout.toByteArray(appendEvent))); + String to = defaultRecipient; + if (appendEvent.getMarker() instanceof MediaCubeFinishMarker) { + MediaCubeFinishMarker mcm = (MediaCubeFinishMarker) appendEvent.getMarker(); + storeSessionEvent(mcm, appendEvent); + sendSessionEvents(mcm, layout); + return; + } - final String contentType = layout.getContentType(); - final String encoding = getEncoding(rawBytes, contentType); - final byte[] encodedBytes = encodeContentToBytes(rawBytes, encoding); + if (appendEvent.getMarker() instanceof MediaCubeMarker) { + MediaCubeMarker mcm = (MediaCubeMarker) appendEvent.getMarker(); + if (mcm.getSessionID() == null) { + if (mcm.getTo() != null) + to = mcm.getTo(); + } else { + storeSessionEvent(mcm, appendEvent); + return; + } - final InternetHeaders headers = getHeaders(contentType, encoding); - final MimeMultipart mp = getMimeMultipart(encodedBytes, headers); - sendMultipartMessage(message, mp); - } catch (final MessagingException | IOException | RuntimeException e) { - logError("Caught exception while sending e-mail notification.", e); - throw new LoggingException("Error occurred while sending email", e); } + + sendEvents(layout, Arrays.asList(appendEvent), to); } protected void sendMultipartMessage(final MimeMessage msg, final MimeMultipart mp) throws MessagingException { @@ -326,20 +355,36 @@ public class SmtpManager extends AbstractManager { } } - protected void writeBuffer(final LogEvent[] priorEvents, final LogEvent appendEvent, final Layout layout, final OutputStream out) throws IOException { - // for (final LogEvent priorEvent : priorEvents) { - // final byte[] bytes = layout.toByteArray(priorEvent); - // out.write(bytes); - // } + private void sendSessionEvents(final MediaCubeMarker mcm, final Layout layout) { + String sessionID = mcm.getSessionID(); + List events = sessionEvents.get(sessionID); + if (events == null) + return; + String to = mcm.getTo() == null ? defaultRecipient : mcm.getTo(); + sendEvents(layout, events, to); + sessionEvents.remove(sessionID); + } - final byte[] bytes = layout.toByteArray(appendEvent); - out.write(bytes); + private void storeSessionEvent(MediaCubeMarker mcm, LogEvent appendEvent) { + String sessionID = mcm.getSessionID(); + List events = sessionEvents.get(sessionID); + if (events == null) { + events = new ArrayList<>(); + sessionEvents.put(sessionID, events); + } + events.add(appendEvent); + } + + protected void writeBuffer(List events, final Layout layout, final OutputStream out) throws IOException { + for (final LogEvent priorEvent : events) { + final byte[] bytes = layout.toByteArray(priorEvent); + out.write(bytes); + } } - private void writeContent(final LogEvent[] priorEvents, final LogEvent appendEvent, final Layout layout, final ByteArrayOutputStream out) - throws IOException { + private void writeContent(final List events, final Layout layout, final ByteArrayOutputStream out) throws IOException { writeHeader(layout, out); - writeBuffer(priorEvents, appendEvent, layout, out); + writeBuffer(events, layout, out); writeFooter(layout, out); } diff --git a/server/user.jobengine.executors/src/user/jobengine/server/steps/CopyForArchiveNEXIORecordingsStep.java b/server/user.jobengine.executors/src/user/jobengine/server/steps/CopyForArchiveNEXIORecordingsStep.java index ad37811f..958b2a9c 100644 --- a/server/user.jobengine.executors/src/user/jobengine/server/steps/CopyForArchiveNEXIORecordingsStep.java +++ b/server/user.jobengine.executors/src/user/jobengine/server/steps/CopyForArchiveNEXIORecordingsStep.java @@ -183,7 +183,7 @@ public class CopyForArchiveNEXIORecordingsStep extends JobStep { return null; DBObject rundown = octopusAPI.getRundown(scheduledStart); if (rundown == null) { - logger.warn(getMarker(), "A '{}' anyaghoz nem talllható tükör '{}' kezdéssel, ezért nem archiválható.", clipName, scheduledStart); + logger.error(getMarker(), "A '{}' anyaghoz nem található tükör '{}' kezdéssel, ezért nem archiválható.", clipName, scheduledStart); return null; } @@ -225,9 +225,6 @@ public class CopyForArchiveNEXIORecordingsStep extends JobStep { return null; RundownArchive result = new RundownArchive(); - long id = NoSQLUtils.asLong(rundown, IOctopusAPI.ID); - if (id == 0) - return null; String name = NoSQLUtils.asString(NoSQLUtils.asDBObject(rundown, IOctopusAPI.RUNDOWN_TYPE), IOctopusAPI.NAME); if (StringUtils.isBlank(name)) return null; @@ -237,7 +234,7 @@ public class CopyForArchiveNEXIORecordingsStep extends JobStep { return null; String start = CalendarUtils.toString(CalendarUtils.createCalendar(scheduledStart), SCHEDULED_FORMAT); result.setScheduleDate(scheduledStart); - result.setItemHouseId(String.valueOf(id)); + result.setItemHouseId(String.valueOf(rundownID)); result.setItemTitle(String.format("%s %s %s", start, name, channel)); StringBuilder sb = new StringBuilder(); diff --git a/server/user.jobengine.executors/src/user/jobengine/server/steps/FakeNoParamsStep.java b/server/user.jobengine.executors/src/user/jobengine/server/steps/FakeNoParamsStep.java index abd1b76f..f90b470d 100644 --- a/server/user.jobengine.executors/src/user/jobengine/server/steps/FakeNoParamsStep.java +++ b/server/user.jobengine.executors/src/user/jobengine/server/steps/FakeNoParamsStep.java @@ -3,7 +3,6 @@ package user.jobengine.server.steps; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; -import user.commons.MediaCubeMarker; import user.jobengine.server.IJobEngine; import user.jobengine.server.IJobRuntime; @@ -13,21 +12,21 @@ public class FakeNoParamsStep extends JobStep { @StepEntry public Object[] execute(IJobEngine jobEngine, IJobRuntime jobRuntime) throws InterruptedException { - MediaCubeMarker toMarker = new MediaCubeMarker("vasary@gmail.com"); - logger.info(toMarker, "A folyamat elindult"); - logger.warn(toMarker, "Egy WARN"); - logger.error(toMarker, "Egy ERROR"); - logger.warn(toMarker, "Egy WARN {}", "paraméterrel"); - logger.error(toMarker, "Egy ERROR {}", "paraméterrel"); + logger.info(jobRuntime.getMarker(), "A folyamat elindult"); + logger.warn(jobRuntime.getMarker(), "Egy WARN"); + logger.error(jobRuntime.getMarker(), "Egy ERROR"); + // logger.warn(toMarker, "Egy WARN {}", "paraméterrel"); + // logger.error(toMarker, "Egy ERROR {}", "paraméterrel"); for (int i = 0; i < count; i++) { if (!canContinue()) break; jobRuntime.incrementProgress((i + 1) * count); Thread.sleep(100); - logger.info(getMarker(), "Log markered" + i); + logger.info(jobRuntime.getMarker(), "Log markered" + i); + // logger.info(toMarker, "Log markered" + i); //logger.info("Progress {}", jobRuntime.getProgress()); } - logger.info(toMarker, "A folyamat véget ért"); + logger.info(jobRuntime.getFinishMarker(), "A folyamat véget ért"); //logger.info(LogCollector.FINISH); return null; } diff --git a/server/user.jobengine.executors/src/user/jobengine/server/steps/OctopusDataMiner.java b/server/user.jobengine.executors/src/user/jobengine/server/steps/OctopusDataMiner.java index f710e641..1f565ec5 100644 --- a/server/user.jobengine.executors/src/user/jobengine/server/steps/OctopusDataMiner.java +++ b/server/user.jobengine.executors/src/user/jobengine/server/steps/OctopusDataMiner.java @@ -7,7 +7,6 @@ import java.util.HashSet; import java.util.List; import java.util.Map; import java.util.Set; -import java.util.concurrent.ConcurrentHashMap; import javax.swing.event.EventListenerList; import javax.ws.rs.client.Entity; @@ -15,6 +14,7 @@ import javax.ws.rs.client.Invocation.Builder; import javax.ws.rs.core.MediaType; import javax.ws.rs.core.Response; +import org.apache.commons.lang.StringUtils; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.jboss.resteasy.client.jaxrs.BasicAuthentication; @@ -29,8 +29,6 @@ import com.ibm.nosql.json.api.BasicDBObject; import com.ibm.nosql.json.api.DB; import com.ibm.nosql.json.api.DBCollection; import com.ibm.nosql.json.api.DBCursor; -import com.ibm.nosql.json.api.QueryBuilder; -import com.ibm.nosql.json.api.WriteResult; import user.commons.CalendarUtils; import user.commons.ListUtils; @@ -38,19 +36,16 @@ import user.commons.nosql.NoSQLUtils; import user.commons.octopus.IOctopusAPI; import user.commons.remotestore.IProgressEventListener; import user.commons.remotestore.ProgressEvent; +import user.jobengine.server.steps.MetadataTypeDetector.MetadataType; public class OctopusDataMiner { + private static final Logger logger = LogManager.getLogger(); private static final String ARCHIVED = "archived"; private static final String FILTER = "filter"; - private static final String _TMP = "_tmp"; - private static final Logger logger = LogManager.getLogger(); private static final String LINEFEED = "\r\n"; private static final String SIMPLE_LINEFEED = "\n"; private static final String SAVING_STORY_ID = "Saving story {}"; - // private static final String SAVING_RUNDOWN = "Saving rundown : {} {}"; - // private static final String CHECKING_RUNDOWN = "Checking Rundown {} ({}/{})"; - private static final String FIELDS_STORYFOLDER_STORIES = "stories,Story.modified,Story.name,Story.id,Story.mosObjects,Story.script,Story.type,Story.format,Story.customColumns,CustomColumn.label,CustomColumn.value"; - private static final String FIELDS_RUNDOWN_STORIES = "slugs,Slug.story,Slug.position,Story.name,Story.id,Story.modified,Story.mosObjects,Story.script,Story.type,Story.format,Story.customColumns,CustomColumn.label,CustomColumn.value"; + private static final String FIELDS_STORIES = "id,name,modified,type,format,mosObjects,script,customColumns,CustomColumn.label,CustomColumn.value,reporters,User.longName,User.userName"; private static final String FIELDS_RUNDOWN = "id,name,modified,scheduledStart,channel,Channel.name,rundownType,RundownType.name"; private static final String FIELDS_RUNDOWN_STORYIDS = "id,slugs,Slug.storyId,Slug.position"; private static final String FIELDS_STORYFOLDER = "id,name,modified"; @@ -59,7 +54,6 @@ public class OctopusDataMiner { private static final String OCTOPUS_DEVICE_NAME = "Octopus-Device-Name"; private static final String OCTOPUS_DEVICE_ID = "Octopus-Device-Id"; private static final String FIELDS = "fields"; - private static final String CHECKING_STORY_FOLDER = "Checking StoryFolder %s (%d/%d)"; private static final String EXIT = "Exit"; private static final String RESULT = "result"; private static final String STORY_FOLDER = "StoryFolder"; @@ -67,6 +61,7 @@ public class OctopusDataMiner { private static final String FINISHED = "Finished"; private static final String STARTING = "Starting"; private static final String MOSOBJECT = "Bejátszó: "; + private static final Object STORY = "Story"; private DB db; private ResteasyWebTarget webTarget; @@ -82,11 +77,10 @@ public class OctopusDataMiner { private Map currentRundowns; private Map currentFolder; private Map currentStories; - private String RUNDOWN_COLLECTION; - private String FOLDER_COLLECTION; - private String STORY_COLLECTION; private boolean includeArchived; private Calendar zeroDate = CalendarUtils.createCalendar(2017, 11, 4); + private int objectCount; + private int currentObjectIndex; public OctopusDataMiner() { db = NoSQLUtils.getNoSQLDB(); @@ -117,12 +111,15 @@ public class OctopusDataMiner { List stories = NoSQLUtils.asList(storyFolderWithStoryIds, IOctopusAPI.STORIES); if (stories == null) continue; - if (!folderIDs.contains(storyFolderId)) - folderIDs.add(storyFolderId); + + folderIDs.add(storyFolderId); + long position = 1; for (BasicDBObject story : stories) { long storyId = story.getLong(IOctopusAPI.ID); + storyIDs.add(storyId); + BasicDBList references = result.get(storyId); if (references == null) { references = new BasicDBList(); @@ -146,13 +143,15 @@ public class OctopusDataMiner { if (slugs == null) continue; - if (!rundownIDs.contains(rundownId)) - rundownIDs.add(rundownId); + rundownIDs.add(rundownId); for (BasicDBObject slug : slugs) { if (!slug.containsKey(IOctopusAPI.STORYID)) continue; long storyId = slug.getLong(IOctopusAPI.STORYID); + + storyIDs.add(storyId); + BasicDBList references = result.get(storyId); if (references == null) { references = new BasicDBList(); @@ -168,26 +167,9 @@ public class OctopusDataMiner { } public void clear() { - db.getCollection(RUNDOWN_COLLECTION).remove(); - db.getCollection(STORY_COLLECTION).remove(); - db.getCollection(FOLDER_COLLECTION).remove(); - db.getCollection(IOctopusAPI.TIME_COLLECTION_NAME).remove(); - } - - private String concatParentsToStoryFolder(BasicDBObject actual, String name) { - - String fields = "name,id,parent"; - Response response = query("StoryFolder/" + actual.getLong("id"), fields).get(); - String json = response.readEntity(String.class); - BasicDBObject resultObject = (BasicDBObject) JSONUtil.jsonToDbObject(json); - BasicDBObject res = (BasicDBObject) resultObject.get("result"); - BasicDBObject parent = (BasicDBObject) res.get("parent"); - if (parent == null || parent.isEmpty()) - return name; - String parentName = parent.getString(IOctopusAPI.NAME); - String actualName = actual.getString(IOctopusAPI.NAME); - String newName = String.format("%s/%s", parentName, actualName); - return concatParentsToStoryFolder(parent, newName); + db.getCollection(IOctopusAPI.RUNDOWN_COLLECTION).remove(); + db.getCollection(IOctopusAPI.FOLDER_COLLECTION).remove(); + db.getCollection(IOctopusAPI.STORY_COLLECTION).remove(); } private void deleteDiff(Set oldIDs, Set newIDs, String collectionName) { @@ -202,82 +184,13 @@ public class OctopusDataMiner { } } - private void deleteOrphanRundowns() { - try { - DBCollection collection = db.getCollection(RUNDOWN_COLLECTION); - BasicDBObject query = (BasicDBObject) QueryBuilder.start().put(IOctopusAPI.ID).notIn(storyRundowns.keySet().toArray()).get(); - WriteResult res = collection.remove(query); - logger.trace(String.format("Deleted orphan rundowns: %d", res.getN())); - } catch (Exception e) { - logger.error(e); - } - } - - private void deleteOrphanStories() { - try { - DBCollection collection = db.getCollection(STORY_COLLECTION); - BasicDBObject query = (BasicDBObject) QueryBuilder.start().put("id").notIn(storyIDs.toArray()).get(); - WriteResult res = collection.remove(query); - logger.trace(String.format("Deleted orphan stories: %d", res.getN())); - } catch (Exception e) { - logger.error(e); - } - } - - private void deleteOrphanStoryFolders() { - try { - DBCollection collection = db.getCollection(RUNDOWN_COLLECTION); - BasicDBObject query = (BasicDBObject) QueryBuilder.start().put(IOctopusAPI.ID).notIn(storyStoryFolders.keySet().toArray()).get(); - WriteResult res = collection.remove(query); - logger.trace(String.format("Deleted orphan rundowns: %d", res.getN())); - } catch (Exception e) { - logger.error(e); - } - } - - private void ensureIndexes() { - DBCollection collection = db.getCollection(FOLDER_COLLECTION); - if (collection.count() == 0) - collection.ensureIndex(IOctopusAPI.ID); - collection = db.getCollection(RUNDOWN_COLLECTION); - if (collection.count() == 0) { - collection.ensureIndex(IOctopusAPI.ID); - collection.ensureIndex(IOctopusAPI.SCHEDULED_START); - } - collection = db.getCollection(STORY_COLLECTION); - if (collection.count() == 0) - collection.ensureIndex(IOctopusAPI.ID); - } - public void execute(boolean includeArchived) throws Exception { this.includeArchived = includeArchived; logger.trace(STARTING); - - // String MAIN_RUNDOWN_COLLECTION = IOctopusAPI.RUNDOWN_COLLECTION; - // String MAIN_FOLDER_COLLECTION = IOctopusAPI.FOLDER_COLLECTION; - // String MAIN_STORY_COLLECTION = IOctopusAPI.STORY_COLLECTION; - - // RUNDOWN_COLLECTION = MAIN_RUNDOWN_COLLECTION + _TMP; - // FOLDER_COLLECTION = MAIN_FOLDER_COLLECTION + _TMP; - // STORY_COLLECTION = MAIN_STORY_COLLECTION + _TMP; - // - // try { - // db.getCollection(RUNDOWN_COLLECTION).drop(); - // db.getCollection(FOLDER_COLLECTION).drop(); - // db.getCollection(STORY_COLLECTION).drop(); - // } catch (Exception e) { - // logger.catching(e); - // throw e; - // } - - RUNDOWN_COLLECTION = IOctopusAPI.RUNDOWN_COLLECTION; - FOLDER_COLLECTION = IOctopusAPI.FOLDER_COLLECTION; - STORY_COLLECTION = IOctopusAPI.STORY_COLLECTION; - - //NoSQLUtils.asList((BasicDBList) - currentRundowns = getCurrentIDs(RUNDOWN_COLLECTION); - currentFolder = getCurrentIDs(FOLDER_COLLECTION); - currentStories = getCurrentIDs(STORY_COLLECTION); + //current = korábbi szinkronizálás + currentRundowns = getCurrentIDs(IOctopusAPI.RUNDOWN_COLLECTION); + currentFolder = getCurrentIDs(IOctopusAPI.FOLDER_COLLECTION); + currentStories = getCurrentIDs(IOctopusAPI.STORY_COLLECTION); BasicDBList rundowns = null; BasicDBList storyFolders = null; @@ -289,48 +202,16 @@ public class OctopusDataMiner { throw e; } - processRundowns(rundowns); - processStoryFolders(storyFolders); - - deleteDiff(currentRundowns.keySet(), rundownIDs, RUNDOWN_COLLECTION); - deleteDiff(currentFolder.keySet(), folderIDs, FOLDER_COLLECTION); - deleteDiff(currentStories.keySet(), storyIDs, STORY_COLLECTION); - // deleteOrphanRundowns(); - // deleteOrphanStoryFolders(); - // deleteOrphanStories(); - - //a sorrend fontos ! - // updateDiff(MAIN_STORY_COLLECTION, STORY_COLLECTION, IOctopusAPI.ID); - // updateDiff(MAIN_RUNDOWN_COLLECTION, RUNDOWN_COLLECTION, IOctopusAPI.ID); - // updateDiff(MAIN_FOLDER_COLLECTION, FOLDER_COLLECTION, IOctopusAPI.ID); - // - // updateDeleteDiff(MAIN_RUNDOWN_COLLECTION, RUNDOWN_COLLECTION, IOctopusAPI.ID); - // updateDeleteDiff(MAIN_FOLDER_COLLECTION, FOLDER_COLLECTION, IOctopusAPI.ID); - // updateDeleteDiff(MAIN_STORY_COLLECTION, STORY_COLLECTION, IOctopusAPI.ID); - - // try { - // db.getCollection(RUNDOWN_COLLECTION).rename(MAIN_RUNDOWN_COLLECTION, true); - // db.getCollection(FOLDER_COLLECTION).rename(MAIN_FOLDER_COLLECTION, true); - // db.getCollection(STORY_COLLECTION).rename(MAIN_STORY_COLLECTION, true); - // } catch (Exception e) { - // logger.catching(e); - // throw e; - // } - - // logger.info("Activate"); + objectCount = rundownIDs.size() + folderIDs.size() + storyIDs.size(); - // try { - // long ts = new Date().getTime(); - // db.getCollection(RUNDOWN_COLLECTION).exportFile(String.format("/opt/mediacube-test/log/%s-%s.js", RUNDOWN_COLLECTION, ts)); - // db.getCollection(FOLDER_COLLECTION).exportFile(String.format("/opt/mediacube-test/log/%s-%s.js", FOLDER_COLLECTION, ts)); - // db.getCollection(STORY_COLLECTION).exportFile(String.format("/opt/mediacube-test/log/%s-%s.js", STORY_COLLECTION, ts)); - // } catch (Exception e) { - // logger.catching(e); - // throw e; - // } + storeStories(); + storeRundowns(rundowns); + storeStoryFolders(storyFolders); + deleteDiff(currentRundowns.keySet(), rundownIDs, IOctopusAPI.RUNDOWN_COLLECTION); + deleteDiff(currentFolder.keySet(), folderIDs, IOctopusAPI.FOLDER_COLLECTION); + deleteDiff(currentStories.keySet(), storyIDs, IOctopusAPI.STORY_COLLECTION); logger.trace(FINISHED); - //throw new Exception("Mérés"); } private String extractContent(BasicDBObject content) { @@ -398,6 +279,8 @@ public class OctopusDataMiner { List mosObjects = NoSQLUtils.asList(story, IOctopusAPI.MOS_OBJECTS); if (mosObjects == null) return null; + + Map mosLabels = extractScriptMosObjectIDs(story); BasicDBList result = null; for (BasicDBObject mosObject : mosObjects) { if (!mosObject.containsKey(IOctopusAPI.MOS_ID)) @@ -405,8 +288,19 @@ public class OctopusDataMiner { String mosId = mosObject.getString(IOctopusAPI.MOS_ID); if (!IOctopusAPI.NEXIO_MOS.equals(mosId)) continue; + String objId = mosObject.getString(IOctopusAPI.OBJ_ID); + if (objId == null) + continue; + + MetadataType metadataType = MetadataTypeDetector.GuessMetadataType(objId); + if (!MetadataType.OCTOPUSPLACEHOLDER.equals(metadataType) && !MetadataType.OCTOPUSSTORY.equals(metadataType)) + continue; if (result == null) result = new BasicDBList(); + + String label = mosLabels.get(objId); + if (StringUtils.isNotBlank(label)) + mosObject.append(IOctopusAPI.LABEL, label); result.add(mosObject); } return result; @@ -438,6 +332,49 @@ public class OctopusDataMiner { return sb.length() == 0 ? null : sb.toString(); } + private Map extractScriptMosObjectIDs(BasicDBObject story) { + BasicDBObject script = NoSQLUtils.asDBObject(story, IOctopusAPI.SCRIPT); + if (script == null || script.isEmpty()) + return null; + + List body = NoSQLUtils.asList(script, IOctopusAPI.BODY); + if (body == null || body.size() == 0) + return null; + + Map result = new HashMap<>(); + + for (BasicDBObject bodyItem : body) { + String label = bodyItem.getString(IOctopusAPI.LABEL); + if (StringUtils.isBlank(label)) + continue; + BasicDBObject clip = NoSQLUtils.asDBObject(bodyItem, IOctopusAPI.CLIP); + if (clip == null) + continue; + BasicDBObject obj = NoSQLUtils.asDBObject(clip, IOctopusAPI.OBJECT); + if (obj == null) + continue; + String mosId = obj.getString(IOctopusAPI.MOS_ID); + if (!IOctopusAPI.NEXIO_MOS.equals(mosId)) + continue; + String objId = obj.getString(IOctopusAPI.OBJ_ID); + if (objId == null) + continue; + result.put(mosId, label); + + } + return result; + } + + private void fireProgressEvent() { + currentObjectIndex++; + logger.info("currentObjectIndex {}", currentObjectIndex); + int progress = currentObjectIndex * 100 / objectCount; + if (progress != progressEvent.getProgress()) { + progressEvent.setProgress(progress); + fireProgressEvent(progressEvent); + } + } + private void fireProgressEvent(ProgressEvent evt) { logger.debug("Progress changed to " + evt.getProgress() + "%"); if (progressListenerList == null) @@ -470,36 +407,6 @@ public class OctopusDataMiner { return result; } - private void processRundowns(BasicDBList rundowns) throws Exception { - if (rundowns == null || rundowns.size() == 0) { - progressEvent.setProgress(50); - fireProgressEvent(progressEvent); - } else { - logger.info("Process rundowns"); - try { - storeRundowns(rundowns, null); - } catch (Exception e) { - logger.catching(e); - throw e; - } - } - } - - private void processStoryFolders(BasicDBList storyFolders) throws Exception { - if (storyFolders == null || storyFolders.size() == 0) { - progressEvent.setProgress(100); - fireProgressEvent(progressEvent); - } else { - logger.info("Process story folders"); - try { - storeStoryFolders(storyFolders, null); - } catch (Exception e) { - logger.catching(e); - throw e; - } - } - } - private Builder query(String path, String fields) { ResteasyWebTarget target = webTarget.path(path).queryParam(FIELDS, fields); Builder result = target.request().header(OCTOPUS_DEVICE_ID, apiUser).header(OCTOPUS_DEVICE_NAME, apiPwd); @@ -557,6 +464,20 @@ public class OctopusDataMiner { return result; } + private BasicDBObject queryStory(long storyID) { + logger.trace(ENTER); + BasicDBObject result = null; + Response response = query(String.format("%s/%d", STORY, storyID), FIELDS_STORIES).get(); + String json = response.readEntity(String.class); + BasicDBObject resultObject = (BasicDBObject) JSONUtil.jsonToDbObject(json); + if (resultObject == null) + logger.error("Story {} is not available", storyID); + else + result = NoSQLUtils.asDBObject(resultObject, RESULT); + logger.trace(EXIT); + return result; + } + private BasicDBObject queryStoryFolder(BasicDBObject storyFolder, String fields) { logger.trace(ENTER); BasicDBObject result = null; @@ -609,50 +530,47 @@ public class OctopusDataMiner { logger.trace(ENTER); String name = rundown.containsKey(IOctopusAPI.NAME) ? rundown.getString(IOctopusAPI.NAME) : null; logger.debug("Storing rundown {} {}", name, rundown.get(IOctopusAPI.SCHEDULED_START)); - BasicDBObject rundownWithStories = queryRundown(rundown, FIELDS_RUNDOWN_STORIES); - if (rundownWithStories != null) { - Date scheduledStart = toDate(rundown, IOctopusAPI.SCHEDULED_START); - if (scheduledStart != null && scheduledStart.after(zeroDate.getTime())) { - BasicDBList stories = NoSQLUtils.asDBList(rundownWithStories, IOctopusAPI.SLUGS); - if (stories != null) - storeRundownStories(stories); - rundown.put(IOctopusAPI.SCHEDULED_START, toDate(rundown, IOctopusAPI.SCHEDULED_START)); - rundown.put(IOctopusAPI.MODIFIED, toDate(rundown, IOctopusAPI.MODIFIED)); - DBCollection collection = db.getCollection(RUNDOWN_COLLECTION); - long rundownID = NoSQLUtils.asLong(rundown, IOctopusAPI.ID); - setObjectID(currentRundowns, rundownID, rundown); - collection.save(rundown); - } + Date scheduledStart = toDate(rundown, IOctopusAPI.SCHEDULED_START); + if (scheduledStart != null && scheduledStart.after(zeroDate.getTime())) { + rundown.put(IOctopusAPI.SCHEDULED_START, toDate(rundown, IOctopusAPI.SCHEDULED_START)); + rundown.put(IOctopusAPI.MODIFIED, toDate(rundown, IOctopusAPI.MODIFIED)); + DBCollection collection = db.getCollection(IOctopusAPI.RUNDOWN_COLLECTION); + long rundownID = NoSQLUtils.asLong(rundown, IOctopusAPI.ID); + setObjectID(currentRundowns, rundownID, rundown); + collection.save(rundown); } - logger.trace(EXIT); + logger.trace(ENTER); } - private void storeRundowns(BasicDBList rundowns, Date lastUpdateTime) { + private void storeRundowns(BasicDBList rundowns) { logger.trace(ENTER); List rundownsList = NoSQLUtils.asList(rundowns); - int idx = 1; for (BasicDBObject rundown : rundownsList) { - //logger.info(CHECKING_RUNDOWN, rundown.getLong(IOctopusAPI.ID), rundownsList.size(), idx); - storeRundown(rundown); - int progress = idx * 50 / rundownsList.size(); - if (progress - progressEvent.getProgress() > 0) { - progressEvent.setProgress(progress); - fireProgressEvent(progressEvent); + try { + storeRundown(rundown); + } catch (Exception e) { + logger.catching(e); + throw e; } - - idx++; + fireProgressEvent(); } - logger.trace(EXIT); + logger.trace(ENTER); } - private void storeRundownStories(BasicDBList slugs) { + private void storeStories() { logger.trace(ENTER); - List slugsList = NoSQLUtils.asList(slugs); - for (BasicDBObject slug : slugsList) { - if (slug.containsKey(IOctopusAPI.STORY)) - storeStory((BasicDBObject) slug.get(IOctopusAPI.STORY)); + for (long storyID : storyIDs) { + try { + BasicDBObject story = queryStory(storyID); + if (story != null) + storeStory(story); + } catch (Exception e) { + logger.catching(e); + throw e; + } + fireProgressEvent(); } - logger.trace(EXIT); + logger.trace(ENTER); } private void storeStory(BasicDBObject story) { @@ -662,16 +580,12 @@ public class OctopusDataMiner { return; } long storyID = story.getLong(IOctopusAPI.ID); - boolean isContains = storyIDs.contains(storyID); - if (isContains) - return; - storyIDs.add(storyID); + story.put(IOctopusAPI.MODIFIED, toDate(story, IOctopusAPI.MODIFIED)); BasicDBList rundownRef = storyRundowns.get(storyID); BasicDBList storyFolderRef = storyStoryFolders.get(storyID); BasicDBList modifiedMOS = extractRelevantMOSObjects(story); - - DBCollection collection = db.getCollection(STORY_COLLECTION); + DBCollection collection = db.getCollection(IOctopusAPI.STORY_COLLECTION); if (rundownRef != null) story.put(IOctopusAPI.REF_RUNDOWN, rundownRef); @@ -697,49 +611,33 @@ public class OctopusDataMiner { logger.trace(EXIT); } - private void storeStoryFolder(BasicDBObject storyFolder, Date lastUpdateTime) { + private void storeStoryFolder(BasicDBObject storyFolder) { logger.trace(ENTER); - BasicDBObject storyFoldersWithStories = queryStoryFolder(storyFolder, FIELDS_STORYFOLDER_STORIES); - if (storyFoldersWithStories != null) { - BasicDBList stories = NoSQLUtils.asDBList(storyFoldersWithStories, IOctopusAPI.STORIES); - if (stories != null) - storeStoryFolderStories(stories); - storyFolder.put(IOctopusAPI.MODIFIED, toDate(storyFolder, IOctopusAPI.MODIFIED)); - DBCollection collection = db.getCollection(FOLDER_COLLECTION); - String name = storyFolder.getString(IOctopusAPI.NAME); - logger.debug("Storing story folder {}", name); - long folderID = NoSQLUtils.asLong(storyFolder, IOctopusAPI.ID); - setObjectID(currentFolder, folderID, storyFolder); - collection.save(storyFolder); - } + storyFolder.put(IOctopusAPI.MODIFIED, toDate(storyFolder, IOctopusAPI.MODIFIED)); + DBCollection collection = db.getCollection(IOctopusAPI.FOLDER_COLLECTION); + String name = storyFolder.getString(IOctopusAPI.NAME); + logger.debug("Storing story folder {}", name); + long folderID = NoSQLUtils.asLong(storyFolder, IOctopusAPI.ID); + setObjectID(currentFolder, folderID, storyFolder); + collection.save(storyFolder); logger.trace(EXIT); } - private void storeStoryFolders(BasicDBList storyFolders, Date lastUpdateTime) { + private void storeStoryFolders(BasicDBList storyFolders) { logger.trace(ENTER); List storyFolderList = NoSQLUtils.asList(storyFolders); - int idx = 1; for (BasicDBObject storyFolder : storyFolderList) { - logger.debug(String.format(CHECKING_STORY_FOLDER, storyFolder.getLong(IOctopusAPI.ID), storyFolderList.size(), idx)); - storeStoryFolder(storyFolder, lastUpdateTime); - int progress = 50 + (idx * 50 / storyFolderList.size()); - if (progress - progressEvent.getProgress() > 0) { - progressEvent.setProgress(progress); - fireProgressEvent(progressEvent); + try { + storeStoryFolder(storyFolder); + } catch (Exception e) { + logger.catching(e); + throw e; } - idx++; + fireProgressEvent(); } logger.trace(EXIT); } - private void storeStoryFolderStories(BasicDBList stories) { - logger.trace(ENTER); - List list = NoSQLUtils.asList(stories); - for (BasicDBObject story : list) - storeStory(story); - logger.trace(EXIT); - } - private Date toDate(BasicDBObject obj, String name) { Date result = null; if (obj.containsKey(name)) { @@ -753,96 +651,4 @@ public class OctopusDataMiner { return result; } - private void updateDeleteDiff(String oldCollectionName, String newCollectionName, String idFieldName) { - DBCollection oldCollection = db.getCollection(oldCollectionName); - DBCollection newCollection = db.getCollection(newCollectionName); - DBCursor oldCollectionCursor = oldCollection.find(new BasicDBObject(), new BasicDBObject(idFieldName, 1)); - if (!oldCollectionCursor.hasNext()) { - logger.error("{} collection is empty", newCollectionName); - return; - } - List oldItems = ListUtils.cast(oldCollectionCursor.toArray()); - - DBCursor newCollectionCursor = newCollection.find(); - ConcurrentHashMap newItems = null; - if (newCollectionCursor.hasNext()) { - List newList = ListUtils.cast(newCollectionCursor.toArray()); - newItems = ListUtils.map(newList, item -> item.getLong(IOctopusAPI.ID)); - } - if (newItems == null) - newItems = new ConcurrentHashMap<>(); - - for (BasicDBObject oldItem : oldItems) { - if (oldItem == null) { - logger.error("Item is null"); - continue; - } - if (!oldItem.containsKey(idFieldName)) { - logger.error("{} is null", idFieldName); - continue; - } - long id = oldItem.getLong(idFieldName); - BasicDBObject newItem = newItems.get(id); - if (newItem == null) { - //remove - logger.info("Deleting {}", oldItem.toPrettyString(null)); - oldCollection.remove(new BasicDBObject(idFieldName, id)); - } - } - } - - private void updateDiff(String oldCollectionName, String newCollectionName, String idFieldName) { - DBCollection oldCollection = db.getCollection(oldCollectionName); - DBCollection newCollection = db.getCollection(newCollectionName); - DBCursor newCollectionCursor = newCollection.find(); - if (!newCollectionCursor.hasNext()) { - logger.error("{} collection is empty", newCollectionName); - return; - } - - List newItems = ListUtils.cast(newCollectionCursor.toArray()); - - DBCursor oldCollectionCursor = oldCollection.find(); - ConcurrentHashMap oldItems = null; - if (oldCollectionCursor.hasNext()) { - List oldList = ListUtils.cast(oldCollectionCursor.toArray()); - oldItems = ListUtils.map(oldList, item -> item.getLong(IOctopusAPI.ID)); - } - if (oldItems == null) - oldItems = new ConcurrentHashMap<>(); - for (BasicDBObject newItem : newItems) { - if (newItem == null) { - logger.error("Item is null"); - continue; - } - if (!newItem.containsKey(idFieldName)) { - logger.error("{} is null", idFieldName); - continue; - } - long id = newItem.getLong(idFieldName); - BasicDBObject oldItem = oldItems.get(id); - boolean save = false; - if (oldItem == null) { - //save - save = true; - } else { - //compare - Object oldDbId = oldItem.getID(); - newItem.remove(IOctopusAPI._ID); - oldItem.remove(IOctopusAPI._ID); - - if (!newItem.equals(oldItem)) { - //save - newItem.put(IOctopusAPI._ID, oldDbId); - save = true; - } - } - - if (save) { - logger.info("Saving to {} item {}", oldCollectionName, id); - oldCollection.save(newItem); - } - } - } - } diff --git a/server/user.jobengine.executors/src/user/jobengine/server/steps/OctopusDataMinerStable.java b/server/user.jobengine.executors/src/user/jobengine/server/steps/OctopusDataMinerStable.java new file mode 100644 index 00000000..74969aef --- /dev/null +++ b/server/user.jobengine.executors/src/user/jobengine/server/steps/OctopusDataMinerStable.java @@ -0,0 +1,848 @@ +package user.jobengine.server.steps; + +import java.util.Calendar; +import java.util.Date; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.Set; +import java.util.concurrent.ConcurrentHashMap; + +import javax.swing.event.EventListenerList; +import javax.ws.rs.client.Entity; +import javax.ws.rs.client.Invocation.Builder; +import javax.ws.rs.core.MediaType; +import javax.ws.rs.core.Response; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.jboss.resteasy.client.jaxrs.BasicAuthentication; +import org.jboss.resteasy.client.jaxrs.ResteasyClient; +import org.jboss.resteasy.client.jaxrs.ResteasyClientBuilder; +import org.jboss.resteasy.client.jaxrs.ResteasyWebTarget; +import org.joda.time.DateTime; + +import com.ibm.nosql.json.JSONUtil; +import com.ibm.nosql.json.api.BasicDBList; +import com.ibm.nosql.json.api.BasicDBObject; +import com.ibm.nosql.json.api.DB; +import com.ibm.nosql.json.api.DBCollection; +import com.ibm.nosql.json.api.DBCursor; +import com.ibm.nosql.json.api.QueryBuilder; +import com.ibm.nosql.json.api.WriteResult; + +import user.commons.CalendarUtils; +import user.commons.ListUtils; +import user.commons.nosql.NoSQLUtils; +import user.commons.octopus.IOctopusAPI; +import user.commons.remotestore.IProgressEventListener; +import user.commons.remotestore.ProgressEvent; + +public class OctopusDataMinerStable { + private static final String ARCHIVED = "archived"; + private static final String FILTER = "filter"; + private static final String _TMP = "_tmp"; + private static final Logger logger = LogManager.getLogger(); + private static final String LINEFEED = "\r\n"; + private static final String SIMPLE_LINEFEED = "\n"; + private static final String SAVING_STORY_ID = "Saving story {}"; + // private static final String SAVING_RUNDOWN = "Saving rundown : {} {}"; + // private static final String CHECKING_RUNDOWN = "Checking Rundown {} ({}/{})"; + private static final String FIELDS_STORYFOLDER_STORIES = "stories,Story.modified,Story.name,Story.id,Story.mosObjects,Story.script,Story.type,Story.format,Story.customColumns,CustomColumn.label,CustomColumn.value"; + private static final String FIELDS_RUNDOWN_STORIES = "slugs,Slug.story,Slug.position,Story.name,Story.id,Story.modified,Story.mosObjects,Story.script,Story.type,Story.format,Story.customColumns,CustomColumn.label,CustomColumn.value"; + private static final String FIELDS_RUNDOWN = "id,name,modified,scheduledStart,channel,Channel.name,rundownType,RundownType.name"; + private static final String FIELDS_RUNDOWN_STORYIDS = "id,slugs,Slug.storyId,Slug.position"; + private static final String FIELDS_STORYFOLDER = "id,name,modified"; + private static final String FIELDS_STORYFOLDER_STORYIDS = "id,stories,Story.id"; + private static final String RUNDOWN = "Rundown"; + private static final String OCTOPUS_DEVICE_NAME = "Octopus-Device-Name"; + private static final String OCTOPUS_DEVICE_ID = "Octopus-Device-Id"; + private static final String FIELDS = "fields"; + private static final String CHECKING_STORY_FOLDER = "Checking StoryFolder %s (%d/%d)"; + private static final String EXIT = "Exit"; + private static final String RESULT = "result"; + private static final String STORY_FOLDER = "StoryFolder"; + private static final String ENTER = "Enter"; + private static final String FINISHED = "Finished"; + private static final String STARTING = "Starting"; + private static final String MOSOBJECT = "Bejátszó: "; + + private DB db; + private ResteasyWebTarget webTarget; + private String apiUser; + private String apiPwd; + private HashSet storyIDs = new HashSet<>(); + private HashSet folderIDs = new HashSet<>(); + private HashSet rundownIDs = new HashSet<>(); + private EventListenerList progressListenerList; + private ProgressEvent progressEvent = new ProgressEvent(this, 0); + private Map storyRundowns; + private Map storyStoryFolders; + private Map currentRundowns; + private Map currentFolder; + private Map currentStories; + private String RUNDOWN_COLLECTION; + private String FOLDER_COLLECTION; + private String STORY_COLLECTION; + private boolean includeArchived; + private Calendar zeroDate = CalendarUtils.createCalendar(2017, 11, 4); + + public OctopusDataMinerStable() { + db = NoSQLUtils.getNoSQLDB(); + + String apiAddress = System.getProperty("jobengine.octopus.api.address"); + apiUser = System.getProperty("jobengine.octopus.api.user"); + apiPwd = System.getProperty("jobengine.octopus.api.password"); + + // ResteasyClient client = new ResteasyClientBuilder().register(JacksonJsonProvider.class).build(); + ResteasyClient client = new ResteasyClientBuilder().build(); + webTarget = client.target(apiAddress).register(new BasicAuthentication(apiUser, apiPwd)); + } + + public void addProgressListener(IProgressEventListener listener) { + if (progressListenerList == null) + progressListenerList = new EventListenerList(); + progressListenerList.add(IProgressEventListener.class, listener); + } + + private Map buildFolderReferences(BasicDBList storyFolders) { + Map result = new HashMap<>(); + List storyFolderList = NoSQLUtils.asList(storyFolders); + for (BasicDBObject storyFolder : storyFolderList) { + if (storyFolder == null || !storyFolder.containsKey(IOctopusAPI.ID)) + continue; + BasicDBObject storyFolderWithStoryIds = queryStoryFolder(storyFolder, FIELDS_STORYFOLDER_STORYIDS); + long storyFolderId = storyFolderWithStoryIds.getLong(IOctopusAPI.ID); + List stories = NoSQLUtils.asList(storyFolderWithStoryIds, IOctopusAPI.STORIES); + if (stories == null) + continue; + if (!folderIDs.contains(storyFolderId)) + folderIDs.add(storyFolderId); + long position = 1; + for (BasicDBObject story : stories) { + long storyId = story.getLong(IOctopusAPI.ID); + + BasicDBList references = result.get(storyId); + if (references == null) { + references = new BasicDBList(); + result.put(storyId, references); + } + references.add(new BasicDBObject(IOctopusAPI.ID, storyFolderId).append(IOctopusAPI.POSITION, position++)); + } + } + return result; + } + + private Map buildRundownReferences(BasicDBList rundowns) { + Map result = new HashMap<>(); + List rundownsList = NoSQLUtils.asList(rundowns); + for (BasicDBObject rundown : rundownsList) { + if (rundown == null || !rundown.containsKey(IOctopusAPI.ID)) + continue; + BasicDBObject rundownWithStoryids = queryRundown(rundown, FIELDS_RUNDOWN_STORYIDS); + long rundownId = rundownWithStoryids.getLong(IOctopusAPI.ID); + List slugs = NoSQLUtils.asList(rundownWithStoryids, IOctopusAPI.SLUGS); + if (slugs == null) + continue; + + if (!rundownIDs.contains(rundownId)) + rundownIDs.add(rundownId); + + for (BasicDBObject slug : slugs) { + if (!slug.containsKey(IOctopusAPI.STORYID)) + continue; + long storyId = slug.getLong(IOctopusAPI.STORYID); + BasicDBList references = result.get(storyId); + if (references == null) { + references = new BasicDBList(); + result.put(storyId, references); + } + long position = slug.getLong(IOctopusAPI.POSITION); + if (slug.containsKey(IOctopusAPI.POSITION)) + position = slug.getLong(IOctopusAPI.POSITION); + references.add(new BasicDBObject(IOctopusAPI.ID, rundownId).append(IOctopusAPI.POSITION, position)); + } + } + return result; + } + + public void clear() { + db.getCollection(RUNDOWN_COLLECTION).remove(); + db.getCollection(STORY_COLLECTION).remove(); + db.getCollection(FOLDER_COLLECTION).remove(); + db.getCollection(IOctopusAPI.TIME_COLLECTION_NAME).remove(); + } + + private String concatParentsToStoryFolder(BasicDBObject actual, String name) { + + String fields = "name,id,parent"; + Response response = query("StoryFolder/" + actual.getLong("id"), fields).get(); + String json = response.readEntity(String.class); + BasicDBObject resultObject = (BasicDBObject) JSONUtil.jsonToDbObject(json); + BasicDBObject res = (BasicDBObject) resultObject.get("result"); + BasicDBObject parent = (BasicDBObject) res.get("parent"); + if (parent == null || parent.isEmpty()) + return name; + String parentName = parent.getString(IOctopusAPI.NAME); + String actualName = actual.getString(IOctopusAPI.NAME); + String newName = String.format("%s/%s", parentName, actualName); + return concatParentsToStoryFolder(parent, newName); + } + + private void deleteDiff(Set oldIDs, Set newIDs, String collectionName) { + if (oldIDs == null || oldIDs.size() == 0) + return; + if (newIDs != null && newIDs.size() > 0) + oldIDs.removeAll(newIDs); + DBCollection collection = db.getCollection(collectionName); + for (long id : oldIDs) { + logger.info("Deleting {} from {}", id, collectionName); + collection.remove(new BasicDBObject(IOctopusAPI.ID, id)); + } + } + + private void deleteOrphanRundowns() { + try { + DBCollection collection = db.getCollection(RUNDOWN_COLLECTION); + BasicDBObject query = (BasicDBObject) QueryBuilder.start().put(IOctopusAPI.ID).notIn(storyRundowns.keySet().toArray()).get(); + WriteResult res = collection.remove(query); + logger.trace(String.format("Deleted orphan rundowns: %d", res.getN())); + } catch (Exception e) { + logger.error(e); + } + } + + private void deleteOrphanStories() { + try { + DBCollection collection = db.getCollection(STORY_COLLECTION); + BasicDBObject query = (BasicDBObject) QueryBuilder.start().put("id").notIn(storyIDs.toArray()).get(); + WriteResult res = collection.remove(query); + logger.trace(String.format("Deleted orphan stories: %d", res.getN())); + } catch (Exception e) { + logger.error(e); + } + } + + private void deleteOrphanStoryFolders() { + try { + DBCollection collection = db.getCollection(RUNDOWN_COLLECTION); + BasicDBObject query = (BasicDBObject) QueryBuilder.start().put(IOctopusAPI.ID).notIn(storyStoryFolders.keySet().toArray()).get(); + WriteResult res = collection.remove(query); + logger.trace(String.format("Deleted orphan rundowns: %d", res.getN())); + } catch (Exception e) { + logger.error(e); + } + } + + private void ensureIndexes() { + DBCollection collection = db.getCollection(FOLDER_COLLECTION); + if (collection.count() == 0) + collection.ensureIndex(IOctopusAPI.ID); + collection = db.getCollection(RUNDOWN_COLLECTION); + if (collection.count() == 0) { + collection.ensureIndex(IOctopusAPI.ID); + collection.ensureIndex(IOctopusAPI.SCHEDULED_START); + } + collection = db.getCollection(STORY_COLLECTION); + if (collection.count() == 0) + collection.ensureIndex(IOctopusAPI.ID); + } + + public void execute(boolean includeArchived) throws Exception { + this.includeArchived = includeArchived; + logger.trace(STARTING); + + // String MAIN_RUNDOWN_COLLECTION = IOctopusAPI.RUNDOWN_COLLECTION; + // String MAIN_FOLDER_COLLECTION = IOctopusAPI.FOLDER_COLLECTION; + // String MAIN_STORY_COLLECTION = IOctopusAPI.STORY_COLLECTION; + + // RUNDOWN_COLLECTION = MAIN_RUNDOWN_COLLECTION + _TMP; + // FOLDER_COLLECTION = MAIN_FOLDER_COLLECTION + _TMP; + // STORY_COLLECTION = MAIN_STORY_COLLECTION + _TMP; + // + // try { + // db.getCollection(RUNDOWN_COLLECTION).drop(); + // db.getCollection(FOLDER_COLLECTION).drop(); + // db.getCollection(STORY_COLLECTION).drop(); + // } catch (Exception e) { + // logger.catching(e); + // throw e; + // } + + RUNDOWN_COLLECTION = IOctopusAPI.RUNDOWN_COLLECTION; + FOLDER_COLLECTION = IOctopusAPI.FOLDER_COLLECTION; + STORY_COLLECTION = IOctopusAPI.STORY_COLLECTION; + + //current = korábbi szinkronizálás + currentRundowns = getCurrentIDs(RUNDOWN_COLLECTION); + currentFolder = getCurrentIDs(FOLDER_COLLECTION); + currentStories = getCurrentIDs(STORY_COLLECTION); + + BasicDBList rundowns = null; + BasicDBList storyFolders = null; + try { + rundowns = queryBuildRefRundowns(); + storyFolders = queryBuildRefFolders(); + } catch (Exception e) { + logger.catching(e); + throw e; + } + + processRundowns(rundowns); + processStoryFolders(storyFolders); + + deleteDiff(currentRundowns.keySet(), rundownIDs, RUNDOWN_COLLECTION); + deleteDiff(currentFolder.keySet(), folderIDs, FOLDER_COLLECTION); + deleteDiff(currentStories.keySet(), storyIDs, STORY_COLLECTION); + // deleteOrphanRundowns(); + // deleteOrphanStoryFolders(); + // deleteOrphanStories(); + + //a sorrend fontos ! + // updateDiff(MAIN_STORY_COLLECTION, STORY_COLLECTION, IOctopusAPI.ID); + // updateDiff(MAIN_RUNDOWN_COLLECTION, RUNDOWN_COLLECTION, IOctopusAPI.ID); + // updateDiff(MAIN_FOLDER_COLLECTION, FOLDER_COLLECTION, IOctopusAPI.ID); + // + // updateDeleteDiff(MAIN_RUNDOWN_COLLECTION, RUNDOWN_COLLECTION, IOctopusAPI.ID); + // updateDeleteDiff(MAIN_FOLDER_COLLECTION, FOLDER_COLLECTION, IOctopusAPI.ID); + // updateDeleteDiff(MAIN_STORY_COLLECTION, STORY_COLLECTION, IOctopusAPI.ID); + + // try { + // db.getCollection(RUNDOWN_COLLECTION).rename(MAIN_RUNDOWN_COLLECTION, true); + // db.getCollection(FOLDER_COLLECTION).rename(MAIN_FOLDER_COLLECTION, true); + // db.getCollection(STORY_COLLECTION).rename(MAIN_STORY_COLLECTION, true); + // } catch (Exception e) { + // logger.catching(e); + // throw e; + // } + + // logger.info("Activate"); + + // try { + // long ts = new Date().getTime(); + // db.getCollection(RUNDOWN_COLLECTION).exportFile(String.format("/opt/mediacube-test/log/%s-%s.js", RUNDOWN_COLLECTION, ts)); + // db.getCollection(FOLDER_COLLECTION).exportFile(String.format("/opt/mediacube-test/log/%s-%s.js", FOLDER_COLLECTION, ts)); + // db.getCollection(STORY_COLLECTION).exportFile(String.format("/opt/mediacube-test/log/%s-%s.js", STORY_COLLECTION, ts)); + // } catch (Exception e) { + // logger.catching(e); + // throw e; + // } + + logger.trace(FINISHED); + //throw new Exception("Mérés"); + } + + private String extractContent(BasicDBObject content) { + String scriptContent = ""; + if (!content.containsKey(IOctopusAPI.TYPE)) + return scriptContent; + String type = content.getString(IOctopusAPI.TYPE); + switch (type) { + case IOctopusAPI.TEXT: { + if (content.containsKey(IOctopusAPI.TEXT)) { + String text = content.getString(IOctopusAPI.TEXT); + if (text != null) + scriptContent += String.format("%s%s", text.replaceAll(SIMPLE_LINEFEED, LINEFEED), LINEFEED); + } + break; + } + case IOctopusAPI.MOS: { + BasicDBObject mosObject = NoSQLUtils.asDBObject(content, IOctopusAPI.OBJECT); + if (mosObject != null && !mosObject.isEmpty()) { + if (mosObject.containsKey(IOctopusAPI.OBJ_ID)) { + String objID = mosObject.getString(IOctopusAPI.OBJ_ID); + scriptContent += String.format("%s %s%s", MOSOBJECT, objID, LINEFEED); + } + } + break; + } + default: { + if (content.containsKey(IOctopusAPI.CONTENT)) { + List innerContents = NoSQLUtils.asList(content, IOctopusAPI.CONTENT); + if (innerContents != null) { + for (BasicDBObject actualInnerContent : innerContents) { + if (actualInnerContent != null && actualInnerContent.isEmpty()) + scriptContent += extractContent(actualInnerContent); + } + } + } + break; + } + } + return scriptContent; + } + + private String extractCustomColumnValue(String columnName, BasicDBObject story) { + List customColumns = NoSQLUtils.asList(story, IOctopusAPI.CUSTOM_COLUMNS); + if (customColumns == null) + return null; + String result = null; + for (BasicDBObject customColumn : customColumns) { + if (!customColumn.containsKey(IOctopusAPI.LABEL)) + continue; + String currentName = customColumn.getString(IOctopusAPI.LABEL); + if (currentName == null) + continue; + if (!currentName.toLowerCase().equals(columnName.toLowerCase())) + continue; + if (!customColumn.containsKey(IOctopusAPI.VALUE)) + continue; + result = customColumn.getString(IOctopusAPI.VALUE); + break; + } + return result; + } + + private BasicDBList extractRelevantMOSObjects(BasicDBObject story) { + List mosObjects = NoSQLUtils.asList(story, IOctopusAPI.MOS_OBJECTS); + if (mosObjects == null) + return null; + BasicDBList result = null; + for (BasicDBObject mosObject : mosObjects) { + if (!mosObject.containsKey(IOctopusAPI.MOS_ID)) + continue; + String mosId = mosObject.getString(IOctopusAPI.MOS_ID); + if (!IOctopusAPI.NEXIO_MOS.equals(mosId)) + continue; + if (result == null) + result = new BasicDBList(); + result.add(mosObject); + } + return result; + } + + private String extractScriptContent(BasicDBObject story) { + BasicDBObject script = NoSQLUtils.asDBObject(story, IOctopusAPI.SCRIPT); + if (script == null || script.isEmpty()) + return null; + + List body = NoSQLUtils.asList(script, IOctopusAPI.BODY); + if (body == null || body.size() == 0) + return null; + + StringBuilder sb = new StringBuilder(); + + for (BasicDBObject bodyItem : body) { + if (bodyItem.containsKey(IOctopusAPI.LABEL)) { + sb.append(bodyItem.getString(IOctopusAPI.LABEL)); + sb.append(LINEFEED); + } + List contents = NoSQLUtils.asList(bodyItem, IOctopusAPI.CONTENT); + if (contents == null) + continue; + for (BasicDBObject content : contents) { + sb.append(extractContent(content)); + } + } + return sb.length() == 0 ? null : sb.toString(); + } + + private void fireProgressEvent(ProgressEvent evt) { + logger.debug("Progress changed to " + evt.getProgress() + "%"); + if (progressListenerList == null) + return; + Object[] listeners = progressListenerList.getListenerList(); + for (int i = 0; i < listeners.length; i += 2) { + if (listeners[i] == IProgressEventListener.class) + ((IProgressEventListener) listeners[i + 1]).progressChanged(evt); + } + } + + public Map getCurrentIDs(String name) { + Map result = new HashMap<>(); + + DBCursor cursor = db.getCollection(name).find(new BasicDBObject(), new BasicDBObject(IOctopusAPI.ID, 1)); + if (cursor.hasNext()) { + List objects = ListUtils.cast(cursor.toArray()); + for (BasicDBObject obj : objects) { + if (obj == null) + continue; + long id = NoSQLUtils.asLong(obj, IOctopusAPI.ID); + if (id == 0) + continue; + Object _id = obj.getID(); + if (_id == null) + continue; + result.put(id, _id); + } + } + return result; + } + + private void processRundowns(BasicDBList rundowns) throws Exception { + if (rundowns == null || rundowns.size() == 0) { + progressEvent.setProgress(50); + fireProgressEvent(progressEvent); + } else { + logger.info("Process rundowns"); + try { + storeRundowns(rundowns, null); + } catch (Exception e) { + logger.catching(e); + throw e; + } + } + } + + private void processStoryFolders(BasicDBList storyFolders) throws Exception { + if (storyFolders == null || storyFolders.size() == 0) { + progressEvent.setProgress(100); + fireProgressEvent(progressEvent); + } else { + logger.info("Process story folders"); + try { + storeStoryFolders(storyFolders, null); + } catch (Exception e) { + logger.catching(e); + throw e; + } + } + } + + private Builder query(String path, String fields) { + ResteasyWebTarget target = webTarget.path(path).queryParam(FIELDS, fields); + Builder result = target.request().header(OCTOPUS_DEVICE_ID, apiUser).header(OCTOPUS_DEVICE_NAME, apiPwd); + return result; + } + + private BasicDBList queryBuildRefFolders() { + BasicDBList storyFolders; + logger.info("Fetch story folders"); + storyFolders = queryStoryFolders(); + logger.info("Fetch folder story references"); + storyStoryFolders = buildFolderReferences(storyFolders); + return storyFolders; + } + + private BasicDBList queryBuildRefRundowns() { + BasicDBList rundowns; + logger.info("Fetch rundowns"); + rundowns = queryRundowns(); + logger.info("Fetch rundown story references"); + storyRundowns = buildRundownReferences(rundowns); + return rundowns; + } + + private BasicDBObject queryRundown(BasicDBObject rundown, String fields) { + logger.trace(ENTER); + BasicDBObject result = null; + long id = NoSQLUtils.asLong(rundown, IOctopusAPI.ID); + Response response = query(String.format("%s/%d", RUNDOWN, id), fields).get(); + String json = response.readEntity(String.class); + BasicDBObject resultObject = (BasicDBObject) JSONUtil.jsonToDbObject(json); + if (resultObject == null) + logger.error("Rundown {} {} is not available", id, rundown.getString(IOctopusAPI.NAME)); + else + result = NoSQLUtils.asDBObject(resultObject, RESULT); + logger.trace(EXIT); + return result; + } + + private BasicDBList queryRundowns() { + logger.trace(ENTER); + BasicDBList result = null; + Builder query = query(RUNDOWN, FIELDS_RUNDOWN); + Response response = null; + if (includeArchived) { + response = query.post(Entity.entity(new BasicDBObject(FILTER, new BasicDBObject(ARCHIVED, true)).toString(), MediaType.APPLICATION_JSON)); + } else + response = query.get(); + + String json = response.readEntity(String.class); + BasicDBObject resultObject = (BasicDBObject) JSONUtil.jsonToDbObject(json); + if (resultObject != null) + result = NoSQLUtils.asDBList(resultObject, RESULT); + logger.trace(EXIT); + return result; + } + + private BasicDBObject queryStoryFolder(BasicDBObject storyFolder, String fields) { + logger.trace(ENTER); + BasicDBObject result = null; + long id = NoSQLUtils.asLong(storyFolder, IOctopusAPI.ID); + Response response = query(String.format("%s/%d", STORY_FOLDER, id), fields).get(); + String json = response.readEntity(String.class); + BasicDBObject resultObject = (BasicDBObject) JSONUtil.jsonToDbObject(json); + if (resultObject == null) + logger.error("StoryFolder {} {} is not available", id, storyFolder.getString(IOctopusAPI.NAME)); + else + result = NoSQLUtils.asDBObject(resultObject, RESULT); + logger.trace(EXIT); + return result; + } + + private BasicDBList queryStoryFolders() { + logger.trace(ENTER); + BasicDBList result = null; + Response response = query(STORY_FOLDER, FIELDS_STORYFOLDER).get(); + String json = response.readEntity(String.class); + BasicDBObject resultObject = (BasicDBObject) JSONUtil.jsonToDbObject(json); + if (resultObject != null) + result = NoSQLUtils.asDBList(resultObject, RESULT); + + // /* teszt */ + // List list = NoSQLUtils.asList(result); + // for (BasicDBObject actual : list) { + // String fullName = concatParentsToStoryFolder(actual, actual.getString(IOctopusAPI.NAME)); + // //logger.info("Checking StoryFolder {}", fullName); + // actual.remove(IOctopusAPI.NAME); + // actual.append(IOctopusAPI.NAME, fullName); + // } + // + logger.trace(EXIT); + return result; + } + + public void removeProgressListener(IProgressEventListener listener) { + progressListenerList.remove(IProgressEventListener.class, listener); + } + + void setObjectID(Map current, long id, BasicDBObject objectToSave) { + Object _id = current.get(id); + if (_id == null) + return; + objectToSave.put(IOctopusAPI._ID, _id); + } + + private void storeRundown(BasicDBObject rundown) { + logger.trace(ENTER); + String name = rundown.containsKey(IOctopusAPI.NAME) ? rundown.getString(IOctopusAPI.NAME) : null; + logger.debug("Storing rundown {} {}", name, rundown.get(IOctopusAPI.SCHEDULED_START)); + BasicDBObject rundownWithStories = queryRundown(rundown, FIELDS_RUNDOWN_STORIES); + if (rundownWithStories != null) { + Date scheduledStart = toDate(rundown, IOctopusAPI.SCHEDULED_START); + if (scheduledStart != null && scheduledStart.after(zeroDate.getTime())) { + BasicDBList stories = NoSQLUtils.asDBList(rundownWithStories, IOctopusAPI.SLUGS); + if (stories != null) + storeRundownStories(stories); + rundown.put(IOctopusAPI.SCHEDULED_START, toDate(rundown, IOctopusAPI.SCHEDULED_START)); + rundown.put(IOctopusAPI.MODIFIED, toDate(rundown, IOctopusAPI.MODIFIED)); + DBCollection collection = db.getCollection(RUNDOWN_COLLECTION); + long rundownID = NoSQLUtils.asLong(rundown, IOctopusAPI.ID); + setObjectID(currentRundowns, rundownID, rundown); + collection.save(rundown); + } + } + logger.trace(EXIT); + } + + private void storeRundowns(BasicDBList rundowns, Date lastUpdateTime) { + logger.trace(ENTER); + List rundownsList = NoSQLUtils.asList(rundowns); + int idx = 1; + for (BasicDBObject rundown : rundownsList) { + //logger.info(CHECKING_RUNDOWN, rundown.getLong(IOctopusAPI.ID), rundownsList.size(), idx); + storeRundown(rundown); + int progress = idx * 50 / rundownsList.size(); + if (progress - progressEvent.getProgress() > 0) { + progressEvent.setProgress(progress); + fireProgressEvent(progressEvent); + } + + idx++; + } + logger.trace(EXIT); + } + + private void storeRundownStories(BasicDBList slugs) { + logger.trace(ENTER); + List slugsList = NoSQLUtils.asList(slugs); + for (BasicDBObject slug : slugsList) { + if (slug.containsKey(IOctopusAPI.STORY)) + storeStory((BasicDBObject) slug.get(IOctopusAPI.STORY)); + } + logger.trace(EXIT); + } + + private void storeStory(BasicDBObject story) { + logger.trace(ENTER); + if (!story.containsKey(IOctopusAPI.ID)) { + logger.error("Missing id in story {}", story.toPrettyString(null)); + return; + } + long storyID = story.getLong(IOctopusAPI.ID); + boolean isContains = storyIDs.contains(storyID); + if (isContains) + return; + storyIDs.add(storyID); + story.put(IOctopusAPI.MODIFIED, toDate(story, IOctopusAPI.MODIFIED)); + BasicDBList rundownRef = storyRundowns.get(storyID); + BasicDBList storyFolderRef = storyStoryFolders.get(storyID); + BasicDBList modifiedMOS = extractRelevantMOSObjects(story); + + DBCollection collection = db.getCollection(STORY_COLLECTION); + + if (rundownRef != null) + story.put(IOctopusAPI.REF_RUNDOWN, rundownRef); + if (storyFolderRef != null) + story.put(IOctopusAPI.REF_STORYFOLDER, storyFolderRef); + String scriptContent = extractScriptContent(story); + story.put(IOctopusAPI.SCRIPT_CONTENT, scriptContent); + if (modifiedMOS == null || modifiedMOS.isEmpty()) { + if (story.containsKey(IOctopusAPI.MOS_OBJECTS)) + story.remove(IOctopusAPI.MOS_OBJECTS); + } else + story.put(IOctopusAPI.MOS_OBJECTS, modifiedMOS); + + String parentStoryId = extractCustomColumnValue(IOctopusAPI.PARENT_STORY_ID, story); + if (parentStoryId == null) { + logger.warn("Missing {} in story", IOctopusAPI.PARENT_STORY_ID, story.toPrettyString(null)); + story.append(IOctopusAPI.PARENT_STORY_ID, storyID); + } else + story.append(IOctopusAPI.PARENT_STORY_ID, parentStoryId); + logger.debug(SAVING_STORY_ID, storyID); + setObjectID(currentStories, storyID, story); + collection.save(story); + logger.trace(EXIT); + } + + private void storeStoryFolder(BasicDBObject storyFolder, Date lastUpdateTime) { + logger.trace(ENTER); + BasicDBObject storyFoldersWithStories = queryStoryFolder(storyFolder, FIELDS_STORYFOLDER_STORIES); + if (storyFoldersWithStories != null) { + BasicDBList stories = NoSQLUtils.asDBList(storyFoldersWithStories, IOctopusAPI.STORIES); + if (stories != null) + storeStoryFolderStories(stories); + storyFolder.put(IOctopusAPI.MODIFIED, toDate(storyFolder, IOctopusAPI.MODIFIED)); + DBCollection collection = db.getCollection(FOLDER_COLLECTION); + String name = storyFolder.getString(IOctopusAPI.NAME); + logger.debug("Storing story folder {}", name); + long folderID = NoSQLUtils.asLong(storyFolder, IOctopusAPI.ID); + setObjectID(currentFolder, folderID, storyFolder); + collection.save(storyFolder); + } + logger.trace(EXIT); + } + + private void storeStoryFolders(BasicDBList storyFolders, Date lastUpdateTime) { + logger.trace(ENTER); + List storyFolderList = NoSQLUtils.asList(storyFolders); + int idx = 1; + for (BasicDBObject storyFolder : storyFolderList) { + logger.debug(String.format(CHECKING_STORY_FOLDER, storyFolder.getLong(IOctopusAPI.ID), storyFolderList.size(), idx)); + storeStoryFolder(storyFolder, lastUpdateTime); + int progress = 50 + (idx * 50 / storyFolderList.size()); + if (progress - progressEvent.getProgress() > 0) { + progressEvent.setProgress(progress); + fireProgressEvent(progressEvent); + } + idx++; + } + logger.trace(EXIT); + } + + private void storeStoryFolderStories(BasicDBList stories) { + logger.trace(ENTER); + List list = NoSQLUtils.asList(stories); + for (BasicDBObject story : list) + storeStory(story); + logger.trace(EXIT); + } + + private Date toDate(BasicDBObject obj, String name) { + Date result = null; + if (obj.containsKey(name)) { + String dt = obj.getString(name); + if (dt != null) { + // create jodatime from date + DateTime jdt = new DateTime(dt); + result = jdt.toDate(); + } + } + return result; + } + + private void updateDeleteDiff(String oldCollectionName, String newCollectionName, String idFieldName) { + DBCollection oldCollection = db.getCollection(oldCollectionName); + DBCollection newCollection = db.getCollection(newCollectionName); + DBCursor oldCollectionCursor = oldCollection.find(new BasicDBObject(), new BasicDBObject(idFieldName, 1)); + if (!oldCollectionCursor.hasNext()) { + logger.error("{} collection is empty", newCollectionName); + return; + } + List oldItems = ListUtils.cast(oldCollectionCursor.toArray()); + + DBCursor newCollectionCursor = newCollection.find(); + ConcurrentHashMap newItems = null; + if (newCollectionCursor.hasNext()) { + List newList = ListUtils.cast(newCollectionCursor.toArray()); + newItems = ListUtils.map(newList, item -> item.getLong(IOctopusAPI.ID)); + } + if (newItems == null) + newItems = new ConcurrentHashMap<>(); + + for (BasicDBObject oldItem : oldItems) { + if (oldItem == null) { + logger.error("Item is null"); + continue; + } + if (!oldItem.containsKey(idFieldName)) { + logger.error("{} is null", idFieldName); + continue; + } + long id = oldItem.getLong(idFieldName); + BasicDBObject newItem = newItems.get(id); + if (newItem == null) { + //remove + logger.info("Deleting {}", oldItem.toPrettyString(null)); + oldCollection.remove(new BasicDBObject(idFieldName, id)); + } + } + } + + private void updateDiff(String oldCollectionName, String newCollectionName, String idFieldName) { + DBCollection oldCollection = db.getCollection(oldCollectionName); + DBCollection newCollection = db.getCollection(newCollectionName); + DBCursor newCollectionCursor = newCollection.find(); + if (!newCollectionCursor.hasNext()) { + logger.error("{} collection is empty", newCollectionName); + return; + } + + List newItems = ListUtils.cast(newCollectionCursor.toArray()); + + DBCursor oldCollectionCursor = oldCollection.find(); + ConcurrentHashMap oldItems = null; + if (oldCollectionCursor.hasNext()) { + List oldList = ListUtils.cast(oldCollectionCursor.toArray()); + oldItems = ListUtils.map(oldList, item -> item.getLong(IOctopusAPI.ID)); + } + if (oldItems == null) + oldItems = new ConcurrentHashMap<>(); + for (BasicDBObject newItem : newItems) { + if (newItem == null) { + logger.error("Item is null"); + continue; + } + if (!newItem.containsKey(idFieldName)) { + logger.error("{} is null", idFieldName); + continue; + } + long id = newItem.getLong(idFieldName); + BasicDBObject oldItem = oldItems.get(id); + boolean save = false; + if (oldItem == null) { + //save + save = true; + } else { + //compare + Object oldDbId = oldItem.getID(); + newItem.remove(IOctopusAPI._ID); + oldItem.remove(IOctopusAPI._ID); + + if (!newItem.equals(oldItem)) { + //save + newItem.put(IOctopusAPI._ID, oldDbId); + save = true; + } + } + + if (save) { + logger.info("Saving to {} item {}", oldCollectionName, id); + oldCollection.save(newItem); + } + } + } + +} diff --git a/server/user.jobengine.osgi.commons/src/user/commons/MediaCubeFinishMarker.java b/server/user.jobengine.osgi.commons/src/user/commons/MediaCubeFinishMarker.java new file mode 100644 index 00000000..e2b252ac --- /dev/null +++ b/server/user.jobengine.osgi.commons/src/user/commons/MediaCubeFinishMarker.java @@ -0,0 +1,14 @@ +package user.commons; + +@SuppressWarnings("serial") +public class MediaCubeFinishMarker extends MediaCubeMarker { + + public MediaCubeFinishMarker(MediaCubeMarker parent) { + setSessionID(parent.getSessionID()); + } + + public MediaCubeFinishMarker(MediaCubeMarker parent, String to) { + super(to); + setSessionID(parent.getSessionID()); + } +} diff --git a/server/user.jobengine.osgi.commons/src/user/commons/MediaCubeMarker.java b/server/user.jobengine.osgi.commons/src/user/commons/MediaCubeMarker.java index 52fe5ed9..f1dbf54e 100644 --- a/server/user.jobengine.osgi.commons/src/user/commons/MediaCubeMarker.java +++ b/server/user.jobengine.osgi.commons/src/user/commons/MediaCubeMarker.java @@ -6,18 +6,40 @@ import org.apache.logging.log4j.MarkerManager.Log4jMarker; public class MediaCubeMarker extends Log4jMarker { private static final String MEDIACUBE = "MEDIACUBE"; private String to; + private String sessionID; + + public MediaCubeMarker() { + super(MEDIACUBE); + } public MediaCubeMarker(String to) { super(MEDIACUBE); this.to = to; } + public String getSessionID() { + return sessionID; + } + public String getTo() { return to; } + public void setSessionID(String sessionID) { + this.sessionID = sessionID; + } + public void setTo(String to) { this.to = to; } + // @Override + // public boolean equals(Object o) { + // if (o == null || !o.getClass().equals(MediaCubeMarker.class)) + // return false; + // + // + // return true; + // } + } diff --git a/server/user.jobengine.osgi.commons/src/user/commons/octopus/IOctopusAPI.java b/server/user.jobengine.osgi.commons/src/user/commons/octopus/IOctopusAPI.java index c4505874..9cdbdd58 100644 --- a/server/user.jobengine.osgi.commons/src/user/commons/octopus/IOctopusAPI.java +++ b/server/user.jobengine.osgi.commons/src/user/commons/octopus/IOctopusAPI.java @@ -22,6 +22,7 @@ public interface IOctopusAPI { static final String MOS_OBJECTS = "mosObjects"; static final String NAME = "name"; static final String LABEL = "label"; + static final String CLIP = "clip"; static final String OBJ_ID = "objId"; static final String OBJECT = "object"; static final String MOS = "mos"; diff --git a/server/user.jobengine.osgi.commons/src/user/commons/octopus/OctopusAPI.java b/server/user.jobengine.osgi.commons/src/user/commons/octopus/OctopusAPI.java index b9679a43..c2eca5bf 100644 --- a/server/user.jobengine.osgi.commons/src/user/commons/octopus/OctopusAPI.java +++ b/server/user.jobengine.osgi.commons/src/user/commons/octopus/OctopusAPI.java @@ -155,7 +155,7 @@ public class OctopusAPI implements IOctopusAPI { public List getStories() { List result = null; DBCollection collection = db.getCollection(STORY_COLLECTION); - DBCursor find = collection.find(); + DBCursor find = collection.find().limit(1000); if (find.hasNext()) result = find.toArray(); return result; diff --git a/server/user.jobengine.osgi.server/.settings/org.eclipse.core.resources.prefs b/server/user.jobengine.osgi.server/.settings/org.eclipse.core.resources.prefs index 0813a5b5..54d10621 100644 --- a/server/user.jobengine.osgi.server/.settings/org.eclipse.core.resources.prefs +++ b/server/user.jobengine.osgi.server/.settings/org.eclipse.core.resources.prefs @@ -1,3 +1,4 @@ eclipse.preferences.version=1 encoding//pages/login.zul=UTF-8 +encoding//resources/i3-label_hu.properties=UTF-8 encoding//src/user/jobengine/server/JobRuntime.java=UTF-8 diff --git a/server/user.jobengine.osgi.server/pages/index.zul b/server/user.jobengine.osgi.server/pages/index.zul index 8e44f53b..7f68e668 100644 --- a/server/user.jobengine.osgi.server/pages/index.zul +++ b/server/user.jobengine.osgi.server/pages/index.zul @@ -1,7 +1,7 @@ - -
- - - - - @@ -72,6 +67,7 @@ +
diff --git a/server/user.jobengine.osgi.server/resources/i3-label_hu.properties b/server/user.jobengine.osgi.server/resources/i3-label_hu.properties index 50d11374..1d9c70ba 100644 --- a/server/user.jobengine.osgi.server/resources/i3-label_hu.properties +++ b/server/user.jobengine.osgi.server/resources/i3-label_hu.properties @@ -1,4 +1,5 @@ -footer=2016 © Copyright User Rendszerház Kft. +version=2.2.2 +footer=2016 © Copyright User Rendszerház Kft. login_info=Információ login_dialogtitle=Bejelentkezés diff --git a/server/user.jobengine.osgi.server/src/user/jobengine/server/IJobRuntime.java b/server/user.jobengine.osgi.server/src/user/jobengine/server/IJobRuntime.java index 3d6cfe51..12276fc6 100644 --- a/server/user.jobengine.osgi.server/src/user/jobengine/server/IJobRuntime.java +++ b/server/user.jobengine.osgi.server/src/user/jobengine/server/IJobRuntime.java @@ -4,6 +4,8 @@ import java.util.Map; import java.util.NoSuchElementException; import java.util.Stack; +import org.apache.logging.log4j.Marker; + import user.commons.IEntityPersister; import user.commons.IJob; import user.commons.JobStatus; @@ -26,8 +28,12 @@ public interface IJobRuntime extends IJob { IInstruction getCurrentInstruction(); + Marker getFinishMarker(); + int getIp(); + Marker getMarker(); + IInstruction getNextInstruction() throws NoSuchElementException; Object getParameter(String name); diff --git a/server/user.jobengine.osgi.server/src/user/jobengine/server/JobRuntime.java b/server/user.jobengine.osgi.server/src/user/jobengine/server/JobRuntime.java index 12b312c1..c4357fc4 100644 --- a/server/user.jobengine.osgi.server/src/user/jobengine/server/JobRuntime.java +++ b/server/user.jobengine.osgi.server/src/user/jobengine/server/JobRuntime.java @@ -17,12 +17,15 @@ import javax.swing.event.EventListenerList; import org.apache.commons.lang.ClassUtils; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; +import org.apache.logging.log4j.Marker; import org.apache.logging.log4j.message.Message; import user.commons.IEntityPersister; import user.commons.IJob; import user.commons.Job; import user.commons.JobStatus; +import user.commons.MediaCubeFinishMarker; +import user.commons.MediaCubeMarker; import user.jobengine.db.ItemManagerData.SignalType; import user.jobengine.db.JobParameters; import user.jobengine.server.instructions.CallJobStepInstruction; @@ -45,6 +48,7 @@ public class JobRuntime extends Job implements IJobRuntime { private double currentProgress; private int runtimeWeight; private List children; + private MediaCubeMarker sessionMarker; public JobRuntime() { this.listeners = new EventListenerList(); @@ -206,11 +210,25 @@ public class JobRuntime extends Job implements IJobRuntime { return program.get(getIp()); } + @Override + public Marker getFinishMarker() { + return new MediaCubeFinishMarker((MediaCubeMarker) getMarker()); + } + @Override public int getIp() { return ip; } + @Override + public Marker getMarker() { + if (sessionMarker == null) { + sessionMarker = new MediaCubeMarker(); + sessionMarker.setSessionID(String.valueOf(getId())); + } + return sessionMarker; + } + @Override public IInstruction getNextInstruction() throws NoSuchElementException { if (!hasNextInstruction()) diff --git a/server/user.jobengine.osgi.server/src/user/jobengine/server/JobStepExecutor.java b/server/user.jobengine.osgi.server/src/user/jobengine/server/JobStepExecutor.java index 930f5def..b7741fa4 100644 --- a/server/user.jobengine.osgi.server/src/user/jobengine/server/JobStepExecutor.java +++ b/server/user.jobengine.osgi.server/src/user/jobengine/server/JobStepExecutor.java @@ -64,6 +64,8 @@ public class JobStepExecutor implements IJobStepExecutor { logger.catching(t); jobEngine.suspendJob(t, jobRuntime); } finally { + logger.info(jobRuntime.getFinishMarker(), "A folyamat futása véget ért"); + if (step != null) { step.cleanup(); jobRuntime = null; diff --git a/server/user.jobengine.osgi.server/src/user/jobengine/server/steps/LogCollector.java b/server/user.jobengine.osgi.server/src/user/jobengine/server/steps/LogCollector.java deleted file mode 100644 index 09afb95c..00000000 --- a/server/user.jobengine.osgi.server/src/user/jobengine/server/steps/LogCollector.java +++ /dev/null @@ -1,106 +0,0 @@ -package user.jobengine.server.steps; - -public class LogCollector { -} -//public class LogCollector extends Logger { -// -// class CollectedMessage { -// private String message; -// private Level level; -// -// CollectedMessage(Level level, Message message) { -// this.level = level; -// this.message = message.getFormattedMessage(); -// } -// -// @Override -// public String toString() { -// return String.format("%s %s\r\n", level, message); -// } -// } -// -// private static final String LOGCOLLECTOR_FINISH = "LOGCOLLECTOR_FINISH"; -// public static Marker FINISH = MarkerManager.getMarker(LOGCOLLECTOR_FINISH); -// -// public static org.apache.logging.log4j.Logger getLogger() { -// Logger logger = (Logger) LogManager.getLogger(ReflectionUtil.getCallerClass(2)); -// LogCollector result = new LogCollector(logger.getContext(), logger.getName(), logger.getMessageFactory()); -// return result; -// } -// -// private Map> messages = new HashMap<>(); -// -// protected LogCollector(LoggerContext context, String name, MessageFactory messageFactory) { -// super(context, name, messageFactory); -// //Map appenders = super.getAppenders(); -// } -// -// private void collect(Marker marker, Level level, Message message) { -// //LogEvent event = Log4jLogEvent.newBuilder().setMessage(message).setMarker(marker).setLevel(level).setLoggerName(super.getName()).build(); -// List msgList = messages.get(marker); -// if (msgList == null) { -// msgList = new ArrayList<>(); -// messages.put(marker, msgList); -// } -// msgList.add(new CollectedMessage(level, message)); -// } -// -// @Override -// public void error(Marker marker, String message) { -// Message msg = getMessageFactory().newMessage(message); -// collect(marker, Level.ERROR, msg); -// super.error(marker, message); -// } -// -// @Override -// public void error(Marker marker, String message, Object... params) { -// Message msg = getMessageFactory().newMessage(message, params); -// collect(marker, Level.ERROR, msg); -// super.error(marker, message, params); -// } -// -// private void finish() { -// Set markers = messages.keySet(); -// for (Marker marker : markers) { -// List msgList = messages.get(marker); -// StringBuilder sb = new StringBuilder(); -// for (CollectedMessage msg : msgList) { -// sb.append(msg.toString()); -// } -// super.info(marker, sb.toString()); -// } -// messages.clear(); -// } -// -// @Override -// public void info(Marker marker, String message) { -// Message msg = getMessageFactory().newMessage(message); -// collect(marker, Level.INFO, msg); -// } -// -// @Override -// public void info(Marker marker, String message, Object... params) { -// Message msg = getMessageFactory().newMessage(message, params); -// collect(marker, Level.INFO, msg); -// } -// -// @Override -// public void info(Object message) { -// if (message instanceof Marker && LOGCOLLECTOR_FINISH.equals(((Marker) message).getName())) { -// finish(); -// return; -// } -// } -// -// @Override -// public void warn(Marker marker, String message) { -// Message msg = getMessageFactory().newMessage(message); -// collect(marker, Level.WARN, msg); -// } -// -// @Override -// public void warn(Marker marker, String message, Object... params) { -// Message msg = getMessageFactory().newMessage(message, params); -// collect(marker, Level.WARN, msg); -// } -//} diff --git a/server/user.jobengine.osgi.server/src/user/jobengine/zk/util/LDAPHandler.java b/server/user.jobengine.osgi.server/src/user/jobengine/zk/util/LDAPHandler.java index 36ad6269..7f2a40b1 100644 --- a/server/user.jobengine.osgi.server/src/user/jobengine/zk/util/LDAPHandler.java +++ b/server/user.jobengine.osgi.server/src/user/jobengine/zk/util/LDAPHandler.java @@ -26,10 +26,13 @@ import org.apache.logging.log4j.Logger; import user.commons.ListUtils; public class LDAPHandler { + private static final String OTHER_MAILBOX = "otherMailbox"; private static final Logger logger = LogManager.getLogger(); private static final String LDAP_ADDRESS = "ldap://%s:%s"; private static final String AD_NONSECUREPORT = "ad_nonsecureport"; private static final String AD_HOST = "ad_host"; + // private static final String TECH_ACCOUNT = "tech_account"; + // private static final String TECH_PASSWORD = "tech_password"; private static final String SIMPLE_AUTH = "simple"; private static final String CTX_FACTORY = "com.sun.jndi.ldap.LdapCtxFactory"; private static final String DOMAIN_ACCOUNT = "%s@%s"; @@ -77,7 +80,7 @@ public class LDAPHandler { LdapContext ctx = null; NamingEnumeration answer = null; try { - ctx = getContext(); + ctx = getContext(account, password); SearchControls searchCtls = new SearchControls(); searchCtls.setSearchScope(SearchControls.SUBTREE_SCOPE); String searchFilter = "(&(objectClass=group))"; @@ -113,7 +116,7 @@ public class LDAPHandler { return result; } - private LdapContext getContext() throws Exception { + private LdapContext getContext(String account, String password) throws Exception { Hashtable env = new Hashtable<>(); String ldapURL = String.format(LDAP_ADDRESS, cfg.getProperty(AD_HOST), cfg.getProperty(AD_NONSECUREPORT)); env.put(Context.PROVIDER_URL, ldapURL); @@ -131,7 +134,7 @@ public class LDAPHandler { try { SearchControls searchCtls = new SearchControls(); searchCtls.setSearchScope(SearchControls.SUBTREE_SCOPE); - String searchFilter = "(&(CN=" + cn + "))"; + String searchFilter = String.format("(&(CN=%s))", cn); String searchBase = cfg.getProperty("ad_base_dn"); answer = ctx.search(searchBase, searchFilter, searchCtls); if (answer.hasMoreElements()) { @@ -172,10 +175,10 @@ public class LDAPHandler { LdapContext ctx = null; NamingEnumeration answer = null; try { - ctx = getContext(); + ctx = getContext(account, password); SearchControls searchCtls = new SearchControls(); searchCtls.setSearchScope(SearchControls.SUBTREE_SCOPE); - String searchFilter = "(&(CN=" + account + "))"; + String searchFilter = String.format("(&(sAMAccountName=%s))", account); String searchBase = cfg.getProperty(AD_BASE_DN); answer = ctx.search(searchBase, searchFilter, searchCtls); if (answer.hasMoreElements()) { @@ -193,7 +196,11 @@ public class LDAPHandler { Attribute email = srAttrs.get(MAIL); if (email != null) result.setEmail(String.valueOf(email.get(0))); - + else { + email = srAttrs.get(OTHER_MAILBOX); + if (email != null) + result.setEmail(String.valueOf(email.get(0))); + } } } catch (Exception e) { logger.catching(e); diff --git a/server/user.jobengine.osgi.server/src/user/jobengine/zk/util/UserPrincipal.java b/server/user.jobengine.osgi.server/src/user/jobengine/zk/util/UserPrincipal.java index 9fc4befd..d5b69338 100644 --- a/server/user.jobengine.osgi.server/src/user/jobengine/zk/util/UserPrincipal.java +++ b/server/user.jobengine.osgi.server/src/user/jobengine/zk/util/UserPrincipal.java @@ -23,13 +23,14 @@ public class UserPrincipal implements java.io.Serializable { private String email; public UserPrincipal(String account) { - this.account = account; + this(account, false, false); } public UserPrincipal(String account, boolean isAdmin, boolean isJobSubmitter) { this.account = account; - setAdmin(isAdmin); - setJobSubmitter(isJobSubmitter); + this.displayName = account; + this.admin = isAdmin; + this.jobSubmitter = isJobSubmitter; } public String getAccount() { diff --git a/server/user.jobengine.osgi.services/src/user/jobengine/osgi/rest/ServiceObjectMapper.java b/server/user.jobengine.osgi.services/src/user/jobengine/osgi/rest/ServiceObjectMapper.java index ce019833..e08be765 100644 --- a/server/user.jobengine.osgi.services/src/user/jobengine/osgi/rest/ServiceObjectMapper.java +++ b/server/user.jobengine.osgi.services/src/user/jobengine/osgi/rest/ServiceObjectMapper.java @@ -9,9 +9,10 @@ import com.fasterxml.jackson.datatype.joda.JodaModule; public class ServiceObjectMapper { public static ObjectMapper createMapper() { - return new ObjectMapper().disable(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES) - .disable(SerializationFeature.WRITE_DATES_AS_TIMESTAMPS).registerModule(new JodaModule()) - .setDateFormat(new RFC3339DateFormat()).setSerializationInclusion(Include.NON_NULL); + //újraindítás kell a módoosításokhoz + return new ObjectMapper().disable(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES).disable(SerializationFeature.WRITE_DATES_AS_TIMESTAMPS) + .registerModule(new JodaModule()).setDateFormat(new RFC3339DateFormat()).setSerializationInclusion(Include.NON_NULL); + // .enable(SerializationFeature.INDENT_OUTPUT); } } diff --git a/server/user.jobengine.osgi.services/src/user/jobengine/osgi/rest/octopus/OctopusRESTService.java b/server/user.jobengine.osgi.services/src/user/jobengine/osgi/rest/octopus/OctopusRESTService.java index b92266cc..67566e6c 100644 --- a/server/user.jobengine.osgi.services/src/user/jobengine/osgi/rest/octopus/OctopusRESTService.java +++ b/server/user.jobengine.osgi.services/src/user/jobengine/osgi/rest/octopus/OctopusRESTService.java @@ -117,6 +117,23 @@ public class OctopusRESTService { return result; } + @GET + @Path("/stories") + @Consumes({ MediaType.APPLICATION_JSON }) + @Produces({ MediaType.APPLICATION_JSON }) + public Response getStories() { + logger.trace("Entry"); + Response result = null; + try { + List stories = octopusService.getStories(); + result = Response.ok(stories).build(); + } catch (Exception e) { + result = createErrorResponse(e); + } + logger.trace("Exit"); + return result; + } + @GET @Path("/storiesByID/{id}") @Consumes({ MediaType.APPLICATION_JSON })