From 81aab7acae01c81c9de6f7ace30cb3ddd1a3c5e0 Mon Sep 17 00:00:00 2001 From: =?utf8?q?V=C3=A1s=C3=A1ry=20D=C3=A1niel?= Date: Mon, 18 Dec 2017 14:27:23 +0000 Subject: [PATCH] git-tfs-id: [http://tfs.userrendszerhaz.hu:8080/tfs/DefaultCollection]$/MediaCube;C30812 --- .../META-INF/MANIFEST.MF | 17 +- .../CopyForArchiveNEXIOMaterialsStep.java | 93 +- .../server/steps/OctopusDataMiner.java | 907 ++++++++++++++++++ .../commons/octopus/OctopusDataMiner.java | 30 +- .../pages/index.zul | 2 +- .../jobengine/server/steps/EscortFiles.java | 7 +- 6 files changed, 1016 insertions(+), 40 deletions(-) create mode 100644 server/user.jobengine.executors/src/user/jobengine/server/steps/OctopusDataMiner.java diff --git a/server/user.jobengine.executors/META-INF/MANIFEST.MF b/server/user.jobengine.executors/META-INF/MANIFEST.MF index 27286a2e..2fc6fbcc 100644 --- a/server/user.jobengine.executors/META-INF/MANIFEST.MF +++ b/server/user.jobengine.executors/META-INF/MANIFEST.MF @@ -13,5 +13,18 @@ Require-Bundle: user.jobengine.osgi.commons;bundle-version="1.0.0", org.apache.logging.log4j.api;bundle-version="2.8.2" Bundle-ClassPath: . Bundle-RequiredExecutionEnvironment: JavaSE-1.8 -Import-Package: org.apache.commons.io;version="2.2.0", - org.apache.commons.net.ftp;version="3.6.0" +Import-Package: com.fasterxml.jackson.annotation;version="2.4.5", + com.fasterxml.jackson.core;version="2.4.5", + com.fasterxml.jackson.databind;version="2.4.5", + com.fasterxml.jackson.databind.util;version="2.4.5", + com.fasterxml.jackson.datatype.joda;version="2.4.5", + com.fasterxml.jackson.jaxrs.json;version="2.4.5", + javax.ws.rs.client, + javax.ws.rs.core;version="2.0.1", + org.apache.commons.io;version="2.2.0", + org.apache.commons.net.ftp;version="3.6.0", + org.jboss.resteasy.client.jaxrs, + org.jboss.resteasy.plugins.providers, + org.jboss.resteasy.spi, + org.joda.time;version="2.2.0", + org.joda.time.base;version="2.2.0" diff --git a/server/user.jobengine.executors/src/user/jobengine/server/steps/CopyForArchiveNEXIOMaterialsStep.java b/server/user.jobengine.executors/src/user/jobengine/server/steps/CopyForArchiveNEXIOMaterialsStep.java index b2a474c8..efcbb66a 100644 --- a/server/user.jobengine.executors/src/user/jobengine/server/steps/CopyForArchiveNEXIOMaterialsStep.java +++ b/server/user.jobengine.executors/src/user/jobengine/server/steps/CopyForArchiveNEXIOMaterialsStep.java @@ -37,7 +37,6 @@ import user.jobengine.server.steps.MetadataTypeDetector.MetadataType; public class CopyForArchiveNEXIOMaterialsStep extends JobStep { private static final String SCHEDULED_FORMAT = "yyyy.MM.dd HH:mm"; private static final Logger logger = LogManager.getLogger(); - private static final String ARCHIVED = "ARCHIVED"; private static final String UTF_8 = "utf-8"; private static final String JSON_EXT = ".json"; private static final String XML_EXT = ".xml"; @@ -84,21 +83,23 @@ public class CopyForArchiveNEXIOMaterialsStep extends JobStep { private void copyFile(FileArchive fileArchive, RundownArchive rundownArchive, StoryArchive storyArchive) throws Exception { String fileName = fileArchive.getFileName(); + //TODO file existance + // targetFtp = ((FtpDirectoryLister) RemoteFileHandler.createLister(targetUri)).connect(); + // int version = 1; + // while (true) { + // targetFtp.listNames(arg0) + // } + // + // if (targetUri != null) + // targetUri.cleanUp(); String videoFileName = fileName + MXFEXT; + transferFile(videoFileName); BasicDBObject metadata = createMetadata(rundownArchive, storyArchive, fileArchive); - try { - if (!targetFtp.changeWorkingDirectory(EscortFiles.STATUSFOLDER)) - targetFtp.makeDirectory(EscortFiles.STATUSFOLDER); - if (!targetFtp.changeWorkingDirectory(EscortFiles.STATUSFOLDER)) - throw new Exception("!STATUSFOLDER"); - } catch (Exception e) { - logger.catching(e); - throw e; - } transferMetadata(videoFileName, metadata); - targetFtp.changeToParentDirectory(); + //targetFtp.changeToParentDirectory(); createSourceKillDateFile(rundownArchive, fileName); + } private BasicDBObject createMetadata(RundownArchive rundownArchive, StoryArchive storyArchive, FileArchive fileArchive) { @@ -127,15 +128,27 @@ public class CopyForArchiveNEXIOMaterialsStep extends JobStep { } private void createSourceKillDateFile(RundownArchive rundownArchive, String fileName) throws Exception { - Calendar killDate = CalendarUtils.createCalendar(rundownArchive.getScheduleDate()); - killDate.add(Calendar.DAY_OF_YEAR, nexioKillDateDays); - byte[] killDateFile = EscortFiles.createNEXIOKillDateFile(fileName, killDate.getTime(), null, nexioAgency); - try (OutputStream outStream = sourceFtp.storeFileStream(fileName + XML_EXT)) { + //sourceFtp = ((FtpDirectoryLister) RemoteFileHandler.createLister(sourceUri)).connect(); + OutputStream outStream = null; + try { + sourceFtp.enterLocalActiveMode(); + Calendar killDate = CalendarUtils.createCalendar(rundownArchive.getScheduleDate()); + killDate.add(Calendar.DAY_OF_YEAR, nexioKillDateDays); + byte[] killDateFile = EscortFiles.createNEXIOKillDateFile(fileName, killDate.getTime(), null, nexioAgency); + outStream = sourceFtp.storeFileStream(fileName + XML_EXT); + if (outStream == null) { + throw new NullPointerException("Can not open: " + fileName + XML_EXT + " Reply:" + sourceFtp.getReplyString()); + } outStream.write(killDateFile); outStream.flush(); } catch (Exception e) { logger.catching(e); throw e; + } finally { + if (outStream != null) + outStream.close(); + // if (sourceUri != null) + // sourceUri.cleanUp(); } } @@ -152,6 +165,8 @@ public class CopyForArchiveNEXIOMaterialsStep extends JobStep { return null; } + sourceFtp = ((FtpDirectoryLister) RemoteFileHandler.createLister(sourceUri)).connect(); + targetFtp = ((FtpDirectoryLister) RemoteFileHandler.createLister(targetUri)).connect(); processRundowns(rundowns); if (sourceUri != null) sourceUri.cleanUp(); @@ -212,10 +227,11 @@ public class CopyForArchiveNEXIOMaterialsStep extends JobStep { } private void processRundowns(List rundowns) { - //db.getCollection(ARCHIVEDRUNDOWNS).drop(); + db.getCollection(ARCHIVEDRUNDOWNS).drop(); List archivedRundowns = queryArchivedRundowns(); int index = 1; + for (DBObject r : rundowns) { BasicDBObject rundown = (BasicDBObject) r; setProgress(index * 100 / rundowns.size()); @@ -245,6 +261,7 @@ public class CopyForArchiveNEXIOMaterialsStep extends JobStep { } index++; } + } private StoryArchive processStory(BasicDBObject rundown, DBObject s) throws Exception { @@ -329,13 +346,16 @@ public class CopyForArchiveNEXIOMaterialsStep extends JobStep { targetUri.setUserName(archiveUserName); targetUri.setPassword(archivePassword); - sourceFtp = ((FtpDirectoryLister) RemoteFileHandler.createLister(sourceUri)).connect(); - targetFtp = ((FtpDirectoryLister) RemoteFileHandler.createLister(targetUri)).connect(); - } private void transferFile(String fileName) throws Exception { int reply = 0; + // sourceFtp = ((FtpDirectoryLister) RemoteFileHandler.createLister(sourceUri)).connect(); + // targetFtp = ((FtpDirectoryLister) RemoteFileHandler.createLister(targetUri)).connect(); + // if (sourceUri != null) + // sourceUri.cleanUp(); + // if (targetUri != null) + // targetUri.cleanUp(); if (!targetFtp.enterRemotePassiveMode()) throw new Exception("!PASV"); @@ -358,20 +378,49 @@ public class CopyForArchiveNEXIOMaterialsStep extends JobStep { reply = sourceFtp.stat(); if (!FTPReply.isPositiveCompletion(reply)) throw new Exception("!STAT"); + String replyText = sourceFtp.getReplyString(); - if ("226 RETR Transfer Complete(TRANSACTION_SUCCESS)".equals(replyText)) + //logger.info("Status: {}", replyText); + if (reply == 226) { break; - Thread.sleep(500); + } + Thread.sleep(100); } + + // if (sourceUri != null) + // sourceUri.cleanUp(); + // if (targetUri != null) + // targetUri.cleanUp(); + } private void transferMetadata(String fileName, BasicDBObject metadata) throws Exception { - try (OutputStream outStream = targetFtp.storeFileStream(fileName + JSON_EXT)) { + OutputStream outStream = null; + if (targetUri != null) + targetUri.cleanUp(); + targetFtp = ((FtpDirectoryLister) RemoteFileHandler.createLister(targetUri)).connect(); + try { + if (!targetFtp.changeWorkingDirectory(EscortFiles.STATUSFOLDER)) { + targetFtp.makeDirectory(EscortFiles.STATUSFOLDER); + if (!targetFtp.changeWorkingDirectory(EscortFiles.STATUSFOLDER)) + throw new Exception("!STATUSFOLDER"); + } + + outStream = targetFtp.storeFileStream(fileName + JSON_EXT); + if (outStream == null) { + throw new NullPointerException("Can not open: " + fileName + JSON_EXT + " Reply:" + targetFtp.getReplyString()); + } outStream.write(metadata.toString().getBytes(UTF_8)); outStream.flush(); + targetFtp.changeToParentDirectory(); } catch (Exception e) { logger.catching(e); throw e; + } finally { + if (outStream != null) + outStream.close(); + // if (targetUri != null) + // targetUri.cleanUp(); } } diff --git a/server/user.jobengine.executors/src/user/jobengine/server/steps/OctopusDataMiner.java b/server/user.jobengine.executors/src/user/jobengine/server/steps/OctopusDataMiner.java new file mode 100644 index 00000000..7bc8223e --- /dev/null +++ b/server/user.jobengine.executors/src/user/jobengine/server/steps/OctopusDataMiner.java @@ -0,0 +1,907 @@ +package user.jobengine.server.steps; + +import java.util.Date; +import java.util.HashMap; +import java.util.HashSet; +import java.util.List; +import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; + +import javax.swing.event.EventListenerList; +import javax.ws.rs.client.Entity; +import javax.ws.rs.client.Invocation.Builder; +import javax.ws.rs.core.MediaType; +import javax.ws.rs.core.Response; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.jboss.resteasy.client.jaxrs.BasicAuthentication; +import org.jboss.resteasy.client.jaxrs.ResteasyClient; +import org.jboss.resteasy.client.jaxrs.ResteasyClientBuilder; +import org.jboss.resteasy.client.jaxrs.ResteasyWebTarget; +import org.joda.time.DateTime; + +import com.ibm.nosql.json.JSONUtil; +import com.ibm.nosql.json.api.BasicDBList; +import com.ibm.nosql.json.api.BasicDBObject; +import com.ibm.nosql.json.api.DB; +import com.ibm.nosql.json.api.DBCollection; +import com.ibm.nosql.json.api.DBCursor; +import com.ibm.nosql.json.api.DBObject; +import com.ibm.nosql.json.api.QueryBuilder; +import com.ibm.nosql.json.api.WriteResult; + +import user.commons.ListUtils; +import user.commons.nosql.NoSQLUtils; +import user.commons.octopus.IOctopusAPI; +import user.commons.octopus.OctopusAPI; +import user.commons.remotestore.IProgressEventListener; +import user.commons.remotestore.ProgressEvent; + +public class OctopusDataMiner implements Runnable { + private static final String _TMP = "_tmp"; + private static final Logger logger = LogManager.getLogger(); + private static final String LINEFEED = "\r\n"; + private static final String SIMPLE_LINEFEED = "\n"; + private static final String SAVING_STORY_ID = "Saving story {}"; + // private static final String SAVING_RUNDOWN = "Saving rundown : {} {}"; + // private static final String CHECKING_RUNDOWN = "Checking Rundown {} ({}/{})"; + private static final String FIELDS_STORYFOLDER_STORIES = "stories,Story.modified,Story.name,Story.id,Story.mosObjects,Story.script,Story.type,Story.format,Story.customColumns,CustomColumn.label,CustomColumn.value"; + private static final String FIELDS_RUNDOWN_STORIES = "slugs,Slug.story,Slug.position,Story.name,Story.id,Story.modified,Story.mosObjects,Story.script,Story.type,Story.format,Story.customColumns,CustomColumn.label,CustomColumn.value"; + private static final String FIELDS_RUNDOWN_STORYIDS = "id,name,modified,scheduledStart,channel,Channel.name,rundownType,RundownType.name,slugs,Slug.storyId,Slug.position"; + private static final String FIELDS_STORY_FOLDER_LIST = "id,name,modified,stories,Story.id"; + private static final String RUNDOWN = "Rundown"; + private static final String OCTOPUS_DEVICE_NAME = "Octopus-Device-Name"; + private static final String OCTOPUS_DEVICE_ID = "Octopus-Device-Id"; + private static final String FIELDS = "fields"; + private static final String CHECKING_STORY_FOLDER = "Checking StoryFolder %s (%d/%d)"; + private static final String EXIT = "Exit"; + private static final String RESULT = "result"; + private static final String STORY_FOLDER = "StoryFolder"; + private static final String ENTER = "Enter"; + private static final String FINISHED = "Finished"; + private static final String STARTING = "Starting"; + private static final String MOSOBJECT = "Bejátszó: "; + + private DB db; + private ResteasyWebTarget webTarget; + private String apiUser; + private String apiPwd; + private HashSet storyIDs = new HashSet<>(); + private EventListenerList progressListenerList; + private ProgressEvent progressEvent = new ProgressEvent(this, 0); + private Map storyRundowns; + private Map storyStoryFolders; + private Map storedStoryRundowns; + private Map storedStoryStoryFolders; + private Map storedStoryMosObjects; + private String RUNDOWN_COLLECTION = IOctopusAPI.RUNDOWN_COLLECTION; + private String FOLDER_COLLECTION = IOctopusAPI.FOLDER_COLLECTION; + private String STORY_COLLECTION = IOctopusAPI.STORY_COLLECTION; + + private Map newRundowns = new HashMap<>(); + private Map newStoryFolders = new HashMap<>(); + private Map newStories = new HashMap<>(); + + public OctopusDataMiner() { + db = NoSQLUtils.getNoSQLDB(); + + String apiAddress = System.getProperty("jobengine.octopus.api.address"); + apiUser = System.getProperty("jobengine.octopus.api.user"); + apiPwd = System.getProperty("jobengine.octopus.api.password"); + + // ResteasyClient client = new ResteasyClientBuilder().register(JacksonJsonProvider.class).build(); + ResteasyClient client = new ResteasyClientBuilder().build(); + webTarget = client.target(apiAddress).register(new BasicAuthentication(apiUser, apiPwd)); + } + + public void addProgressListener(IProgressEventListener listener) { + if (progressListenerList == null) + progressListenerList = new EventListenerList(); + progressListenerList.add(IProgressEventListener.class, listener); + } + + private Map buildFolderReferences(BasicDBList storyFolders) { + Map result = new HashMap<>(); + List storyFolderList = NoSQLUtils.asList(storyFolders); + for (BasicDBObject storyFolder : storyFolderList) { + long storyFolderId = storyFolder.getLong(IOctopusAPI.ID); + List stories = NoSQLUtils.asList(storyFolder, IOctopusAPI.STORIES); + if (stories == null) + continue; + long position = 1; + for (BasicDBObject story : stories) { + long storyId = story.getLong(IOctopusAPI.ID); + BasicDBList references = result.get(storyId); + if (references == null) { + references = new BasicDBList(); + result.put(storyId, references); + } + references.add(new BasicDBObject(IOctopusAPI.ID, storyFolderId).append(IOctopusAPI.POSITION, position++)); + } + } + return result; + } + + private Map buildRundownReferences(BasicDBList rundowns) { + Map result = new HashMap<>(); + List rundownsList = NoSQLUtils.asList(rundowns); + for (BasicDBObject rundown : rundownsList) { + if (!rundown.containsKey(IOctopusAPI.ID)) + continue; + long rundownId = rundown.getLong(IOctopusAPI.ID); + List slugs = NoSQLUtils.asList(rundown, IOctopusAPI.SLUGS); + if (slugs == null) + continue; + for (BasicDBObject slug : slugs) { + if (!slug.containsKey(IOctopusAPI.STORYID)) + continue; + long storyId = slug.getLong(IOctopusAPI.STORYID); + BasicDBList references = result.get(storyId); + if (references == null) { + references = new BasicDBList(); + result.put(storyId, references); + } + long position = slug.getLong(IOctopusAPI.POSITION); + if (slug.containsKey(IOctopusAPI.POSITION)) + position = slug.getLong(IOctopusAPI.POSITION); + references.add(new BasicDBObject(IOctopusAPI.ID, rundownId).append(IOctopusAPI.POSITION, position)); + } + } + return result; + } + + private void buildStoriesReferences() { + DBCollection collection = db.getCollection(STORY_COLLECTION); + DBCursor cursor = collection.find(null, new BasicDBObject(IOctopusAPI.ID, 1).append(IOctopusAPI.REF_RUNDOWN, 1).append(IOctopusAPI.REF_STORYFOLDER, 1) + .append(IOctopusAPI.MOS_OBJECTS, 1)); + //DBCursor find = collection.find(QueryBuilder.start(ID).greaterThan(0).get()); + try { + + while (cursor.hasNext()) { + BasicDBObject story = (BasicDBObject) cursor.next(); + long storyId = story.getLong(IOctopusAPI.ID); + BasicDBList rundownRef = NoSQLUtils.asDBList(story, IOctopusAPI.REF_RUNDOWN); + if (rundownRef != null) { + if (storedStoryRundowns == null) + storedStoryRundowns = new HashMap<>(); + storedStoryRundowns.put(storyId, rundownRef); + } + BasicDBList storyFolderRef = NoSQLUtils.asDBList(story, IOctopusAPI.REF_STORYFOLDER); + if (storyFolderRef != null) { + if (storedStoryStoryFolders == null) + storedStoryStoryFolders = new HashMap<>(); + storedStoryStoryFolders.put(storyId, storyFolderRef); + } + + BasicDBList storyMosObjects = NoSQLUtils.asDBList(story, IOctopusAPI.MOS_OBJECTS); + if (storyMosObjects != null) { + if (storedStoryMosObjects == null) + storedStoryMosObjects = new HashMap<>(); + storedStoryMosObjects.put(storyId, storyMosObjects); + } + } + } catch (Exception e) { + logger.catching(e); + throw e; + } finally { + + } + } + + public void clear() { + db.getCollection(RUNDOWN_COLLECTION).remove(); + db.getCollection(STORY_COLLECTION).remove(); + db.getCollection(FOLDER_COLLECTION).remove(); + db.getCollection(IOctopusAPI.TIME_COLLECTION_NAME).remove(); + } + + private String concatParentsToStoryFolder(BasicDBObject actual, String name) { + + String fields = "name,id,parent"; + Response response = query("StoryFolder/" + actual.getLong("id"), fields).get(); + String json = response.readEntity(String.class); + BasicDBObject resultObject = (BasicDBObject) JSONUtil.jsonToDbObject(json); + BasicDBObject res = (BasicDBObject) resultObject.get("result"); + BasicDBObject parent = (BasicDBObject) res.get("parent"); + if (parent == null || parent.isEmpty()) + return name; + String parentName = parent.getString(IOctopusAPI.NAME); + String actualName = actual.getString(IOctopusAPI.NAME); + String newName = String.format("%s/%s", parentName, actualName); + return concatParentsToStoryFolder(parent, newName); + } + + private void deleteDiff(String oldCollectionName, String newCollectionName, String idFieldName) { + DBCollection oldCollection = db.getCollection(oldCollectionName); + DBCollection newCollection = db.getCollection(newCollectionName); + DBCursor oldCollectionCursor = oldCollection.find(new BasicDBObject(), new BasicDBObject(idFieldName, 1)); + if (!oldCollectionCursor.hasNext()) { + logger.error("{} collection is empty", newCollectionName); + return; + } + List oldItems = ListUtils.cast(oldCollectionCursor.toArray()); + + DBCursor newCollectionCursor = newCollection.find(); + ConcurrentHashMap newItems = null; + if (newCollectionCursor.hasNext()) { + List newList = ListUtils.cast(newCollectionCursor.toArray()); + newItems = ListUtils.map(newList, item -> item.getLong(IOctopusAPI.ID)); + } + if (newItems == null) + newItems = new ConcurrentHashMap<>(); + + for (BasicDBObject oldItem : oldItems) { + if (oldItem == null) { + logger.error("Item is null"); + continue; + } + if (!oldItem.containsKey(idFieldName)) { + logger.error("{} is null", idFieldName); + continue; + } + long id = oldItem.getLong(idFieldName); + BasicDBObject newItem = newItems.get(id); + if (newItem == null) { + //remove + logger.info("Deleting {}", oldItem.toPrettyString(null)); + oldCollection.remove(new BasicDBObject(idFieldName, id)); + } + } + } + + private void deleteOrphanStories() { + try { + DBCollection collection = db.getCollection(STORY_COLLECTION); + BasicDBObject query = (BasicDBObject) QueryBuilder.start().put("id").notIn(storyIDs.toArray()).get(); + WriteResult res = collection.remove(query); + logger.trace(String.format("Deleted orphan stories: %d", res.getN())); + } catch (Exception e) { + logger.error(e); + } + } + + private void ensureIndexes() { + DBCollection collection = db.getCollection(FOLDER_COLLECTION); + if (collection.count() == 0) + collection.ensureIndex(IOctopusAPI.ID); + collection = db.getCollection(RUNDOWN_COLLECTION); + if (collection.count() == 0) { + collection.ensureIndex(IOctopusAPI.ID); + collection.ensureIndex(IOctopusAPI.SCHEDULED_START); + } + collection = db.getCollection(STORY_COLLECTION); + if (collection.count() == 0) + collection.ensureIndex(IOctopusAPI.ID); + } + + public void execute() throws Exception { + logger.trace(STARTING); + //{"filter" :{ "archived" : true }} + Response response = query(RUNDOWN, "id,name,modified,scheduledStart,channel,Channel.name,rundownType,RundownType.name") + .post(Entity.entity(new BasicDBObject("filter", new BasicDBObject("archived", true)).toPrettyString(null), MediaType.APPLICATION_JSON)); + String json = response.readEntity(String.class); + + RUNDOWN_COLLECTION = IOctopusAPI.RUNDOWN_COLLECTION + _TMP; + FOLDER_COLLECTION = IOctopusAPI.FOLDER_COLLECTION + _TMP; + STORY_COLLECTION = IOctopusAPI.STORY_COLLECTION + _TMP; + + try { + db.getCollection(RUNDOWN_COLLECTION).drop(); + db.getCollection(FOLDER_COLLECTION).drop(); + db.getCollection(STORY_COLLECTION).drop(); + } catch (Exception e) { + logger.catching(e); + throw e; + } + + BasicDBList rundowns = null; + BasicDBList storyFolders = null; + + try { + rundowns = queryBuildRefRundowns(); + storyFolders = queryBuildRefFolders(); + } catch (Exception e) { + logger.catching(e); + throw e; + } + + processRundowns(rundowns); + processStoryFolders(storyFolders); + + //a sorrend fontos ! + updateDiff(IOctopusAPI.STORY_COLLECTION, STORY_COLLECTION, IOctopusAPI.ID); + updateDiff(IOctopusAPI.RUNDOWN_COLLECTION, RUNDOWN_COLLECTION, IOctopusAPI.ID); + updateDiff(IOctopusAPI.FOLDER_COLLECTION, FOLDER_COLLECTION, IOctopusAPI.ID); + deleteDiff(IOctopusAPI.RUNDOWN_COLLECTION, RUNDOWN_COLLECTION, IOctopusAPI.ID); + //deleteDiff(IOctopusAPI.FOLDER_COLLECTION, FOLDER_COLLECTION, IOctopusAPI.ID); + deleteDiff(IOctopusAPI.STORY_COLLECTION, STORY_COLLECTION, IOctopusAPI.ID); + + //setLastUpdateTime(new Date()); + logger.info("Activate"); + + // db.getCollection(RUNDOWN_COLLECTION).rename(IOctopusAPI.RUNDOWN_COLLECTION, true); + // db.getCollection(FOLDER_COLLECTION).rename(IOctopusAPI.FOLDER_COLLECTION, true); + // db.getCollection(STORY_COLLECTION).rename(IOctopusAPI.STORY_COLLECTION, true); + logger.trace(FINISHED); + + } + + public void executetest() { + // ResteasyWebTarget target = webTarget.path(RUNDOWN); + // Builder result = target.request().header(OCTOPUS_DEVICE_ID, apiUser).header(OCTOPUS_DEVICE_NAME, apiPwd); + // Response r = result.get(); + // String x = r.readEntity(String.class); + // logger.info(x); + // return; + } + + private String extractContent(BasicDBObject content) { + String scriptContent = ""; + if (!content.containsKey(IOctopusAPI.TYPE)) + return scriptContent; + String type = content.getString(IOctopusAPI.TYPE); + switch (type) { + case IOctopusAPI.TEXT: { + if (content.containsKey(IOctopusAPI.TEXT)) { + String text = content.getString(IOctopusAPI.TEXT); + if (text != null) + scriptContent += String.format("%s%s", text.replaceAll(SIMPLE_LINEFEED, LINEFEED), LINEFEED); + } + break; + } + case IOctopusAPI.MOS: { + BasicDBObject mosObject = NoSQLUtils.asDBObject(content, IOctopusAPI.OBJECT); + if (mosObject != null && !mosObject.isEmpty()) { + if (mosObject.containsKey(IOctopusAPI.OBJ_ID)) { + String objID = mosObject.getString(IOctopusAPI.OBJ_ID); + scriptContent += String.format("%s %s%s", MOSOBJECT, objID, LINEFEED); + } + } + break; + } + default: { + if (content.containsKey(IOctopusAPI.CONTENT)) { + List innerContents = NoSQLUtils.asList(content, IOctopusAPI.CONTENT); + if (innerContents != null) { + for (BasicDBObject actualInnerContent : innerContents) { + if (actualInnerContent != null && actualInnerContent.isEmpty()) + scriptContent += extractContent(actualInnerContent); + } + } + } + break; + } + } + return scriptContent; + } + + private String extractCustomColumnValue(String columnName, BasicDBObject story) { + List customColumns = NoSQLUtils.asList(story, IOctopusAPI.CUSTOM_COLUMNS); + if (customColumns == null) + return null; + String result = null; + for (BasicDBObject customColumn : customColumns) { + if (!customColumn.containsKey(IOctopusAPI.LABEL)) + continue; + String currentName = customColumn.getString(IOctopusAPI.LABEL); + if (currentName == null) + continue; + if (!currentName.toLowerCase().equals(columnName.toLowerCase())) + continue; + if (!customColumn.containsKey(IOctopusAPI.VALUE)) + continue; + result = customColumn.getString(IOctopusAPI.VALUE); + break; + } + return result; + } + + private BasicDBList extractRelevantMOSObjects(BasicDBObject story) { + List mosObjects = NoSQLUtils.asList(story, IOctopusAPI.MOS_OBJECTS); + if (mosObjects == null) + return null; + BasicDBList result = null; + for (BasicDBObject mosObject : mosObjects) { + if (!mosObject.containsKey(IOctopusAPI.MOS_ID)) + continue; + String mosId = mosObject.getString(IOctopusAPI.MOS_ID); + if (!IOctopusAPI.NEXIO_MOS.equals(mosId)) + continue; + if (result == null) + result = new BasicDBList(); + result.add(mosObject); + } + return result; + } + + private String extractScriptContent(BasicDBObject story) { + BasicDBObject script = NoSQLUtils.asDBObject(story, IOctopusAPI.SCRIPT); + if (script == null || script.isEmpty()) + return null; + + List body = NoSQLUtils.asList(script, IOctopusAPI.BODY); + if (body == null || body.size() == 0) + return null; + + StringBuilder sb = new StringBuilder(); + + for (BasicDBObject bodyItem : body) { + if (bodyItem.containsKey(IOctopusAPI.LABEL)) { + sb.append(bodyItem.getString(IOctopusAPI.LABEL)); + sb.append(LINEFEED); + } + List contents = NoSQLUtils.asList(bodyItem, IOctopusAPI.CONTENT); + if (contents == null) + continue; + for (BasicDBObject content : contents) { + sb.append(extractContent(content)); + } + } + return sb.length() == 0 ? null : sb.toString(); + } + + private void fireProgressEvent(ProgressEvent evt) { + logger.debug("Progress changed to " + evt.getProgress() + "%"); + if (progressListenerList == null) + return; + Object[] listeners = progressListenerList.getListenerList(); + for (int i = 0; i < listeners.length; i += 2) { + if (listeners[i] == IProgressEventListener.class) + ((IProgressEventListener) listeners[i + 1]).progressChanged(evt); + } + } + + private Date getLastUpdateTime() { + Date result = null; + DBCollection collection = db.getCollection(IOctopusAPI.TIME_COLLECTION_NAME); + DBObject timeObject = collection.findOne(); + if (timeObject != null) + result = (Date) timeObject.get(IOctopusAPI.LASTUPDATE_TIME); + return result; + } + + // private boolean isModified(Date date, BasicDBObject object, String name) { + // Date actualModifiedString = toDate(object, name); + // if (actualModifiedString == null) + // logger.trace(ACTUAL_MODIFIED_STRING_IS_NULL); + // int result = date.compareTo(actualModifiedString); + // return result <= 0; + // } + + private boolean isModified(Date date, BasicDBObject object) { + if (date == null) + return true; + Date modified = (Date) object.get(IOctopusAPI.MODIFIED); + int result = date.compareTo(modified); + return result <= 0; + } + + private void processRundowns(BasicDBList rundowns) throws Exception { + if (rundowns == null || rundowns.size() == 0) { + progressEvent.setProgress(50); + fireProgressEvent(progressEvent); + } else { + logger.info("Process rundowns"); + try { + storeRundowns(rundowns, null); + } catch (Exception e) { + logger.catching(e); + throw e; + } + } + } + + private void processStoryFolders(BasicDBList storyFolders) throws Exception { + if (storyFolders == null || storyFolders.size() == 0) { + progressEvent.setProgress(100); + fireProgressEvent(progressEvent); + } else { + logger.info("Process story folders"); + try { + storeStoryFolders(storyFolders, null); + } catch (Exception e) { + logger.catching(e); + throw e; + } + } + } + + private Builder query(String path, String fields) { + //logger.info("Class loader {}", getClass().getClassLoader()); + // try { + // //TODO kell e? + // ResteasyDeployment deployment = new ResteasyDeployment(); + // deployment.start(); + // } catch (Exception e) { + // logger.catching(e); + // } + ResteasyWebTarget target = webTarget.path(path).queryParam(FIELDS, fields); + Builder result = target.request().header(OCTOPUS_DEVICE_ID, apiUser).header(OCTOPUS_DEVICE_NAME, apiPwd); + return result; + } + + private BasicDBList queryBuildRefFolders() { + BasicDBList storyFolders; + logger.info("Fetch story folders"); + storyFolders = queryStoryFolders(); + logger.info("Fetch folder story references"); + storyStoryFolders = buildFolderReferences(storyFolders); + return storyFolders; + } + + private BasicDBList queryBuildRefRundowns() { + BasicDBList rundowns; + logger.info("Fetch rundowns"); + rundowns = queryRundowns(); + logger.info("Fetch rundown story references"); + storyRundowns = buildRundownReferences(rundowns); + return rundowns; + } + + private BasicDBObject queryRundown(BasicDBObject rundown) { + logger.trace(ENTER); + BasicDBObject result = null; + long id = NoSQLUtils.asLong(rundown, IOctopusAPI.ID); + Response response = query(String.format("%s/", RUNDOWN) + id, FIELDS_RUNDOWN_STORIES).get(); + String json = response.readEntity(String.class); + BasicDBObject resultObject = (BasicDBObject) JSONUtil.jsonToDbObject(json); + if (resultObject == null) + logger.error("Rundown {} {} is not available", id, rundown.getString(IOctopusAPI.NAME)); + else + result = NoSQLUtils.asDBObject(resultObject, RESULT); + logger.trace(EXIT); + return result; + } + + private BasicDBList queryRundowns() { + logger.trace(ENTER); + BasicDBList result = null; + Response response = query(RUNDOWN, FIELDS_RUNDOWN_STORYIDS).get(); + String json = response.readEntity(String.class); + BasicDBObject resultObject = (BasicDBObject) JSONUtil.jsonToDbObject(json); + if (resultObject != null) + result = NoSQLUtils.asDBList(resultObject, RESULT); + logger.trace(EXIT); + return result; + } + + private BasicDBObject queryStoryFolder(BasicDBObject storyFolder) { + logger.trace(ENTER); + BasicDBObject result = null; + long id = NoSQLUtils.asLong(storyFolder, IOctopusAPI.ID); + Response response = query(String.format("%s/", STORY_FOLDER) + id, FIELDS_STORYFOLDER_STORIES).get(); + String json = response.readEntity(String.class); + BasicDBObject resultObject = (BasicDBObject) JSONUtil.jsonToDbObject(json); + if (resultObject == null) + logger.error("StoryFolder {} {} is not available", id, storyFolder.getString(IOctopusAPI.NAME)); + else + result = NoSQLUtils.asDBObject(resultObject, RESULT); + logger.trace(EXIT); + return result; + } + + private BasicDBList queryStoryFolders() { + logger.trace(ENTER); + BasicDBList result = null; + Response response = query(STORY_FOLDER, FIELDS_STORY_FOLDER_LIST).get(); + String json = response.readEntity(String.class); + BasicDBObject resultObject = (BasicDBObject) JSONUtil.jsonToDbObject(json); + if (resultObject != null) + result = NoSQLUtils.asDBList(resultObject, RESULT); + + /* teszt */ + List list = NoSQLUtils.asList(result); + for (BasicDBObject actual : list) { + String fullName = concatParentsToStoryFolder(actual, actual.getString(IOctopusAPI.NAME)); + //logger.info("Checking StoryFolder {}", fullName); + actual.remove(IOctopusAPI.NAME); + actual.append(IOctopusAPI.NAME, fullName); + } + + logger.trace(EXIT); + return result; + } + + public void removeProgressListener(IProgressEventListener listener) { + progressListenerList.remove(IProgressEventListener.class, listener); + } + + @Override + public void run() { + logger.trace(STARTING); + try { + ensureIndexes(); + Date lastUpdateTime = getLastUpdateTime(); + + buildStoriesReferences(); + + BasicDBList rundowns = queryRundowns(); + storyRundowns = buildRundownReferences(rundowns); + BasicDBList storyFolders = queryStoryFolders(); + storyStoryFolders = buildFolderReferences(storyFolders); + + if (rundowns == null || rundowns.size() == 0) { + progressEvent.setProgress(50); + fireProgressEvent(progressEvent); + } else { + storeRundowns(rundowns, lastUpdateTime); + } + + if (storyFolders == null || storyFolders.size() == 0) { + progressEvent.setProgress(100); + fireProgressEvent(progressEvent); + } else { + storeStoryFolders(storyFolders, lastUpdateTime); + } + deleteOrphanStories(); + setLastUpdateTime(new Date()); + } catch (Exception e) { + logger.catching(e); + throw e; + } + logger.trace(FINISHED); + } + + public void run(boolean forceFull) { + if (forceFull) { + clear(); + } + run(); + // if (forceFull) { + // RUNDOWN_COLLECTION_NAME = "rundowns"; + // STORY_COLLECTION_NAME = "stories"; + // STORY_FOLDER_COLLECTION_NAME = "storyfolders"; + // TIME_COLLECTION_NAME = "octopusSyncTime"; + // db.getCollection("rundowns_tmp").rename(RUNDOWN_COLLECTION_NAME, true); + // db.getCollection("stories_tmp").rename(STORY_COLLECTION_NAME, true); + // db.getCollection("storyfolders_tmp").rename(STORY_FOLDER_COLLECTION_NAME, true); + // db.getCollection("octopusSyncTime_tmp").rename(TIME_COLLECTION_NAME, true); + // //clear(); + // } + + //TODO reset collection names + } + + public void setLastUpdateTime(Date lastUpdateTime) { + DBCollection collection = db.getCollection(OctopusAPI.TIME_COLLECTION_NAME); + DBObject timeObject = collection.findOne(); + if (timeObject == null) + timeObject = new BasicDBObject(); + timeObject.put(IOctopusAPI.LASTUPDATE_TIME, lastUpdateTime); + + collection.save(timeObject); + } + + private void storeRundown(BasicDBObject rundown, Date lastUpdateTime) { + logger.trace(ENTER); + String name = rundown.containsKey(IOctopusAPI.NAME) ? rundown.getString(IOctopusAPI.NAME) : null; + logger.debug("Storing rundown {} {}", name, rundown.get(IOctopusAPI.SCHEDULED_START)); + BasicDBObject rundownWithStories = queryRundown(rundown); + if (rundownWithStories != null) { + BasicDBList stories = NoSQLUtils.asDBList(rundownWithStories, IOctopusAPI.SLUGS); + if (stories != null) + storeRundownStories(stories, lastUpdateTime); + rundown.put(IOctopusAPI.SCHEDULED_START, toDate(rundown, IOctopusAPI.SCHEDULED_START)); + rundown.put(IOctopusAPI.MODIFIED, toDate(rundown, IOctopusAPI.MODIFIED)); + DBCollection collection = db.getCollection(RUNDOWN_COLLECTION); + if (lastUpdateTime == null || (lastUpdateTime != null && isModified(lastUpdateTime, rundown))) { + //logger.debug(SAVING_RUNDOWN, rundownID, name); + collection.save(rundown); + } + } + logger.trace(EXIT); + } + + private void storeRundowns(BasicDBList rundowns, Date lastUpdateTime) { + logger.trace(ENTER); + List rundownsList = NoSQLUtils.asList(rundowns); + int idx = 1; + for (BasicDBObject rundown : rundownsList) { + //logger.info(CHECKING_RUNDOWN, rundown.getLong(IOctopusAPI.ID), rundownsList.size(), idx); + storeRundown(rundown, lastUpdateTime); + int progress = idx * 50 / rundownsList.size(); + if (progress - progressEvent.getProgress() > 0) { + progressEvent.setProgress(progress); + fireProgressEvent(progressEvent); + } + + idx++; + } + logger.trace(EXIT); + } + + private void storeRundownStories(BasicDBList slugs, Date lastUpdateTime) { + logger.trace(ENTER); + List slugsList = NoSQLUtils.asList(slugs); + for (BasicDBObject slug : slugsList) { + if (slug.containsKey(IOctopusAPI.STORY)) + storeStory((BasicDBObject) slug.get(IOctopusAPI.STORY), lastUpdateTime); + } + logger.trace(EXIT); + } + + private void storeStory(BasicDBObject story, Date lastUpdateTime) { + logger.trace(ENTER); + if (!story.containsKey(IOctopusAPI.ID)) { + logger.error("Missing id in story {}", story.toPrettyString(null)); + return; + } + long storyID = story.getLong(IOctopusAPI.ID); + boolean isContains = storyIDs.contains(storyID); + if (isContains) + return; + storyIDs.add(storyID); + story.put(IOctopusAPI.MODIFIED, toDate(story, IOctopusAPI.MODIFIED)); + BasicDBList rundownRef = storyRundowns.get(storyID); + BasicDBList storyFolderRef = storyStoryFolders.get(storyID); + BasicDBList modifiedMOS = extractRelevantMOSObjects(story); + + if (lastUpdateTime != null) { + rundownRef = (rundownRef == null) ? new BasicDBList() : rundownRef; + storyFolderRef = (storyFolderRef == null) ? new BasicDBList() : storyFolderRef; + modifiedMOS = (modifiedMOS == null) ? new BasicDBList() : modifiedMOS; + + boolean uptodate = true; + if (!isModified(lastUpdateTime, story)) { + BasicDBList storedRundownRef = storedStoryRundowns.get(storyID); + storedRundownRef = (storedRundownRef == null) ? new BasicDBList() : storedRundownRef; + uptodate = storedRundownRef.equals(rundownRef); + + if (uptodate) { + BasicDBList storedStoryFolderRef = storedStoryStoryFolders.get(storyID); + storedStoryFolderRef = (storedStoryFolderRef == null) ? new BasicDBList() : storedStoryFolderRef; + uptodate = storedStoryFolderRef.equals(storyFolderRef); + } + + if (uptodate) { + BasicDBList storedMOS = storedStoryMosObjects.get(storyID); + storedMOS = (storedMOS == null) ? new BasicDBList() : storedMOS; + uptodate = storedMOS.equals(modifiedMOS); + } + + if (uptodate) + return; + } + } + + DBCollection collection = db.getCollection(STORY_COLLECTION); + if (lastUpdateTime != null) { + BasicDBObject orig = (BasicDBObject) collection.findOne(new BasicDBObject(IOctopusAPI.ID, storyID), new BasicDBObject(IOctopusAPI.ID, 1)); + if (orig != null) + story.put("_id", orig.getID()); + } + if (rundownRef != null) + story.put(IOctopusAPI.REF_RUNDOWN, rundownRef); + if (storyFolderRef != null) + story.put(IOctopusAPI.REF_STORYFOLDER, storyFolderRef); + String scriptContent = extractScriptContent(story); + story.put(IOctopusAPI.SCRIPT_CONTENT, scriptContent); + if (modifiedMOS == null || modifiedMOS.isEmpty()) { + if (story.containsKey(IOctopusAPI.MOS_OBJECTS)) + story.remove(IOctopusAPI.MOS_OBJECTS); + } else + story.put(IOctopusAPI.MOS_OBJECTS, modifiedMOS); + + String parentStoryId = extractCustomColumnValue(IOctopusAPI.PARENT_STORY_ID, story); + if (parentStoryId == null) { + logger.warn("Missing {} in story", IOctopusAPI.PARENT_STORY_ID, story.toPrettyString(null)); + story.append(IOctopusAPI.PARENT_STORY_ID, storyID); + } else + story.append(IOctopusAPI.PARENT_STORY_ID, parentStoryId); + logger.debug(SAVING_STORY_ID, storyID); + collection.save(story); + logger.trace(EXIT); + } + + private void storeStoryFolder(BasicDBObject storyFolder, Date lastUpdateTime) { + logger.trace(ENTER); + BasicDBObject storyFoldersWithStories = queryStoryFolder(storyFolder); + if (storyFoldersWithStories != null) { + BasicDBList stories = NoSQLUtils.asDBList(storyFoldersWithStories, IOctopusAPI.STORIES); + if (stories != null) + storeStoryFolderStories(stories, lastUpdateTime); + storyFolder.put(IOctopusAPI.MODIFIED, toDate(storyFolder, IOctopusAPI.MODIFIED)); + DBCollection collection = db.getCollection(FOLDER_COLLECTION); + if (lastUpdateTime == null || (lastUpdateTime != null && isModified(lastUpdateTime, storyFolder))) { + String name = storyFolder.getString(IOctopusAPI.NAME); + logger.debug("Storing story folder {}", name); + collection.save(storyFolder); + } + } + logger.trace(EXIT); + } + + private void storeStoryFolders(BasicDBList storyFolders, Date lastUpdateTime) { + logger.trace(ENTER); + List storyFolderList = NoSQLUtils.asList(storyFolders); + int idx = 1; + for (BasicDBObject storyFolder : storyFolderList) { + logger.debug(String.format(CHECKING_STORY_FOLDER, storyFolder.getLong(IOctopusAPI.ID), storyFolderList.size(), idx)); + storeStoryFolder(storyFolder, lastUpdateTime); + int progress = 50 + (idx * 50 / storyFolderList.size()); + if (progress - progressEvent.getProgress() > 0) { + progressEvent.setProgress(progress); + fireProgressEvent(progressEvent); + } + idx++; + } + logger.trace(EXIT); + } + + private void storeStoryFolderStories(BasicDBList stories, Date lastUpdateTime) { + logger.trace(ENTER); + List list = NoSQLUtils.asList(stories); + for (BasicDBObject story : list) + storeStory(story, lastUpdateTime); + logger.trace(EXIT); + } + + private Date toDate(BasicDBObject obj, String name) { + Date result = null; + if (obj.containsKey(name)) { + String dt = obj.getString(name); + if (dt != null) { + // create jodatime from date + DateTime jdt = new DateTime(dt); + result = jdt.toDate(); + } + } + return result; + } + + private void updateDiff(String oldCollectionName, String newCollectionName, String idFieldName) { + DBCollection oldCollection = db.getCollection(oldCollectionName); + DBCollection newCollection = db.getCollection(newCollectionName); + DBCursor newCollectionCursor = newCollection.find(); + if (!newCollectionCursor.hasNext()) { + logger.error("{} collection is empty", newCollectionName); + return; + } + + List newItems = ListUtils.cast(newCollectionCursor.toArray()); + + DBCursor oldCollectionCursor = oldCollection.find(); + ConcurrentHashMap oldItems = null; + if (oldCollectionCursor.hasNext()) { + List oldList = ListUtils.cast(oldCollectionCursor.toArray()); + oldItems = ListUtils.map(oldList, item -> item.getLong(IOctopusAPI.ID)); + } + if (oldItems == null) + oldItems = new ConcurrentHashMap<>(); + for (BasicDBObject newItem : newItems) { + if (newItem == null) { + logger.error("Item is null"); + continue; + } + if (!newItem.containsKey(idFieldName)) { + logger.error("{} is null", idFieldName); + continue; + } + long id = newItem.getLong(idFieldName); + BasicDBObject oldItem = oldItems.get(id); + boolean save = false; + if (oldItem == null) { + //save + save = true; + } else { + //compare + Object oldDbId = oldItem.getID(); + newItem.remove(IOctopusAPI._ID); + oldItem.remove(IOctopusAPI._ID); + + if (!newItem.equals(oldItem)) { + //save + newItem.put(IOctopusAPI._ID, oldDbId); + save = true; + } + } + + if (save) { + logger.info("Saving to {} item {}", oldCollectionName, id); + oldCollection.save(newItem); + } + } + } + +} diff --git a/server/user.jobengine.osgi.commons/src/user/commons/octopus/OctopusDataMiner.java b/server/user.jobengine.osgi.commons/src/user/commons/octopus/OctopusDataMiner.java index 77826451..73dac323 100644 --- a/server/user.jobengine.osgi.commons/src/user/commons/octopus/OctopusDataMiner.java +++ b/server/user.jobengine.osgi.commons/src/user/commons/octopus/OctopusDataMiner.java @@ -8,7 +8,9 @@ import java.util.Map; import java.util.concurrent.ConcurrentHashMap; import javax.swing.event.EventListenerList; +import javax.ws.rs.client.Entity; import javax.ws.rs.client.Invocation.Builder; +import javax.ws.rs.core.MediaType; import javax.ws.rs.core.Response; import org.apache.logging.log4j.LogManager; @@ -40,13 +42,12 @@ public class OctopusDataMiner implements Runnable { private static final String LINEFEED = "\r\n"; private static final String SIMPLE_LINEFEED = "\n"; private static final String SAVING_STORY_ID = "Saving story {}"; - private static final String SAVING_RUNDOWN = "Saving rundown : {} {}"; - private static final String CHECKING_RUNDOWN = "Checking Rundown {} ({}/{})"; - private static final String FIELDS_STORIES = "stories,Story.modified,Story.name,Story.id,Story.mosObjects,Story.script,Story.type,Story.format,Story.customColumns,CustomColumn.label,CustomColumn.value"; - private static final String FIELDS_SLUGS = "slugs,Slug.story,Slug.position,Story.name,Story.id,Story.modified,Story.mosObjects,Story.script,Story.type,Story.format,Story.customColumns,CustomColumn.label,CustomColumn.value"; - private static final String FIELDS_RUNDOWN_LIST = "id,name,modified,scheduledStart,channel,Channel.name,Channel.id,rundownType,RundownType.name,slugs,Slug.storyId,Slug.position"; + // private static final String SAVING_RUNDOWN = "Saving rundown : {} {}"; + // private static final String CHECKING_RUNDOWN = "Checking Rundown {} ({}/{})"; + private static final String FIELDS_STORYFOLDER_STORIES = "stories,Story.modified,Story.name,Story.id,Story.mosObjects,Story.script,Story.type,Story.format,Story.customColumns,CustomColumn.label,CustomColumn.value"; + private static final String FIELDS_RUNDOWN_STORIES = "slugs,Slug.story,Slug.position,Story.name,Story.id,Story.modified,Story.mosObjects,Story.script,Story.type,Story.format,Story.customColumns,CustomColumn.label,CustomColumn.value"; + private static final String FIELDS_RUNDOWN_STORYIDS = "id,name,modified,scheduledStart,channel,Channel.name,rundownType,RundownType.name,slugs,Slug.storyId,Slug.position"; private static final String FIELDS_STORY_FOLDER_LIST = "id,name,modified,stories,Story.id"; - //private static final String FIELDS_STORY_FOLDER_LIST = "id,name,modified"; private static final String RUNDOWN = "Rundown"; private static final String OCTOPUS_DEVICE_NAME = "Octopus-Device-Name"; private static final String OCTOPUS_DEVICE_ID = "Octopus-Device-Id"; @@ -212,14 +213,14 @@ public class OctopusDataMiner implements Runnable { private void deleteDiff(String oldCollectionName, String newCollectionName, String idFieldName) { DBCollection oldCollection = db.getCollection(oldCollectionName); DBCollection newCollection = db.getCollection(newCollectionName); - DBCursor oldCollectionCursor = newCollection.find(new BasicDBObject(), new BasicDBObject(idFieldName, 1)); + DBCursor oldCollectionCursor = oldCollection.find(new BasicDBObject(), new BasicDBObject(idFieldName, 1)); if (!oldCollectionCursor.hasNext()) { logger.error("{} collection is empty", newCollectionName); return; } List oldItems = ListUtils.cast(oldCollectionCursor.toArray()); - DBCursor newCollectionCursor = oldCollection.find(); + DBCursor newCollectionCursor = newCollection.find(); ConcurrentHashMap newItems = null; if (newCollectionCursor.hasNext()) { List newList = ListUtils.cast(newCollectionCursor.toArray()); @@ -274,6 +275,11 @@ public class OctopusDataMiner implements Runnable { public void execute() throws Exception { logger.trace(STARTING); + //{"filter" :{ "archived" : true }} + Response response = query(RUNDOWN, "id,name,modified,scheduledStart,channel,Channel.name,rundownType,RundownType.name") + .post(Entity.entity(new BasicDBObject("filter", new BasicDBObject("archived", true)).toPrettyString(null), MediaType.APPLICATION_JSON)); + String json = response.readEntity(String.class); + RUNDOWN_COLLECTION = IOctopusAPI.RUNDOWN_COLLECTION + _TMP; FOLDER_COLLECTION = IOctopusAPI.FOLDER_COLLECTION + _TMP; STORY_COLLECTION = IOctopusAPI.STORY_COLLECTION + _TMP; @@ -306,7 +312,7 @@ public class OctopusDataMiner implements Runnable { updateDiff(IOctopusAPI.RUNDOWN_COLLECTION, RUNDOWN_COLLECTION, IOctopusAPI.ID); updateDiff(IOctopusAPI.FOLDER_COLLECTION, FOLDER_COLLECTION, IOctopusAPI.ID); deleteDiff(IOctopusAPI.RUNDOWN_COLLECTION, RUNDOWN_COLLECTION, IOctopusAPI.ID); - deleteDiff(IOctopusAPI.FOLDER_COLLECTION, FOLDER_COLLECTION, IOctopusAPI.ID); + //deleteDiff(IOctopusAPI.FOLDER_COLLECTION, FOLDER_COLLECTION, IOctopusAPI.ID); deleteDiff(IOctopusAPI.STORY_COLLECTION, STORY_COLLECTION, IOctopusAPI.ID); //setLastUpdateTime(new Date()); @@ -535,7 +541,7 @@ public class OctopusDataMiner implements Runnable { logger.trace(ENTER); BasicDBObject result = null; long id = NoSQLUtils.asLong(rundown, IOctopusAPI.ID); - Response response = query(String.format("%s/", RUNDOWN) + id, FIELDS_SLUGS).get(); + Response response = query(String.format("%s/", RUNDOWN) + id, FIELDS_RUNDOWN_STORIES).get(); String json = response.readEntity(String.class); BasicDBObject resultObject = (BasicDBObject) JSONUtil.jsonToDbObject(json); if (resultObject == null) @@ -549,7 +555,7 @@ public class OctopusDataMiner implements Runnable { private BasicDBList queryRundowns() { logger.trace(ENTER); BasicDBList result = null; - Response response = query(RUNDOWN, FIELDS_RUNDOWN_LIST).get(); + Response response = query(RUNDOWN, FIELDS_RUNDOWN_STORYIDS).get(); String json = response.readEntity(String.class); BasicDBObject resultObject = (BasicDBObject) JSONUtil.jsonToDbObject(json); if (resultObject != null) @@ -562,7 +568,7 @@ public class OctopusDataMiner implements Runnable { logger.trace(ENTER); BasicDBObject result = null; long id = NoSQLUtils.asLong(storyFolder, IOctopusAPI.ID); - Response response = query(String.format("%s/", STORY_FOLDER) + id, FIELDS_STORIES).get(); + Response response = query(String.format("%s/", STORY_FOLDER) + id, FIELDS_STORYFOLDER_STORIES).get(); String json = response.readEntity(String.class); BasicDBObject resultObject = (BasicDBObject) JSONUtil.jsonToDbObject(json); if (resultObject == null) diff --git a/server/user.jobengine.osgi.server/pages/index.zul b/server/user.jobengine.osgi.server/pages/index.zul index 7b98f918..c39b3fd1 100644 --- a/server/user.jobengine.osgi.server/pages/index.zul +++ b/server/user.jobengine.osgi.server/pages/index.zul @@ -57,7 +57,7 @@ - +
diff --git a/server/user.jobengine.osgi.server/src/user/jobengine/server/steps/EscortFiles.java b/server/user.jobengine.osgi.server/src/user/jobengine/server/steps/EscortFiles.java index 998d84f2..57509664 100644 --- a/server/user.jobengine.osgi.server/src/user/jobengine/server/steps/EscortFiles.java +++ b/server/user.jobengine.osgi.server/src/user/jobengine/server/steps/EscortFiles.java @@ -67,12 +67,13 @@ public class EscortFiles { root.setAttribute("extendedId", fileName); if (killDate != null) { String sKillDate = CalendarUtils.toString(CalendarUtils.createCalendar(killDate), "MM-dd-yyyy"); - root.appendChild(xmlDocument.createElement("KillDate")).setNodeValue(sKillDate); + root.appendChild(xmlDocument.createElement("KillDate")).appendChild(xmlDocument.createTextNode(sKillDate)); } + if (StringUtils.isNotBlank(description)) - root.appendChild(xmlDocument.createElement("ExtendedDescription")).setNodeValue(description); + root.appendChild(xmlDocument.createElement("ExtendedDescription")).appendChild(xmlDocument.createTextNode(description)); if (StringUtils.isNotBlank(agency)) - root.appendChild(xmlDocument.createElement("ExtendedAgency")).setNodeValue(agency); + root.appendChild(xmlDocument.createElement("ExtendedAgency")).appendChild(xmlDocument.createTextNode(agency)); xmlDocument.appendChild(root); DOMSource domSource = new DOMSource(xmlDocument); -- 2.54.0