--- /dev/null
+package user.jobengine.server.steps;\r
+\r
+import java.util.Date;\r
+import java.util.HashMap;\r
+import java.util.HashSet;\r
+import java.util.List;\r
+import java.util.Map;\r
+import java.util.concurrent.ConcurrentHashMap;\r
+\r
+import javax.swing.event.EventListenerList;\r
+import javax.ws.rs.client.Entity;\r
+import javax.ws.rs.client.Invocation.Builder;\r
+import javax.ws.rs.core.MediaType;\r
+import javax.ws.rs.core.Response;\r
+\r
+import org.apache.logging.log4j.LogManager;\r
+import org.apache.logging.log4j.Logger;\r
+import org.jboss.resteasy.client.jaxrs.BasicAuthentication;\r
+import org.jboss.resteasy.client.jaxrs.ResteasyClient;\r
+import org.jboss.resteasy.client.jaxrs.ResteasyClientBuilder;\r
+import org.jboss.resteasy.client.jaxrs.ResteasyWebTarget;\r
+import org.joda.time.DateTime;\r
+\r
+import com.ibm.nosql.json.JSONUtil;\r
+import com.ibm.nosql.json.api.BasicDBList;\r
+import com.ibm.nosql.json.api.BasicDBObject;\r
+import com.ibm.nosql.json.api.DB;\r
+import com.ibm.nosql.json.api.DBCollection;\r
+import com.ibm.nosql.json.api.DBCursor;\r
+import com.ibm.nosql.json.api.DBObject;\r
+import com.ibm.nosql.json.api.QueryBuilder;\r
+import com.ibm.nosql.json.api.WriteResult;\r
+\r
+import user.commons.ListUtils;\r
+import user.commons.nosql.NoSQLUtils;\r
+import user.commons.octopus.IOctopusAPI;\r
+import user.commons.octopus.OctopusAPI;\r
+import user.commons.remotestore.IProgressEventListener;\r
+import user.commons.remotestore.ProgressEvent;\r
+\r
+public class OctopusDataMiner implements Runnable {\r
+ private static final String _TMP = "_tmp";\r
+ private static final Logger logger = LogManager.getLogger();\r
+ private static final String LINEFEED = "\r\n";\r
+ private static final String SIMPLE_LINEFEED = "\n";\r
+ private static final String SAVING_STORY_ID = "Saving story {}";\r
+ // private static final String SAVING_RUNDOWN = "Saving rundown : {} {}";\r
+ // private static final String CHECKING_RUNDOWN = "Checking Rundown {} ({}/{})";\r
+ private static final String FIELDS_STORYFOLDER_STORIES = "stories,Story.modified,Story.name,Story.id,Story.mosObjects,Story.script,Story.type,Story.format,Story.customColumns,CustomColumn.label,CustomColumn.value";\r
+ private static final String FIELDS_RUNDOWN_STORIES = "slugs,Slug.story,Slug.position,Story.name,Story.id,Story.modified,Story.mosObjects,Story.script,Story.type,Story.format,Story.customColumns,CustomColumn.label,CustomColumn.value";\r
+ private static final String FIELDS_RUNDOWN_STORYIDS = "id,name,modified,scheduledStart,channel,Channel.name,rundownType,RundownType.name,slugs,Slug.storyId,Slug.position";\r
+ private static final String FIELDS_STORY_FOLDER_LIST = "id,name,modified,stories,Story.id";\r
+ private static final String RUNDOWN = "Rundown";\r
+ private static final String OCTOPUS_DEVICE_NAME = "Octopus-Device-Name";\r
+ private static final String OCTOPUS_DEVICE_ID = "Octopus-Device-Id";\r
+ private static final String FIELDS = "fields";\r
+ private static final String CHECKING_STORY_FOLDER = "Checking StoryFolder %s (%d/%d)";\r
+ private static final String EXIT = "Exit";\r
+ private static final String RESULT = "result";\r
+ private static final String STORY_FOLDER = "StoryFolder";\r
+ private static final String ENTER = "Enter";\r
+ private static final String FINISHED = "Finished";\r
+ private static final String STARTING = "Starting";\r
+ private static final String MOSOBJECT = "Bejátszó: ";\r
+\r
+ private DB db;\r
+ private ResteasyWebTarget webTarget;\r
+ private String apiUser;\r
+ private String apiPwd;\r
+ private HashSet<Long> storyIDs = new HashSet<>();\r
+ private EventListenerList progressListenerList;\r
+ private ProgressEvent progressEvent = new ProgressEvent(this, 0);\r
+ private Map<Long, BasicDBList> storyRundowns;\r
+ private Map<Long, BasicDBList> storyStoryFolders;\r
+ private Map<Long, BasicDBList> storedStoryRundowns;\r
+ private Map<Long, BasicDBList> storedStoryStoryFolders;\r
+ private Map<Long, BasicDBList> storedStoryMosObjects;\r
+ private String RUNDOWN_COLLECTION = IOctopusAPI.RUNDOWN_COLLECTION;\r
+ private String FOLDER_COLLECTION = IOctopusAPI.FOLDER_COLLECTION;\r
+ private String STORY_COLLECTION = IOctopusAPI.STORY_COLLECTION;\r
+\r
+ private Map<Long, BasicDBList> newRundowns = new HashMap<>();\r
+ private Map<Long, BasicDBList> newStoryFolders = new HashMap<>();\r
+ private Map<Long, BasicDBList> newStories = new HashMap<>();\r
+\r
+ public OctopusDataMiner() {\r
+ db = NoSQLUtils.getNoSQLDB();\r
+\r
+ String apiAddress = System.getProperty("jobengine.octopus.api.address");\r
+ apiUser = System.getProperty("jobengine.octopus.api.user");\r
+ apiPwd = System.getProperty("jobengine.octopus.api.password");\r
+\r
+ // ResteasyClient client = new ResteasyClientBuilder().register(JacksonJsonProvider.class).build();\r
+ ResteasyClient client = new ResteasyClientBuilder().build();\r
+ webTarget = client.target(apiAddress).register(new BasicAuthentication(apiUser, apiPwd));\r
+ }\r
+\r
+ public void addProgressListener(IProgressEventListener listener) {\r
+ if (progressListenerList == null)\r
+ progressListenerList = new EventListenerList();\r
+ progressListenerList.add(IProgressEventListener.class, listener);\r
+ }\r
+\r
+ private Map<Long, BasicDBList> buildFolderReferences(BasicDBList storyFolders) {\r
+ Map<Long, BasicDBList> result = new HashMap<>();\r
+ List<BasicDBObject> storyFolderList = NoSQLUtils.asList(storyFolders);\r
+ for (BasicDBObject storyFolder : storyFolderList) {\r
+ long storyFolderId = storyFolder.getLong(IOctopusAPI.ID);\r
+ List<BasicDBObject> stories = NoSQLUtils.asList(storyFolder, IOctopusAPI.STORIES);\r
+ if (stories == null)\r
+ continue;\r
+ long position = 1;\r
+ for (BasicDBObject story : stories) {\r
+ long storyId = story.getLong(IOctopusAPI.ID);\r
+ BasicDBList references = result.get(storyId);\r
+ if (references == null) {\r
+ references = new BasicDBList();\r
+ result.put(storyId, references);\r
+ }\r
+ references.add(new BasicDBObject(IOctopusAPI.ID, storyFolderId).append(IOctopusAPI.POSITION, position++));\r
+ }\r
+ }\r
+ return result;\r
+ }\r
+\r
+ private Map<Long, BasicDBList> buildRundownReferences(BasicDBList rundowns) {\r
+ Map<Long, BasicDBList> result = new HashMap<>();\r
+ List<BasicDBObject> rundownsList = NoSQLUtils.asList(rundowns);\r
+ for (BasicDBObject rundown : rundownsList) {\r
+ if (!rundown.containsKey(IOctopusAPI.ID))\r
+ continue;\r
+ long rundownId = rundown.getLong(IOctopusAPI.ID);\r
+ List<BasicDBObject> slugs = NoSQLUtils.asList(rundown, IOctopusAPI.SLUGS);\r
+ if (slugs == null)\r
+ continue;\r
+ for (BasicDBObject slug : slugs) {\r
+ if (!slug.containsKey(IOctopusAPI.STORYID))\r
+ continue;\r
+ long storyId = slug.getLong(IOctopusAPI.STORYID);\r
+ BasicDBList references = result.get(storyId);\r
+ if (references == null) {\r
+ references = new BasicDBList();\r
+ result.put(storyId, references);\r
+ }\r
+ long position = slug.getLong(IOctopusAPI.POSITION);\r
+ if (slug.containsKey(IOctopusAPI.POSITION))\r
+ position = slug.getLong(IOctopusAPI.POSITION);\r
+ references.add(new BasicDBObject(IOctopusAPI.ID, rundownId).append(IOctopusAPI.POSITION, position));\r
+ }\r
+ }\r
+ return result;\r
+ }\r
+\r
+ private void buildStoriesReferences() {\r
+ DBCollection collection = db.getCollection(STORY_COLLECTION);\r
+ DBCursor cursor = collection.find(null, new BasicDBObject(IOctopusAPI.ID, 1).append(IOctopusAPI.REF_RUNDOWN, 1).append(IOctopusAPI.REF_STORYFOLDER, 1)\r
+ .append(IOctopusAPI.MOS_OBJECTS, 1));\r
+ //DBCursor find = collection.find(QueryBuilder.start(ID).greaterThan(0).get());\r
+ try {\r
+\r
+ while (cursor.hasNext()) {\r
+ BasicDBObject story = (BasicDBObject) cursor.next();\r
+ long storyId = story.getLong(IOctopusAPI.ID);\r
+ BasicDBList rundownRef = NoSQLUtils.asDBList(story, IOctopusAPI.REF_RUNDOWN);\r
+ if (rundownRef != null) {\r
+ if (storedStoryRundowns == null)\r
+ storedStoryRundowns = new HashMap<>();\r
+ storedStoryRundowns.put(storyId, rundownRef);\r
+ }\r
+ BasicDBList storyFolderRef = NoSQLUtils.asDBList(story, IOctopusAPI.REF_STORYFOLDER);\r
+ if (storyFolderRef != null) {\r
+ if (storedStoryStoryFolders == null)\r
+ storedStoryStoryFolders = new HashMap<>();\r
+ storedStoryStoryFolders.put(storyId, storyFolderRef);\r
+ }\r
+\r
+ BasicDBList storyMosObjects = NoSQLUtils.asDBList(story, IOctopusAPI.MOS_OBJECTS);\r
+ if (storyMosObjects != null) {\r
+ if (storedStoryMosObjects == null)\r
+ storedStoryMosObjects = new HashMap<>();\r
+ storedStoryMosObjects.put(storyId, storyMosObjects);\r
+ }\r
+ }\r
+ } catch (Exception e) {\r
+ logger.catching(e);\r
+ throw e;\r
+ } finally {\r
+\r
+ }\r
+ }\r
+\r
+ public void clear() {\r
+ db.getCollection(RUNDOWN_COLLECTION).remove();\r
+ db.getCollection(STORY_COLLECTION).remove();\r
+ db.getCollection(FOLDER_COLLECTION).remove();\r
+ db.getCollection(IOctopusAPI.TIME_COLLECTION_NAME).remove();\r
+ }\r
+\r
+ private String concatParentsToStoryFolder(BasicDBObject actual, String name) {\r
+\r
+ String fields = "name,id,parent";\r
+ Response response = query("StoryFolder/" + actual.getLong("id"), fields).get();\r
+ String json = response.readEntity(String.class);\r
+ BasicDBObject resultObject = (BasicDBObject) JSONUtil.jsonToDbObject(json);\r
+ BasicDBObject res = (BasicDBObject) resultObject.get("result");\r
+ BasicDBObject parent = (BasicDBObject) res.get("parent");\r
+ if (parent == null || parent.isEmpty())\r
+ return name;\r
+ String parentName = parent.getString(IOctopusAPI.NAME);\r
+ String actualName = actual.getString(IOctopusAPI.NAME);\r
+ String newName = String.format("%s/%s", parentName, actualName);\r
+ return concatParentsToStoryFolder(parent, newName);\r
+ }\r
+\r
+ private void deleteDiff(String oldCollectionName, String newCollectionName, String idFieldName) {\r
+ DBCollection oldCollection = db.getCollection(oldCollectionName);\r
+ DBCollection newCollection = db.getCollection(newCollectionName);\r
+ DBCursor oldCollectionCursor = oldCollection.find(new BasicDBObject(), new BasicDBObject(idFieldName, 1));\r
+ if (!oldCollectionCursor.hasNext()) {\r
+ logger.error("{} collection is empty", newCollectionName);\r
+ return;\r
+ }\r
+ List<BasicDBObject> oldItems = ListUtils.cast(oldCollectionCursor.toArray());\r
+\r
+ DBCursor newCollectionCursor = newCollection.find();\r
+ ConcurrentHashMap<Long, BasicDBObject> newItems = null;\r
+ if (newCollectionCursor.hasNext()) {\r
+ List<BasicDBObject> newList = ListUtils.cast(newCollectionCursor.toArray());\r
+ newItems = ListUtils.map(newList, item -> item.getLong(IOctopusAPI.ID));\r
+ }\r
+ if (newItems == null)\r
+ newItems = new ConcurrentHashMap<>();\r
+\r
+ for (BasicDBObject oldItem : oldItems) {\r
+ if (oldItem == null) {\r
+ logger.error("Item is null");\r
+ continue;\r
+ }\r
+ if (!oldItem.containsKey(idFieldName)) {\r
+ logger.error("{} is null", idFieldName);\r
+ continue;\r
+ }\r
+ long id = oldItem.getLong(idFieldName);\r
+ BasicDBObject newItem = newItems.get(id);\r
+ if (newItem == null) {\r
+ //remove\r
+ logger.info("Deleting {}", oldItem.toPrettyString(null));\r
+ oldCollection.remove(new BasicDBObject(idFieldName, id));\r
+ }\r
+ }\r
+ }\r
+\r
+ private void deleteOrphanStories() {\r
+ try {\r
+ DBCollection collection = db.getCollection(STORY_COLLECTION);\r
+ BasicDBObject query = (BasicDBObject) QueryBuilder.start().put("id").notIn(storyIDs.toArray()).get();\r
+ WriteResult res = collection.remove(query);\r
+ logger.trace(String.format("Deleted orphan stories: %d", res.getN()));\r
+ } catch (Exception e) {\r
+ logger.error(e);\r
+ }\r
+ }\r
+\r
+ private void ensureIndexes() {\r
+ DBCollection collection = db.getCollection(FOLDER_COLLECTION);\r
+ if (collection.count() == 0)\r
+ collection.ensureIndex(IOctopusAPI.ID);\r
+ collection = db.getCollection(RUNDOWN_COLLECTION);\r
+ if (collection.count() == 0) {\r
+ collection.ensureIndex(IOctopusAPI.ID);\r
+ collection.ensureIndex(IOctopusAPI.SCHEDULED_START);\r
+ }\r
+ collection = db.getCollection(STORY_COLLECTION);\r
+ if (collection.count() == 0)\r
+ collection.ensureIndex(IOctopusAPI.ID);\r
+ }\r
+\r
+ public void execute() throws Exception {\r
+ logger.trace(STARTING);\r
+ //{"filter" :{ "archived" : true }}\r
+ Response response = query(RUNDOWN, "id,name,modified,scheduledStart,channel,Channel.name,rundownType,RundownType.name")\r
+ .post(Entity.entity(new BasicDBObject("filter", new BasicDBObject("archived", true)).toPrettyString(null), MediaType.APPLICATION_JSON));\r
+ String json = response.readEntity(String.class);\r
+\r
+ RUNDOWN_COLLECTION = IOctopusAPI.RUNDOWN_COLLECTION + _TMP;\r
+ FOLDER_COLLECTION = IOctopusAPI.FOLDER_COLLECTION + _TMP;\r
+ STORY_COLLECTION = IOctopusAPI.STORY_COLLECTION + _TMP;\r
+\r
+ try {\r
+ db.getCollection(RUNDOWN_COLLECTION).drop();\r
+ db.getCollection(FOLDER_COLLECTION).drop();\r
+ db.getCollection(STORY_COLLECTION).drop();\r
+ } catch (Exception e) {\r
+ logger.catching(e);\r
+ throw e;\r
+ }\r
+\r
+ BasicDBList rundowns = null;\r
+ BasicDBList storyFolders = null;\r
+\r
+ try {\r
+ rundowns = queryBuildRefRundowns();\r
+ storyFolders = queryBuildRefFolders();\r
+ } catch (Exception e) {\r
+ logger.catching(e);\r
+ throw e;\r
+ }\r
+\r
+ processRundowns(rundowns);\r
+ processStoryFolders(storyFolders);\r
+\r
+ //a sorrend fontos !\r
+ updateDiff(IOctopusAPI.STORY_COLLECTION, STORY_COLLECTION, IOctopusAPI.ID);\r
+ updateDiff(IOctopusAPI.RUNDOWN_COLLECTION, RUNDOWN_COLLECTION, IOctopusAPI.ID);\r
+ updateDiff(IOctopusAPI.FOLDER_COLLECTION, FOLDER_COLLECTION, IOctopusAPI.ID);\r
+ deleteDiff(IOctopusAPI.RUNDOWN_COLLECTION, RUNDOWN_COLLECTION, IOctopusAPI.ID);\r
+ //deleteDiff(IOctopusAPI.FOLDER_COLLECTION, FOLDER_COLLECTION, IOctopusAPI.ID);\r
+ deleteDiff(IOctopusAPI.STORY_COLLECTION, STORY_COLLECTION, IOctopusAPI.ID);\r
+\r
+ //setLastUpdateTime(new Date());\r
+ logger.info("Activate");\r
+\r
+ // db.getCollection(RUNDOWN_COLLECTION).rename(IOctopusAPI.RUNDOWN_COLLECTION, true);\r
+ // db.getCollection(FOLDER_COLLECTION).rename(IOctopusAPI.FOLDER_COLLECTION, true);\r
+ // db.getCollection(STORY_COLLECTION).rename(IOctopusAPI.STORY_COLLECTION, true);\r
+ logger.trace(FINISHED);\r
+\r
+ }\r
+\r
+ public void executetest() {\r
+ // ResteasyWebTarget target = webTarget.path(RUNDOWN);\r
+ // Builder result = target.request().header(OCTOPUS_DEVICE_ID, apiUser).header(OCTOPUS_DEVICE_NAME, apiPwd);\r
+ // Response r = result.get();\r
+ // String x = r.readEntity(String.class);\r
+ // logger.info(x);\r
+ // return;\r
+ }\r
+\r
+ private String extractContent(BasicDBObject content) {\r
+ String scriptContent = "";\r
+ if (!content.containsKey(IOctopusAPI.TYPE))\r
+ return scriptContent;\r
+ String type = content.getString(IOctopusAPI.TYPE);\r
+ switch (type) {\r
+ case IOctopusAPI.TEXT: {\r
+ if (content.containsKey(IOctopusAPI.TEXT)) {\r
+ String text = content.getString(IOctopusAPI.TEXT);\r
+ if (text != null)\r
+ scriptContent += String.format("%s%s", text.replaceAll(SIMPLE_LINEFEED, LINEFEED), LINEFEED);\r
+ }\r
+ break;\r
+ }\r
+ case IOctopusAPI.MOS: {\r
+ BasicDBObject mosObject = NoSQLUtils.asDBObject(content, IOctopusAPI.OBJECT);\r
+ if (mosObject != null && !mosObject.isEmpty()) {\r
+ if (mosObject.containsKey(IOctopusAPI.OBJ_ID)) {\r
+ String objID = mosObject.getString(IOctopusAPI.OBJ_ID);\r
+ scriptContent += String.format("%s %s%s", MOSOBJECT, objID, LINEFEED);\r
+ }\r
+ }\r
+ break;\r
+ }\r
+ default: {\r
+ if (content.containsKey(IOctopusAPI.CONTENT)) {\r
+ List<BasicDBObject> innerContents = NoSQLUtils.asList(content, IOctopusAPI.CONTENT);\r
+ if (innerContents != null) {\r
+ for (BasicDBObject actualInnerContent : innerContents) {\r
+ if (actualInnerContent != null && actualInnerContent.isEmpty())\r
+ scriptContent += extractContent(actualInnerContent);\r
+ }\r
+ }\r
+ }\r
+ break;\r
+ }\r
+ }\r
+ return scriptContent;\r
+ }\r
+\r
+ private String extractCustomColumnValue(String columnName, BasicDBObject story) {\r
+ List<BasicDBObject> customColumns = NoSQLUtils.asList(story, IOctopusAPI.CUSTOM_COLUMNS);\r
+ if (customColumns == null)\r
+ return null;\r
+ String result = null;\r
+ for (BasicDBObject customColumn : customColumns) {\r
+ if (!customColumn.containsKey(IOctopusAPI.LABEL))\r
+ continue;\r
+ String currentName = customColumn.getString(IOctopusAPI.LABEL);\r
+ if (currentName == null)\r
+ continue;\r
+ if (!currentName.toLowerCase().equals(columnName.toLowerCase()))\r
+ continue;\r
+ if (!customColumn.containsKey(IOctopusAPI.VALUE))\r
+ continue;\r
+ result = customColumn.getString(IOctopusAPI.VALUE);\r
+ break;\r
+ }\r
+ return result;\r
+ }\r
+\r
+ private BasicDBList extractRelevantMOSObjects(BasicDBObject story) {\r
+ List<BasicDBObject> mosObjects = NoSQLUtils.asList(story, IOctopusAPI.MOS_OBJECTS);\r
+ if (mosObjects == null)\r
+ return null;\r
+ BasicDBList result = null;\r
+ for (BasicDBObject mosObject : mosObjects) {\r
+ if (!mosObject.containsKey(IOctopusAPI.MOS_ID))\r
+ continue;\r
+ String mosId = mosObject.getString(IOctopusAPI.MOS_ID);\r
+ if (!IOctopusAPI.NEXIO_MOS.equals(mosId))\r
+ continue;\r
+ if (result == null)\r
+ result = new BasicDBList();\r
+ result.add(mosObject);\r
+ }\r
+ return result;\r
+ }\r
+\r
+ private String extractScriptContent(BasicDBObject story) {\r
+ BasicDBObject script = NoSQLUtils.asDBObject(story, IOctopusAPI.SCRIPT);\r
+ if (script == null || script.isEmpty())\r
+ return null;\r
+\r
+ List<BasicDBObject> body = NoSQLUtils.asList(script, IOctopusAPI.BODY);\r
+ if (body == null || body.size() == 0)\r
+ return null;\r
+\r
+ StringBuilder sb = new StringBuilder();\r
+\r
+ for (BasicDBObject bodyItem : body) {\r
+ if (bodyItem.containsKey(IOctopusAPI.LABEL)) {\r
+ sb.append(bodyItem.getString(IOctopusAPI.LABEL));\r
+ sb.append(LINEFEED);\r
+ }\r
+ List<BasicDBObject> contents = NoSQLUtils.asList(bodyItem, IOctopusAPI.CONTENT);\r
+ if (contents == null)\r
+ continue;\r
+ for (BasicDBObject content : contents) {\r
+ sb.append(extractContent(content));\r
+ }\r
+ }\r
+ return sb.length() == 0 ? null : sb.toString();\r
+ }\r
+\r
+ private void fireProgressEvent(ProgressEvent evt) {\r
+ logger.debug("Progress changed to " + evt.getProgress() + "%");\r
+ if (progressListenerList == null)\r
+ return;\r
+ Object[] listeners = progressListenerList.getListenerList();\r
+ for (int i = 0; i < listeners.length; i += 2) {\r
+ if (listeners[i] == IProgressEventListener.class)\r
+ ((IProgressEventListener) listeners[i + 1]).progressChanged(evt);\r
+ }\r
+ }\r
+\r
+ private Date getLastUpdateTime() {\r
+ Date result = null;\r
+ DBCollection collection = db.getCollection(IOctopusAPI.TIME_COLLECTION_NAME);\r
+ DBObject timeObject = collection.findOne();\r
+ if (timeObject != null)\r
+ result = (Date) timeObject.get(IOctopusAPI.LASTUPDATE_TIME);\r
+ return result;\r
+ }\r
+\r
+ // private boolean isModified(Date date, BasicDBObject object, String name) {\r
+ // Date actualModifiedString = toDate(object, name);\r
+ // if (actualModifiedString == null)\r
+ // logger.trace(ACTUAL_MODIFIED_STRING_IS_NULL);\r
+ // int result = date.compareTo(actualModifiedString);\r
+ // return result <= 0;\r
+ // }\r
+\r
+ private boolean isModified(Date date, BasicDBObject object) {\r
+ if (date == null)\r
+ return true;\r
+ Date modified = (Date) object.get(IOctopusAPI.MODIFIED);\r
+ int result = date.compareTo(modified);\r
+ return result <= 0;\r
+ }\r
+\r
+ private void processRundowns(BasicDBList rundowns) throws Exception {\r
+ if (rundowns == null || rundowns.size() == 0) {\r
+ progressEvent.setProgress(50);\r
+ fireProgressEvent(progressEvent);\r
+ } else {\r
+ logger.info("Process rundowns");\r
+ try {\r
+ storeRundowns(rundowns, null);\r
+ } catch (Exception e) {\r
+ logger.catching(e);\r
+ throw e;\r
+ }\r
+ }\r
+ }\r
+\r
+ private void processStoryFolders(BasicDBList storyFolders) throws Exception {\r
+ if (storyFolders == null || storyFolders.size() == 0) {\r
+ progressEvent.setProgress(100);\r
+ fireProgressEvent(progressEvent);\r
+ } else {\r
+ logger.info("Process story folders");\r
+ try {\r
+ storeStoryFolders(storyFolders, null);\r
+ } catch (Exception e) {\r
+ logger.catching(e);\r
+ throw e;\r
+ }\r
+ }\r
+ }\r
+\r
+ private Builder query(String path, String fields) {\r
+ //logger.info("Class loader {}", getClass().getClassLoader());\r
+ // try {\r
+ // //TODO kell e?\r
+ // ResteasyDeployment deployment = new ResteasyDeployment();\r
+ // deployment.start();\r
+ // } catch (Exception e) {\r
+ // logger.catching(e);\r
+ // }\r
+ ResteasyWebTarget target = webTarget.path(path).queryParam(FIELDS, fields);\r
+ Builder result = target.request().header(OCTOPUS_DEVICE_ID, apiUser).header(OCTOPUS_DEVICE_NAME, apiPwd);\r
+ return result;\r
+ }\r
+\r
+ private BasicDBList queryBuildRefFolders() {\r
+ BasicDBList storyFolders;\r
+ logger.info("Fetch story folders");\r
+ storyFolders = queryStoryFolders();\r
+ logger.info("Fetch folder story references");\r
+ storyStoryFolders = buildFolderReferences(storyFolders);\r
+ return storyFolders;\r
+ }\r
+\r
+ private BasicDBList queryBuildRefRundowns() {\r
+ BasicDBList rundowns;\r
+ logger.info("Fetch rundowns");\r
+ rundowns = queryRundowns();\r
+ logger.info("Fetch rundown story references");\r
+ storyRundowns = buildRundownReferences(rundowns);\r
+ return rundowns;\r
+ }\r
+\r
+ private BasicDBObject queryRundown(BasicDBObject rundown) {\r
+ logger.trace(ENTER);\r
+ BasicDBObject result = null;\r
+ long id = NoSQLUtils.asLong(rundown, IOctopusAPI.ID);\r
+ Response response = query(String.format("%s/", RUNDOWN) + id, FIELDS_RUNDOWN_STORIES).get();\r
+ String json = response.readEntity(String.class);\r
+ BasicDBObject resultObject = (BasicDBObject) JSONUtil.jsonToDbObject(json);\r
+ if (resultObject == null)\r
+ logger.error("Rundown {} {} is not available", id, rundown.getString(IOctopusAPI.NAME));\r
+ else\r
+ result = NoSQLUtils.asDBObject(resultObject, RESULT);\r
+ logger.trace(EXIT);\r
+ return result;\r
+ }\r
+\r
+ private BasicDBList queryRundowns() {\r
+ logger.trace(ENTER);\r
+ BasicDBList result = null;\r
+ Response response = query(RUNDOWN, FIELDS_RUNDOWN_STORYIDS).get();\r
+ String json = response.readEntity(String.class);\r
+ BasicDBObject resultObject = (BasicDBObject) JSONUtil.jsonToDbObject(json);\r
+ if (resultObject != null)\r
+ result = NoSQLUtils.asDBList(resultObject, RESULT);\r
+ logger.trace(EXIT);\r
+ return result;\r
+ }\r
+\r
+ private BasicDBObject queryStoryFolder(BasicDBObject storyFolder) {\r
+ logger.trace(ENTER);\r
+ BasicDBObject result = null;\r
+ long id = NoSQLUtils.asLong(storyFolder, IOctopusAPI.ID);\r
+ Response response = query(String.format("%s/", STORY_FOLDER) + id, FIELDS_STORYFOLDER_STORIES).get();\r
+ String json = response.readEntity(String.class);\r
+ BasicDBObject resultObject = (BasicDBObject) JSONUtil.jsonToDbObject(json);\r
+ if (resultObject == null)\r
+ logger.error("StoryFolder {} {} is not available", id, storyFolder.getString(IOctopusAPI.NAME));\r
+ else\r
+ result = NoSQLUtils.asDBObject(resultObject, RESULT);\r
+ logger.trace(EXIT);\r
+ return result;\r
+ }\r
+\r
+ private BasicDBList queryStoryFolders() {\r
+ logger.trace(ENTER);\r
+ BasicDBList result = null;\r
+ Response response = query(STORY_FOLDER, FIELDS_STORY_FOLDER_LIST).get();\r
+ String json = response.readEntity(String.class);\r
+ BasicDBObject resultObject = (BasicDBObject) JSONUtil.jsonToDbObject(json);\r
+ if (resultObject != null)\r
+ result = NoSQLUtils.asDBList(resultObject, RESULT);\r
+\r
+ /* teszt */\r
+ List<BasicDBObject> list = NoSQLUtils.asList(result);\r
+ for (BasicDBObject actual : list) {\r
+ String fullName = concatParentsToStoryFolder(actual, actual.getString(IOctopusAPI.NAME));\r
+ //logger.info("Checking StoryFolder {}", fullName);\r
+ actual.remove(IOctopusAPI.NAME);\r
+ actual.append(IOctopusAPI.NAME, fullName);\r
+ }\r
+\r
+ logger.trace(EXIT);\r
+ return result;\r
+ }\r
+\r
+ public void removeProgressListener(IProgressEventListener listener) {\r
+ progressListenerList.remove(IProgressEventListener.class, listener);\r
+ }\r
+\r
+ @Override\r
+ public void run() {\r
+ logger.trace(STARTING);\r
+ try {\r
+ ensureIndexes();\r
+ Date lastUpdateTime = getLastUpdateTime();\r
+\r
+ buildStoriesReferences();\r
+\r
+ BasicDBList rundowns = queryRundowns();\r
+ storyRundowns = buildRundownReferences(rundowns);\r
+ BasicDBList storyFolders = queryStoryFolders();\r
+ storyStoryFolders = buildFolderReferences(storyFolders);\r
+\r
+ if (rundowns == null || rundowns.size() == 0) {\r
+ progressEvent.setProgress(50);\r
+ fireProgressEvent(progressEvent);\r
+ } else {\r
+ storeRundowns(rundowns, lastUpdateTime);\r
+ }\r
+\r
+ if (storyFolders == null || storyFolders.size() == 0) {\r
+ progressEvent.setProgress(100);\r
+ fireProgressEvent(progressEvent);\r
+ } else {\r
+ storeStoryFolders(storyFolders, lastUpdateTime);\r
+ }\r
+ deleteOrphanStories();\r
+ setLastUpdateTime(new Date());\r
+ } catch (Exception e) {\r
+ logger.catching(e);\r
+ throw e;\r
+ }\r
+ logger.trace(FINISHED);\r
+ }\r
+\r
+ public void run(boolean forceFull) {\r
+ if (forceFull) {\r
+ clear();\r
+ }\r
+ run();\r
+ // if (forceFull) {\r
+ // RUNDOWN_COLLECTION_NAME = "rundowns";\r
+ // STORY_COLLECTION_NAME = "stories";\r
+ // STORY_FOLDER_COLLECTION_NAME = "storyfolders";\r
+ // TIME_COLLECTION_NAME = "octopusSyncTime";\r
+ // db.getCollection("rundowns_tmp").rename(RUNDOWN_COLLECTION_NAME, true);\r
+ // db.getCollection("stories_tmp").rename(STORY_COLLECTION_NAME, true);\r
+ // db.getCollection("storyfolders_tmp").rename(STORY_FOLDER_COLLECTION_NAME, true);\r
+ // db.getCollection("octopusSyncTime_tmp").rename(TIME_COLLECTION_NAME, true);\r
+ // //clear();\r
+ // }\r
+\r
+ //TODO reset collection names\r
+ }\r
+\r
+ public void setLastUpdateTime(Date lastUpdateTime) {\r
+ DBCollection collection = db.getCollection(OctopusAPI.TIME_COLLECTION_NAME);\r
+ DBObject timeObject = collection.findOne();\r
+ if (timeObject == null)\r
+ timeObject = new BasicDBObject();\r
+ timeObject.put(IOctopusAPI.LASTUPDATE_TIME, lastUpdateTime);\r
+\r
+ collection.save(timeObject);\r
+ }\r
+\r
+ private void storeRundown(BasicDBObject rundown, Date lastUpdateTime) {\r
+ logger.trace(ENTER);\r
+ String name = rundown.containsKey(IOctopusAPI.NAME) ? rundown.getString(IOctopusAPI.NAME) : null;\r
+ logger.debug("Storing rundown {} {}", name, rundown.get(IOctopusAPI.SCHEDULED_START));\r
+ BasicDBObject rundownWithStories = queryRundown(rundown);\r
+ if (rundownWithStories != null) {\r
+ BasicDBList stories = NoSQLUtils.asDBList(rundownWithStories, IOctopusAPI.SLUGS);\r
+ if (stories != null)\r
+ storeRundownStories(stories, lastUpdateTime);\r
+ rundown.put(IOctopusAPI.SCHEDULED_START, toDate(rundown, IOctopusAPI.SCHEDULED_START));\r
+ rundown.put(IOctopusAPI.MODIFIED, toDate(rundown, IOctopusAPI.MODIFIED));\r
+ DBCollection collection = db.getCollection(RUNDOWN_COLLECTION);\r
+ if (lastUpdateTime == null || (lastUpdateTime != null && isModified(lastUpdateTime, rundown))) {\r
+ //logger.debug(SAVING_RUNDOWN, rundownID, name);\r
+ collection.save(rundown);\r
+ }\r
+ }\r
+ logger.trace(EXIT);\r
+ }\r
+\r
+ private void storeRundowns(BasicDBList rundowns, Date lastUpdateTime) {\r
+ logger.trace(ENTER);\r
+ List<BasicDBObject> rundownsList = NoSQLUtils.asList(rundowns);\r
+ int idx = 1;\r
+ for (BasicDBObject rundown : rundownsList) {\r
+ //logger.info(CHECKING_RUNDOWN, rundown.getLong(IOctopusAPI.ID), rundownsList.size(), idx);\r
+ storeRundown(rundown, lastUpdateTime);\r
+ int progress = idx * 50 / rundownsList.size();\r
+ if (progress - progressEvent.getProgress() > 0) {\r
+ progressEvent.setProgress(progress);\r
+ fireProgressEvent(progressEvent);\r
+ }\r
+\r
+ idx++;\r
+ }\r
+ logger.trace(EXIT);\r
+ }\r
+\r
+ private void storeRundownStories(BasicDBList slugs, Date lastUpdateTime) {\r
+ logger.trace(ENTER);\r
+ List<BasicDBObject> slugsList = NoSQLUtils.asList(slugs);\r
+ for (BasicDBObject slug : slugsList) {\r
+ if (slug.containsKey(IOctopusAPI.STORY))\r
+ storeStory((BasicDBObject) slug.get(IOctopusAPI.STORY), lastUpdateTime);\r
+ }\r
+ logger.trace(EXIT);\r
+ }\r
+\r
+ private void storeStory(BasicDBObject story, Date lastUpdateTime) {\r
+ logger.trace(ENTER);\r
+ if (!story.containsKey(IOctopusAPI.ID)) {\r
+ logger.error("Missing id in story {}", story.toPrettyString(null));\r
+ return;\r
+ }\r
+ long storyID = story.getLong(IOctopusAPI.ID);\r
+ boolean isContains = storyIDs.contains(storyID);\r
+ if (isContains)\r
+ return;\r
+ storyIDs.add(storyID);\r
+ story.put(IOctopusAPI.MODIFIED, toDate(story, IOctopusAPI.MODIFIED));\r
+ BasicDBList rundownRef = storyRundowns.get(storyID);\r
+ BasicDBList storyFolderRef = storyStoryFolders.get(storyID);\r
+ BasicDBList modifiedMOS = extractRelevantMOSObjects(story);\r
+\r
+ if (lastUpdateTime != null) {\r
+ rundownRef = (rundownRef == null) ? new BasicDBList() : rundownRef;\r
+ storyFolderRef = (storyFolderRef == null) ? new BasicDBList() : storyFolderRef;\r
+ modifiedMOS = (modifiedMOS == null) ? new BasicDBList() : modifiedMOS;\r
+\r
+ boolean uptodate = true;\r
+ if (!isModified(lastUpdateTime, story)) {\r
+ BasicDBList storedRundownRef = storedStoryRundowns.get(storyID);\r
+ storedRundownRef = (storedRundownRef == null) ? new BasicDBList() : storedRundownRef;\r
+ uptodate = storedRundownRef.equals(rundownRef);\r
+\r
+ if (uptodate) {\r
+ BasicDBList storedStoryFolderRef = storedStoryStoryFolders.get(storyID);\r
+ storedStoryFolderRef = (storedStoryFolderRef == null) ? new BasicDBList() : storedStoryFolderRef;\r
+ uptodate = storedStoryFolderRef.equals(storyFolderRef);\r
+ }\r
+\r
+ if (uptodate) {\r
+ BasicDBList storedMOS = storedStoryMosObjects.get(storyID);\r
+ storedMOS = (storedMOS == null) ? new BasicDBList() : storedMOS;\r
+ uptodate = storedMOS.equals(modifiedMOS);\r
+ }\r
+\r
+ if (uptodate)\r
+ return;\r
+ }\r
+ }\r
+\r
+ DBCollection collection = db.getCollection(STORY_COLLECTION);\r
+ if (lastUpdateTime != null) {\r
+ BasicDBObject orig = (BasicDBObject) collection.findOne(new BasicDBObject(IOctopusAPI.ID, storyID), new BasicDBObject(IOctopusAPI.ID, 1));\r
+ if (orig != null)\r
+ story.put("_id", orig.getID());\r
+ }\r
+ if (rundownRef != null)\r
+ story.put(IOctopusAPI.REF_RUNDOWN, rundownRef);\r
+ if (storyFolderRef != null)\r
+ story.put(IOctopusAPI.REF_STORYFOLDER, storyFolderRef);\r
+ String scriptContent = extractScriptContent(story);\r
+ story.put(IOctopusAPI.SCRIPT_CONTENT, scriptContent);\r
+ if (modifiedMOS == null || modifiedMOS.isEmpty()) {\r
+ if (story.containsKey(IOctopusAPI.MOS_OBJECTS))\r
+ story.remove(IOctopusAPI.MOS_OBJECTS);\r
+ } else\r
+ story.put(IOctopusAPI.MOS_OBJECTS, modifiedMOS);\r
+\r
+ String parentStoryId = extractCustomColumnValue(IOctopusAPI.PARENT_STORY_ID, story);\r
+ if (parentStoryId == null) {\r
+ logger.warn("Missing {} in story", IOctopusAPI.PARENT_STORY_ID, story.toPrettyString(null));\r
+ story.append(IOctopusAPI.PARENT_STORY_ID, storyID);\r
+ } else\r
+ story.append(IOctopusAPI.PARENT_STORY_ID, parentStoryId);\r
+ logger.debug(SAVING_STORY_ID, storyID);\r
+ collection.save(story);\r
+ logger.trace(EXIT);\r
+ }\r
+\r
+ private void storeStoryFolder(BasicDBObject storyFolder, Date lastUpdateTime) {\r
+ logger.trace(ENTER);\r
+ BasicDBObject storyFoldersWithStories = queryStoryFolder(storyFolder);\r
+ if (storyFoldersWithStories != null) {\r
+ BasicDBList stories = NoSQLUtils.asDBList(storyFoldersWithStories, IOctopusAPI.STORIES);\r
+ if (stories != null)\r
+ storeStoryFolderStories(stories, lastUpdateTime);\r
+ storyFolder.put(IOctopusAPI.MODIFIED, toDate(storyFolder, IOctopusAPI.MODIFIED));\r
+ DBCollection collection = db.getCollection(FOLDER_COLLECTION);\r
+ if (lastUpdateTime == null || (lastUpdateTime != null && isModified(lastUpdateTime, storyFolder))) {\r
+ String name = storyFolder.getString(IOctopusAPI.NAME);\r
+ logger.debug("Storing story folder {}", name);\r
+ collection.save(storyFolder);\r
+ }\r
+ }\r
+ logger.trace(EXIT);\r
+ }\r
+\r
+ private void storeStoryFolders(BasicDBList storyFolders, Date lastUpdateTime) {\r
+ logger.trace(ENTER);\r
+ List<BasicDBObject> storyFolderList = NoSQLUtils.asList(storyFolders);\r
+ int idx = 1;\r
+ for (BasicDBObject storyFolder : storyFolderList) {\r
+ logger.debug(String.format(CHECKING_STORY_FOLDER, storyFolder.getLong(IOctopusAPI.ID), storyFolderList.size(), idx));\r
+ storeStoryFolder(storyFolder, lastUpdateTime);\r
+ int progress = 50 + (idx * 50 / storyFolderList.size());\r
+ if (progress - progressEvent.getProgress() > 0) {\r
+ progressEvent.setProgress(progress);\r
+ fireProgressEvent(progressEvent);\r
+ }\r
+ idx++;\r
+ }\r
+ logger.trace(EXIT);\r
+ }\r
+\r
+ private void storeStoryFolderStories(BasicDBList stories, Date lastUpdateTime) {\r
+ logger.trace(ENTER);\r
+ List<BasicDBObject> list = NoSQLUtils.asList(stories);\r
+ for (BasicDBObject story : list)\r
+ storeStory(story, lastUpdateTime);\r
+ logger.trace(EXIT);\r
+ }\r
+\r
+ private Date toDate(BasicDBObject obj, String name) {\r
+ Date result = null;\r
+ if (obj.containsKey(name)) {\r
+ String dt = obj.getString(name);\r
+ if (dt != null) {\r
+ // create jodatime from date\r
+ DateTime jdt = new DateTime(dt);\r
+ result = jdt.toDate();\r
+ }\r
+ }\r
+ return result;\r
+ }\r
+\r
+ private void updateDiff(String oldCollectionName, String newCollectionName, String idFieldName) {\r
+ DBCollection oldCollection = db.getCollection(oldCollectionName);\r
+ DBCollection newCollection = db.getCollection(newCollectionName);\r
+ DBCursor newCollectionCursor = newCollection.find();\r
+ if (!newCollectionCursor.hasNext()) {\r
+ logger.error("{} collection is empty", newCollectionName);\r
+ return;\r
+ }\r
+\r
+ List<BasicDBObject> newItems = ListUtils.cast(newCollectionCursor.toArray());\r
+\r
+ DBCursor oldCollectionCursor = oldCollection.find();\r
+ ConcurrentHashMap<Long, BasicDBObject> oldItems = null;\r
+ if (oldCollectionCursor.hasNext()) {\r
+ List<BasicDBObject> oldList = ListUtils.cast(oldCollectionCursor.toArray());\r
+ oldItems = ListUtils.map(oldList, item -> item.getLong(IOctopusAPI.ID));\r
+ }\r
+ if (oldItems == null)\r
+ oldItems = new ConcurrentHashMap<>();\r
+ for (BasicDBObject newItem : newItems) {\r
+ if (newItem == null) {\r
+ logger.error("Item is null");\r
+ continue;\r
+ }\r
+ if (!newItem.containsKey(idFieldName)) {\r
+ logger.error("{} is null", idFieldName);\r
+ continue;\r
+ }\r
+ long id = newItem.getLong(idFieldName);\r
+ BasicDBObject oldItem = oldItems.get(id);\r
+ boolean save = false;\r
+ if (oldItem == null) {\r
+ //save\r
+ save = true;\r
+ } else {\r
+ //compare\r
+ Object oldDbId = oldItem.getID();\r
+ newItem.remove(IOctopusAPI._ID);\r
+ oldItem.remove(IOctopusAPI._ID);\r
+\r
+ if (!newItem.equals(oldItem)) {\r
+ //save\r
+ newItem.put(IOctopusAPI._ID, oldDbId);\r
+ save = true;\r
+ }\r
+ }\r
+\r
+ if (save) {\r
+ logger.info("Saving to {} item {}", oldCollectionName, id);\r
+ oldCollection.save(newItem);\r
+ }\r
+ }\r
+ }\r
+\r
+}\r
https://git.useribm.hu / mediacube.git / commitdiff