import user.commons.remotestore.ProgressEvent;\r
\r
public class OctopusDataMiner implements Runnable {\r
+ private static final String _TMP = "_tmp";\r
private static final Logger logger = LogManager.getLogger();\r
private static final String LINEFEED = "\r\n";\r
private static final String SIMPLE_LINEFEED = "\n";\r
private Map<Long, BasicDBList> storedStoryRundowns;\r
private Map<Long, BasicDBList> storedStoryStoryFolders;\r
private Map<Long, BasicDBList> storedStoryMosObjects;\r
+ private String RUNDOWN_COLLECTION = IOctopusAPI.RUNDOWN_COLLECTION;\r
+ private String FOLDER_COLLECTION = IOctopusAPI.FOLDER_COLLECTION;\r
+ private String STORY_COLLECTION = IOctopusAPI.STORY_COLLECTION;\r
\r
public OctopusDataMiner() {\r
String jdbcUrl = System.getProperty("jobengine.nosql.db.url");\r
}\r
\r
private void buildStoriesReferences() {\r
- DBCollection collection = db.getCollection(IOctopusAPI.STORY_COLLECTION_NAME);\r
+ DBCollection collection = db.getCollection(STORY_COLLECTION);\r
DBCursor cursor = collection.find(null, new BasicDBObject(IOctopusAPI.ID, 1).append(IOctopusAPI.REF_RUNDOWN, 1).append(IOctopusAPI.REF_STORYFOLDER, 1)\r
.append(IOctopusAPI.MOS_OBJECTS, 1));\r
//DBCursor find = collection.find(QueryBuilder.start(ID).greaterThan(0).get());\r
}\r
}\r
\r
- private Map<Long, BasicDBList> buildStoryFolderReferences(BasicDBList storyFolders) {\r
+ private Map<Long, BasicDBList> buildFolderReferences(BasicDBList storyFolders) {\r
Map<Long, BasicDBList> result = new HashMap<>();\r
List<BasicDBObject> storyFolderList = NoSQLUtils.asList(storyFolders);\r
for (BasicDBObject storyFolder : storyFolderList) {\r
}\r
\r
public void clear() {\r
- db.getCollection(IOctopusAPI.RUNDOWN_COLLECTION_NAME).remove();\r
- db.getCollection(IOctopusAPI.STORY_COLLECTION_NAME).remove();\r
- db.getCollection(IOctopusAPI.STORY_FOLDER_COLLECTION_NAME).remove();\r
+ db.getCollection(RUNDOWN_COLLECTION).remove();\r
+ db.getCollection(STORY_COLLECTION).remove();\r
+ db.getCollection(FOLDER_COLLECTION).remove();\r
db.getCollection(IOctopusAPI.TIME_COLLECTION_NAME).remove();\r
}\r
\r
\r
private void deleteOrphanStories() {\r
try {\r
- DBCollection collection = db.getCollection(IOctopusAPI.STORY_COLLECTION_NAME);\r
+ DBCollection collection = db.getCollection(STORY_COLLECTION);\r
BasicDBObject query = (BasicDBObject) QueryBuilder.start().put("id").notIn(storyIDs.toArray()).get();\r
WriteResult res = collection.remove(query);\r
logger.trace(String.format("Deleted orphan stories: %d", res.getN()));\r
}\r
\r
private void ensureIndexes() {\r
- DBCollection collection = db.getCollection(IOctopusAPI.STORY_FOLDER_COLLECTION_NAME);\r
+ DBCollection collection = db.getCollection(FOLDER_COLLECTION);\r
if (collection.count() == 0)\r
collection.ensureIndex(IOctopusAPI.ID);\r
- collection = db.getCollection(IOctopusAPI.RUNDOWN_COLLECTION_NAME);\r
+ collection = db.getCollection(RUNDOWN_COLLECTION);\r
if (collection.count() == 0) {\r
collection.ensureIndex(IOctopusAPI.ID);\r
collection.ensureIndex(IOctopusAPI.SCHEDULED_START);\r
}\r
- collection = db.getCollection(IOctopusAPI.STORY_COLLECTION_NAME);\r
+ collection = db.getCollection(STORY_COLLECTION);\r
if (collection.count() == 0)\r
collection.ensureIndex(IOctopusAPI.ID);\r
}\r
\r
+ public void execute() {\r
+ logger.trace(STARTING);\r
+ try {\r
+ RUNDOWN_COLLECTION = IOctopusAPI.RUNDOWN_COLLECTION + _TMP;\r
+ FOLDER_COLLECTION = IOctopusAPI.FOLDER_COLLECTION + _TMP;\r
+ STORY_COLLECTION = IOctopusAPI.STORY_COLLECTION + _TMP;\r
+\r
+ logger.info("Fetch rundowns");\r
+ BasicDBList rundowns = queryRundowns();\r
+ logger.info("Fetch rundown story references");\r
+ storyRundowns = buildRundownReferences(rundowns);\r
+ logger.info("Fetch story folders");\r
+ BasicDBList storyFolders = queryStoryFolders();\r
+ logger.info("Fetch folder story references");\r
+ storyStoryFolders = buildFolderReferences(storyFolders);\r
+\r
+ if (rundowns == null || rundowns.size() == 0) {\r
+ progressEvent.setProgress(50);\r
+ fireProgressEvent(progressEvent);\r
+ } else {\r
+ logger.info("Process rundowns");\r
+ storeRundowns(rundowns, null);\r
+ }\r
+\r
+ if (storyFolders == null || storyFolders.size() == 0) {\r
+ progressEvent.setProgress(100);\r
+ fireProgressEvent(progressEvent);\r
+ } else {\r
+ logger.info("Process story folders");\r
+ storeStoryFolders(storyFolders, null);\r
+ }\r
+ setLastUpdateTime(new Date());\r
+ logger.info("Activate");\r
+ db.getCollection(RUNDOWN_COLLECTION).rename(IOctopusAPI.RUNDOWN_COLLECTION, true);\r
+ db.getCollection(FOLDER_COLLECTION).rename(IOctopusAPI.FOLDER_COLLECTION, true);\r
+ db.getCollection(STORY_COLLECTION).rename(IOctopusAPI.STORY_COLLECTION, true);\r
+\r
+ } catch (Exception e) {\r
+ logger.catching(e);\r
+ throw e;\r
+ }\r
+ logger.trace(FINISHED);\r
+\r
+ }\r
+\r
+ private BasicDBList extractRelevantMOSObjects(BasicDBObject story) {\r
+ List<BasicDBObject> mosObjects = NoSQLUtils.asList(story, IOctopusAPI.MOS_OBJECTS);\r
+ if (mosObjects == null)\r
+ return null;\r
+ BasicDBList result = null;\r
+ for (BasicDBObject mosObject : mosObjects) {\r
+ if (!mosObject.containsKey(IOctopusAPI.MOS_ID))\r
+ continue;\r
+ String mosId = mosObject.getString(IOctopusAPI.MOS_ID);\r
+ if (!IOctopusAPI.NEXIO_MOS.equals(mosId))\r
+ continue;\r
+ if (result == null)\r
+ result = new BasicDBList();\r
+ result.add(mosObject);\r
+ }\r
+ return result;\r
+ }\r
+\r
+ // private boolean isModified(Date date, BasicDBObject object, String name) {\r
+ // Date actualModifiedString = toDate(object, name);\r
+ // if (actualModifiedString == null)\r
+ // logger.trace(ACTUAL_MODIFIED_STRING_IS_NULL);\r
+ // int result = date.compareTo(actualModifiedString);\r
+ // return result <= 0;\r
+ // }\r
+\r
+ private String extractScriptContent(BasicDBObject story) {\r
+ BasicDBObject script = NoSQLUtils.asDBObject(story, IOctopusAPI.SCRIPT);\r
+ if (script == null || script.isEmpty())\r
+ return null;\r
+\r
+ List<BasicDBObject> body = NoSQLUtils.asList(script, IOctopusAPI.BODY);\r
+ if (body == null || body.size() == 0)\r
+ return null;\r
+\r
+ StringBuilder sb = new StringBuilder();\r
+\r
+ for (BasicDBObject bodyItem : body) {\r
+ if (bodyItem.containsKey(IOctopusAPI.LABEL)) {\r
+ sb.append(bodyItem.getString(IOctopusAPI.LABEL));\r
+ sb.append(LINEFEED);\r
+ }\r
+ List<BasicDBObject> contents = NoSQLUtils.asList(bodyItem, IOctopusAPI.CONTENT);\r
+ if (contents == null)\r
+ continue;\r
+ for (BasicDBObject content : contents) {\r
+ sb.append(extractContent(content));\r
+ }\r
+ }\r
+ return sb.length() == 0 ? null : sb.toString();\r
+ }\r
+\r
private void fireProgressEvent(ProgressEvent evt) {\r
logger.debug("Progress changed to " + evt.getProgress() + "%");\r
if (progressListenerList == null)\r
}\r
}\r
\r
- private String GetContent(BasicDBObject content) {\r
+ private String extractContent(BasicDBObject content) {\r
String scriptContent = "";\r
if (!content.containsKey(IOctopusAPI.TYPE))\r
return scriptContent;\r
if (innerContents != null) {\r
for (BasicDBObject actualInnerContent : innerContents) {\r
if (actualInnerContent != null && actualInnerContent.isEmpty())\r
- scriptContent += GetContent(actualInnerContent);\r
+ scriptContent += extractContent(actualInnerContent);\r
}\r
}\r
}\r
return scriptContent;\r
}\r
\r
- // private boolean isModified(Date date, BasicDBObject object, String name) {\r
- // Date actualModifiedString = toDate(object, name);\r
- // if (actualModifiedString == null)\r
- // logger.trace(ACTUAL_MODIFIED_STRING_IS_NULL);\r
- // int result = date.compareTo(actualModifiedString);\r
- // return result <= 0;\r
- // }\r
-\r
- private String GetCustomColumnValue(String columnName, BasicDBObject story) {\r
+ private String extractCustomColumnValue(String columnName, BasicDBObject story) {\r
List<BasicDBObject> customColumns = NoSQLUtils.asList(story, IOctopusAPI.CUSTOM_COLUMNS);\r
if (customColumns == null)\r
return null;\r
return result;\r
}\r
\r
- private BasicDBList GetRelevantMOSObjects(BasicDBObject story) {\r
- List<BasicDBObject> mosObjects = NoSQLUtils.asList(story, IOctopusAPI.MOS_OBJECTS);\r
- if (mosObjects == null)\r
- return null;\r
- BasicDBList result = null;\r
- for (BasicDBObject mosObject : mosObjects) {\r
- if (!mosObject.containsKey(IOctopusAPI.MOS_ID))\r
- continue;\r
- String mosId = mosObject.getString(IOctopusAPI.MOS_ID);\r
- if (!IOctopusAPI.NEXIO_MOS.equals(mosId))\r
- continue;\r
- if (result == null)\r
- result = new BasicDBList();\r
- result.add(mosObject);\r
- }\r
- return result;\r
- }\r
-\r
- private String GetScriptContent(BasicDBObject story) {\r
- BasicDBObject script = NoSQLUtils.asDBObject(story, IOctopusAPI.SCRIPT);\r
- if (script == null || script.isEmpty())\r
- return null;\r
-\r
- List<BasicDBObject> body = NoSQLUtils.asList(script, IOctopusAPI.BODY);\r
- if (body == null || body.size() == 0)\r
- return null;\r
-\r
- StringBuilder sb = new StringBuilder();\r
-\r
- for (BasicDBObject bodyItem : body) {\r
- if (bodyItem.containsKey(IOctopusAPI.LABEL)) {\r
- sb.append(bodyItem.getString(IOctopusAPI.LABEL));\r
- sb.append(LINEFEED);\r
- }\r
- List<BasicDBObject> contents = NoSQLUtils.asList(bodyItem, IOctopusAPI.CONTENT);\r
- if (contents == null)\r
- continue;\r
- for (BasicDBObject content : contents) {\r
- sb.append(GetContent(content));\r
- }\r
- }\r
- return sb.length() == 0 ? null : sb.toString();\r
- }\r
-\r
private boolean isModified(Date date, BasicDBObject object) {\r
if (date == null)\r
return true;\r
BasicDBList rundowns = queryRundowns();\r
storyRundowns = buildRundownReferences(rundowns);\r
BasicDBList storyFolders = queryStoryFolders();\r
- storyStoryFolders = buildStoryFolderReferences(storyFolders);\r
+ storyStoryFolders = buildFolderReferences(storyFolders);\r
\r
if (rundowns == null || rundowns.size() == 0) {\r
progressEvent.setProgress(50);\r
\r
public void run(boolean forceFull) {\r
if (forceFull) {\r
- // RUNDOWN_COLLECTION_NAME = "rundowns_tmp";\r
- // STORY_COLLECTION_NAME = "stories_tmp";\r
- // STORY_FOLDER_COLLECTION_NAME = "storyfolders_tmp";\r
- // TIME_COLLECTION_NAME = "octopusSyncTime_tmp";\r
clear();\r
}\r
run();\r
storeRundownStories(stories, lastUpdateTime);\r
rundown.put(IOctopusAPI.SCHEDULED_START, toDate(rundown, IOctopusAPI.SCHEDULED_START));\r
rundown.put(IOctopusAPI.MODIFIED, toDate(rundown, IOctopusAPI.MODIFIED));\r
- DBCollection collection = db.getCollection(IOctopusAPI.RUNDOWN_COLLECTION_NAME);\r
+ DBCollection collection = db.getCollection(RUNDOWN_COLLECTION);\r
if (lastUpdateTime == null || (lastUpdateTime != null && isModified(lastUpdateTime, rundown))) {\r
logger.debug(SAVING_RUNDOWN, rundownID, name);\r
collection.save(rundown);\r
storyIDs.add(storyID);\r
story.put(IOctopusAPI.MODIFIED, toDate(story, IOctopusAPI.MODIFIED));\r
BasicDBList rundownRef = storyRundowns.get(storyID);\r
- rundownRef = (rundownRef == null) ? new BasicDBList() : rundownRef;\r
BasicDBList storyFolderRef = storyStoryFolders.get(storyID);\r
- storyFolderRef = (storyFolderRef == null) ? new BasicDBList() : storyFolderRef;\r
- BasicDBList modifiedMOS = GetRelevantMOSObjects(story);\r
- modifiedMOS = (modifiedMOS == null) ? new BasicDBList() : modifiedMOS;\r
-\r
- boolean uptodate = true;\r
- if (!isModified(lastUpdateTime, story)) {\r
- BasicDBList storedRundownRef = storedStoryRundowns.get(storyID);\r
- storedRundownRef = (storedRundownRef == null) ? new BasicDBList() : storedRundownRef;\r
- uptodate = storedRundownRef.equals(rundownRef);\r
-\r
- if (uptodate) {\r
- BasicDBList storedStoryFolderRef = storedStoryStoryFolders.get(storyID);\r
- storedStoryFolderRef = (storedStoryFolderRef == null) ? new BasicDBList() : storedStoryFolderRef;\r
- uptodate = storedStoryFolderRef.equals(storyFolderRef);\r
- }\r
+ BasicDBList modifiedMOS = extractRelevantMOSObjects(story);\r
\r
- if (uptodate) {\r
- BasicDBList storedMOS = storedStoryMosObjects.get(storyID);\r
- storedMOS = (storedMOS == null) ? new BasicDBList() : storedMOS;\r
- uptodate = storedMOS.equals(modifiedMOS);\r
- }\r
+ if (lastUpdateTime != null) {\r
+ rundownRef = (rundownRef == null) ? new BasicDBList() : rundownRef;\r
+ storyFolderRef = (storyFolderRef == null) ? new BasicDBList() : storyFolderRef;\r
+ modifiedMOS = (modifiedMOS == null) ? new BasicDBList() : modifiedMOS;\r
+\r
+ boolean uptodate = true;\r
+ if (!isModified(lastUpdateTime, story)) {\r
+ BasicDBList storedRundownRef = storedStoryRundowns.get(storyID);\r
+ storedRundownRef = (storedRundownRef == null) ? new BasicDBList() : storedRundownRef;\r
+ uptodate = storedRundownRef.equals(rundownRef);\r
+\r
+ if (uptodate) {\r
+ BasicDBList storedStoryFolderRef = storedStoryStoryFolders.get(storyID);\r
+ storedStoryFolderRef = (storedStoryFolderRef == null) ? new BasicDBList() : storedStoryFolderRef;\r
+ uptodate = storedStoryFolderRef.equals(storyFolderRef);\r
+ }\r
+\r
+ if (uptodate) {\r
+ BasicDBList storedMOS = storedStoryMosObjects.get(storyID);\r
+ storedMOS = (storedMOS == null) ? new BasicDBList() : storedMOS;\r
+ uptodate = storedMOS.equals(modifiedMOS);\r
+ }\r
\r
- if (uptodate)\r
- return;\r
+ if (uptodate)\r
+ return;\r
+ }\r
}\r
\r
- DBCollection collection = db.getCollection(IOctopusAPI.STORY_COLLECTION_NAME);\r
+ DBCollection collection = db.getCollection(STORY_COLLECTION);\r
if (lastUpdateTime != null) {\r
BasicDBObject orig = (BasicDBObject) collection.findOne(new BasicDBObject(IOctopusAPI.ID, storyID), new BasicDBObject(IOctopusAPI.ID, 1));\r
if (orig != null)\r
story.put(IOctopusAPI.REF_RUNDOWN, rundownRef);\r
if (storyFolderRef != null)\r
story.put(IOctopusAPI.REF_STORYFOLDER, storyFolderRef);\r
- String scriptContent = GetScriptContent(story);\r
+ String scriptContent = extractScriptContent(story);\r
story.put(SCRIPT_CONTENT, scriptContent);\r
if (modifiedMOS == null || modifiedMOS.isEmpty()) {\r
if (story.containsKey(IOctopusAPI.MOS_OBJECTS))\r
} else\r
story.put(IOctopusAPI.MOS_OBJECTS, modifiedMOS);\r
\r
- String parentStoryId = GetCustomColumnValue(IOctopusAPI.PARENT_STORY_ID, story);\r
+ String parentStoryId = extractCustomColumnValue(IOctopusAPI.PARENT_STORY_ID, story);\r
if (parentStoryId == null)\r
story.append(IOctopusAPI.PARENT_STORY_ID, storyID);\r
else\r
if (stories != null)\r
storeStoryFolderStories(stories, lastUpdateTime);\r
storyFolder.put(IOctopusAPI.MODIFIED, toDate(storyFolder, IOctopusAPI.MODIFIED));\r
- DBCollection collection = db.getCollection(IOctopusAPI.STORY_FOLDER_COLLECTION_NAME);\r
+ DBCollection collection = db.getCollection(FOLDER_COLLECTION);\r
if (lastUpdateTime == null || (lastUpdateTime != null && isModified(lastUpdateTime, storyFolder))) {\r
String name = storyFolder.getString(IOctopusAPI.NAME);\r
logger.debug("Storing story folder {}", name);\r