<Filters>\r
<MarkerFilter marker="MEDIACUBE" onMatch="ACCEPT" onMismatch="DENY" />\r
</Filters>\r
- <PatternLayout pattern="%d{HH:mm:ss} %-5level - %msg %n%n A küldő a %F forráskód %L. sorából a %logger{1}.%M függvény volt. %n" />\r
+<!-- <PatternLayout pattern="%d{HH:mm:ss} %-5level - %msg %n%n A küldő a %F forráskód %L. sorából a %logger{1}.%M függvény volt. %n" /> -->\r
+ <PatternLayout pattern="%d{HH:mm:ss} %level - %msg %n" />\r
+<!-- <HTMLLayout /> -->\r
</HTMLMailAppender>\r
<RollingFile name="RollingFile" fileName="${fileName}" filePattern="${filePattern}">\r
<PatternLayout pattern="%d{HH:mm:ss.SSS} [%-5level] [%t] [%M] [%class{36}] %msg%n %throwable{short}" />\r
ad_nonsecureport=3268\r
ad_base_dn=DC=intra,DC=echotv,DC=hu\r
ad_admin_map=G_ECH_U_INFORMATIKUSOK,G_ECH_U_MUSZAKVEZETOK,ECH-ISILON-ADMINS\r
-local_users=user:password\r
+local_users=user:password;kuka:kuka\r
local_admins=root:password\r
local_jobsubmitters=lebony:lebony
\ No newline at end of file
<stringAttribute key="org.eclipse.jdt.launching.JRE_CONTAINER" value="org.eclipse.jdt.launching.JRE_CONTAINER/org.eclipse.jdt.internal.debug.ui.launcher.StandardVMType/JavaSE-1.8"/>\r
<stringAttribute key="org.eclipse.jdt.launching.PROGRAM_ARGUMENTS" value="-os ${target.os} -ws ${target.ws} -arch ${target.arch} -nl ${target.nl} -consoleLog -console"/>\r
<stringAttribute key="org.eclipse.jdt.launching.SOURCE_PATH_PROVIDER" value="org.eclipse.pde.ui.workbenchClasspathProvider"/>\r
-<stringAttribute key="org.eclipse.jdt.launching.VM_ARGUMENTS" value="-Djavax.ws.rs.ext.RuntimeDelegate=org.jboss.resteasy.spi.ResteasyProviderFactory -Dorg.eclipse.epp.logging.aeri.skipReports=true -Declipse.ignoreApp=true -Dosgi.noShutdown=true -Djobengine.jobsteps.root="${workspace_loc}/user.jobengine.executors/config" -Djobengine.jobtemplates.root="${workspace_loc}/user.jobengine.executors/jobtemplates" -Djobengine.db.url=jdbc:db2://10.10.1.27:50000/mc:retrieveMessagesFromServerOnGetMessage=true; -Djobengine.db.user=db2admin -Djobengine.db.password=password -Djobengine.loglevel=INFO -Dlog4j.configurationFile="${workspace_loc}/-configuration/log4j2.xml" -Djobengine.jobsteps.alternate.root="${workspace_loc}/user.jobengine.executors/bin/user/jobengine/server/steps" -Djetty.home="${workspace_loc:}/-configuration/jetty" -Djetty.etc.config.urls=etc/user-jetty.xml,etc/user-jetty-ssl.xml,etc/user-jetty-ssl-context.xml,,etc/user-jetty-http.xml,etc/user-jetty-https.xml -Djobengine.octopus.api.address=http://10.10.1.11/api/v1 -Djobengine.octopus.api.user=mam -Djobengine.octopus.api.password=napocska -Djobengine.nosql.db.url=jdbc:db2://10.10.1.27:50000/mc:retrieveMessagesFromServerOnGetMessage=true; -Djobengine.nosql.db.user=db2admin -Djobengine.nosql.db.password=password -Djobengine.nosql.db.schema=test -Djobengine.selenio.address=http://10.10.1.71:44000/TranscodeMgrWS?wsdl -Djobengine.selenio.projectfilepath=\\10.10.1.71\Data\Blueprints\MP4_H264_AAC.zenium -Djobengine.jobscheduling.config="${workspace_loc}/-configuration/scheduledjobs.json" -Dnexio.host=10.10.1.55 -Dnexio.disable=true -Djobengine.octopus.rundowns.name=rundowns -Djobengine.octopus.storyfolders.name=storyfolders -Djobengine.octopus.stories.name=stories -Dmediacube.auth.location="${workspace_loc}/-configuration/mediacube-auth.properties""/>\r
+<stringAttribute key="org.eclipse.jdt.launching.VM_ARGUMENTS" value="-Djavax.ws.rs.ext.RuntimeDelegate=org.jboss.resteasy.spi.ResteasyProviderFactory -Dorg.eclipse.epp.logging.aeri.skipReports=true -Declipse.ignoreApp=true -Dosgi.noShutdown=true -Djobengine.jobsteps.root="${workspace_loc}/user.jobengine.executors/config" -Djobengine.jobtemplates.root="${workspace_loc}/user.jobengine.executors/jobtemplates" -Djobengine.db.url=jdbc:db2://10.10.1.27:50000/mc:retrieveMessagesFromServerOnGetMessage=true; -Djobengine.db.user=db2admin -Djobengine.db.password=password -Djobengine.loglevel=INFO -Dlog4j.configurationFile="${workspace_loc}/-configuration/log4j2.xml" -Djobengine.jobsteps.alternate.root="${workspace_loc}/user.jobengine.executors/bin/user/jobengine/server/steps" -Djetty.home="${workspace_loc:}/-configuration/jetty" -Djetty.etc.config.urls=etc/user-jetty.xml,etc/user-jetty-ssl.xml,etc/user-jetty-ssl-context.xml,,etc/user-jetty-http.xml,etc/user-jetty-https.xml -Djobengine.octopus.api.address=http://10.10.1.11/api/v1 -Djobengine.octopus.api.user=mam -Djobengine.octopus.api.password=napocska -Djobengine.nosql.db.url=jdbc:db2://10.10.1.27:50000/mc:retrieveMessagesFromServerOnGetMessage=true; -Djobengine.nosql.db.user=db2admin -Djobengine.nosql.db.password=password -Djobengine.nosql.db.schema=test -Djobengine.selenio.address=http://10.10.1.71:44000/TranscodeMgrWS?wsdl -Djobengine.selenio.projectfilepath=\\10.10.1.71\Data\Blueprints\MP4_H264_AAC.zenium -Djobengine.jobscheduling.config="${workspace_loc}/-configuration/scheduledjobs.json" -Dnexio.host=10.10.1.55 -Dnexio.disable=true -Djobengine.octopus.rundowns.name=new-rundowns -Djobengine.octopus.storyfolders.name=new-storyfolders -Djobengine.octopus.stories.name=new-stories -Dmediacube.auth.location="${workspace_loc}/-configuration/mediacube-auth.properties""/>\r
<stringAttribute key="pde.version" value="3.3"/>\r
<booleanAttribute key="show_selected_only" value="true"/>\r
<stringAttribute key="target_bundles" value="cglib@default:default,com.auth0.java-jwt@default:default,com.fasterxml.jackson.core.jackson-annotations@default:default,com.fasterxml.jackson.core.jackson-core@default:default,com.fasterxml.jackson.core.jackson-databind@default:default,com.fasterxml.jackson.datatype.jackson-datatype-joda@default:default,com.fasterxml.jackson.jaxrs.jackson-jaxrs-base@default:default,com.fasterxml.jackson.jaxrs.jackson-jaxrs-json-provider@default:default,com.fasterxml.jackson.module.jackson-module-jaxb-annotations@default:default,com.google.guava@default:default,com.ibm.db2jcc4@default:default,com.ibm.nosql@default:default,com.microsoft.sqlserver.sqljdbc@default:default,com.sun.el.javax.el@default:default,commons-logging@default:default,javassist@default:default,javax.annotation-api@default:default,javax.inject@default:default,javax.mail@default:default,javax.servlet-api@default:default,javax.servlet.jsp-api@default:default,javax.validation.api@default:default,javax.ws.rs-api@default:default,jcifs@default:default,joda-time@default:default,org.apache.aries.spifly.dynamic.bundle@default:default,org.apache.aries.util@default:default,org.apache.commons.beanutils@default:default,org.apache.commons.codec@default:default,org.apache.commons.collections@default:default,org.apache.commons.digester@default:default,org.apache.commons.io@default:default,org.apache.commons.lang3@default:default,org.apache.commons.lang@default:default,org.apache.commons.logging@default:default,org.apache.commons.net@default:default,org.apache.felix.gogo.command@default:default,org.apache.felix.gogo.runtime@default:default,org.apache.felix.gogo.shell@default:default,org.apache.httpcomponents.httpclient@default:default,org.apache.httpcomponents.httpcore@default:default,org.apache.jasper.glassfish@default:default,org.apache.logging.log4j.api@1:true,org.apache.logging.log4j.core@default:default,org.apache.logging.log4j.slf4j-impl@default:default,org.apache.servicemix.bundles.quartz@default:default,org.eclipse.equinox.common@2:true,org.eclipse.equinox.console@default:default,org.eclipse.equinox.ds@1:true,org.eclipse.equinox.util@default:default,org.eclipse.jdt.core.compiler.batch@default:default,org.eclipse.jetty.client@default:default,org.eclipse.jetty.deploy@default:default,org.eclipse.jetty.http@default:default,org.eclipse.jetty.io@default:default,org.eclipse.jetty.jsp@default:default,org.eclipse.jetty.osgi.boot.jsp@default:false,org.eclipse.jetty.osgi.boot@default:true,org.eclipse.jetty.schemas@default:default,org.eclipse.jetty.security@default:default,org.eclipse.jetty.server@default:default,org.eclipse.jetty.servlet@default:default,org.eclipse.jetty.util@default:default,org.eclipse.jetty.webapp@default:default,org.eclipse.jetty.websocket.api@default:default,org.eclipse.jetty.websocket.client@default:default,org.eclipse.jetty.websocket.common@default:default,org.eclipse.jetty.websocket.server@default:default,org.eclipse.jetty.websocket.servlet@default:default,org.eclipse.jetty.xml@default:default,org.eclipse.osgi.services@default:default,org.eclipse.osgi@-1:true,org.hamcrest.core@default:default,org.jboss.resteasy.client@default:default,org.jboss.resteasy.jaxrs@default:default,org.jmock.junit4@default:default,org.jmock@default:default,org.junit@default:default,org.jvnet.mimepull@default:default,org.objectweb.asm.commons@default:default,org.objectweb.asm.tree@default:default,org.objectweb.asm@default:default,org.reflections@default:default,slf4j.api@default:default,slf4j.simple@default:false"/>\r
import java.io.ByteArrayOutputStream;
import java.io.IOException;
import java.io.OutputStream;
+import java.util.ArrayList;
+import java.util.Arrays;
import java.util.Date;
+import java.util.List;
+import java.util.Map;
import java.util.Properties;
+import java.util.concurrent.ConcurrentHashMap;
import javax.activation.DataSource;
import javax.mail.Authenticator;
import javax.mail.internet.InternetHeaders;
import javax.mail.internet.MimeBodyPart;
import javax.mail.internet.MimeMessage;
-import javax.mail.internet.MimeMessage.RecipientType;
import javax.mail.internet.MimeMultipart;
import javax.mail.internet.MimeUtility;
import javax.mail.util.ByteArrayDataSource;
import org.apache.logging.log4j.util.PropertiesUtil;
import org.apache.logging.log4j.util.Strings;
+import user.commons.MediaCubeFinishMarker;
import user.commons.MediaCubeMarker;
/**
}
private static final SMTPManagerFactory FACTORY = new SMTPManagerFactory();
+ private static Map<String, List<LogEvent>> sessionEvents = new ConcurrentHashMap<>();
private static MimeMessage createMimeMessage(final FactoryData data, final Session session, final LogEvent appendEvent) throws MessagingException {
return new MimeMessageBuilder(session).setFrom(data.from).setReplyTo(data.replyto).setRecipients(Message.RecipientType.TO, data.to)
return encoded.toByteArray();
}
- protected byte[] formatContentToBytes(final LogEvent[] priorEvents, final LogEvent appendEvent, final Layout<?> layout) throws IOException {
+ protected byte[] formatContentToBytes(final List<LogEvent> events, final Layout<?> layout) throws IOException {
final ByteArrayOutputStream raw = new ByteArrayOutputStream();
- writeContent(priorEvents, appendEvent, layout, raw);
+ writeContent(events, layout, raw);
return raw.toByteArray();
}
return mp;
}
+ private void sendEmail(final String to, byte[] content, String contentType) {
+ try {
+ final String encoding = getEncoding(content, contentType);
+ final byte[] encodedBytes = encodeContentToBytes(content, encoding);
+ final InternetHeaders headers = getHeaders(contentType, encoding);
+ final MimeMultipart mp = getMimeMultipart(encodedBytes, headers);
+ sendMultipartMessage(message, mp);
+ } catch (final MessagingException | IOException | RuntimeException e) {
+ logError("Caught exception while sending e-mail notification.", e);
+ throw new LoggingException("Error occurred while sending email", e);
+ }
+ }
+
+ private void sendEvents(final Layout<?> layout, final List<LogEvent> events, final String to) {
+ try {
+ final byte[] rawBytes = formatContentToBytes(events, layout);
+ final String contentType = layout.getContentType();
+ sendEmail(to, rawBytes, contentType);
+ } catch (Exception e) {
+ logError("Caught exception while sending e-mail notification.", e);
+ throw new LoggingException("Error occurred while sending email", e);
+ }
+ }
+
/**
* Send the contents of the cyclic buffer as an e-mail message.
*
* The event that triggered the send.
*/
public void sendEvents(final Layout<?> layout, final LogEvent appendEvent) {
- if (message == null) {
+ if (message == null)
connect(appendEvent);
- }
- try {
- if (appendEvent.getMarker() instanceof MediaCubeMarker) {
- MediaCubeMarker mcm = (MediaCubeMarker) appendEvent.getMarker();
- message.setRecipients(RecipientType.TO, mcm.getTo());
- } else
- message.setRecipients(RecipientType.TO, defaultRecipient);
- //final LogEvent[] priorEvents = buffer.removeAll();
- // LOG4J-310: log appendEvent even if priorEvents is empty
-
- final byte[] rawBytes = formatContentToBytes(null, appendEvent, layout);
+ //System.out.println(appendEvent.getMarker().getClass().getSimpleName() + " " + new String(layout.toByteArray(appendEvent)));
+ String to = defaultRecipient;
+ if (appendEvent.getMarker() instanceof MediaCubeFinishMarker) {
+ MediaCubeFinishMarker mcm = (MediaCubeFinishMarker) appendEvent.getMarker();
+ storeSessionEvent(mcm, appendEvent);
+ sendSessionEvents(mcm, layout);
+ return;
+ }
- final String contentType = layout.getContentType();
- final String encoding = getEncoding(rawBytes, contentType);
- final byte[] encodedBytes = encodeContentToBytes(rawBytes, encoding);
+ if (appendEvent.getMarker() instanceof MediaCubeMarker) {
+ MediaCubeMarker mcm = (MediaCubeMarker) appendEvent.getMarker();
+ if (mcm.getSessionID() == null) {
+ if (mcm.getTo() != null)
+ to = mcm.getTo();
+ } else {
+ storeSessionEvent(mcm, appendEvent);
+ return;
+ }
- final InternetHeaders headers = getHeaders(contentType, encoding);
- final MimeMultipart mp = getMimeMultipart(encodedBytes, headers);
- sendMultipartMessage(message, mp);
- } catch (final MessagingException | IOException | RuntimeException e) {
- logError("Caught exception while sending e-mail notification.", e);
- throw new LoggingException("Error occurred while sending email", e);
}
+
+ sendEvents(layout, Arrays.asList(appendEvent), to);
}
protected void sendMultipartMessage(final MimeMessage msg, final MimeMultipart mp) throws MessagingException {
}
}
- protected void writeBuffer(final LogEvent[] priorEvents, final LogEvent appendEvent, final Layout<?> layout, final OutputStream out) throws IOException {
- // for (final LogEvent priorEvent : priorEvents) {
- // final byte[] bytes = layout.toByteArray(priorEvent);
- // out.write(bytes);
- // }
+ private void sendSessionEvents(final MediaCubeMarker mcm, final Layout<?> layout) {
+ String sessionID = mcm.getSessionID();
+ List<LogEvent> events = sessionEvents.get(sessionID);
+ if (events == null)
+ return;
+ String to = mcm.getTo() == null ? defaultRecipient : mcm.getTo();
+ sendEvents(layout, events, to);
+ sessionEvents.remove(sessionID);
+ }
- final byte[] bytes = layout.toByteArray(appendEvent);
- out.write(bytes);
+ private void storeSessionEvent(MediaCubeMarker mcm, LogEvent appendEvent) {
+ String sessionID = mcm.getSessionID();
+ List<LogEvent> events = sessionEvents.get(sessionID);
+ if (events == null) {
+ events = new ArrayList<>();
+ sessionEvents.put(sessionID, events);
+ }
+ events.add(appendEvent);
+ }
+
+ protected void writeBuffer(List<LogEvent> events, final Layout<?> layout, final OutputStream out) throws IOException {
+ for (final LogEvent priorEvent : events) {
+ final byte[] bytes = layout.toByteArray(priorEvent);
+ out.write(bytes);
+ }
}
- private void writeContent(final LogEvent[] priorEvents, final LogEvent appendEvent, final Layout<?> layout, final ByteArrayOutputStream out)
- throws IOException {
+ private void writeContent(final List<LogEvent> events, final Layout<?> layout, final ByteArrayOutputStream out) throws IOException {
writeHeader(layout, out);
- writeBuffer(priorEvents, appendEvent, layout, out);
+ writeBuffer(events, layout, out);
writeFooter(layout, out);
}
return null;\r
DBObject rundown = octopusAPI.getRundown(scheduledStart);\r
if (rundown == null) {\r
- logger.warn(getMarker(), "A '{}' anyaghoz nem talllható tükör '{}' kezdéssel, ezért nem archiválható.", clipName, scheduledStart);\r
+ logger.error(getMarker(), "A '{}' anyaghoz nem található tükör '{}' kezdéssel, ezért nem archiválható.", clipName, scheduledStart);\r
return null;\r
}\r
\r
return null;\r
RundownArchive result = new RundownArchive();\r
\r
- long id = NoSQLUtils.asLong(rundown, IOctopusAPI.ID);\r
- if (id == 0)\r
- return null;\r
String name = NoSQLUtils.asString(NoSQLUtils.asDBObject(rundown, IOctopusAPI.RUNDOWN_TYPE), IOctopusAPI.NAME);\r
if (StringUtils.isBlank(name))\r
return null;\r
return null;\r
String start = CalendarUtils.toString(CalendarUtils.createCalendar(scheduledStart), SCHEDULED_FORMAT);\r
result.setScheduleDate(scheduledStart);\r
- result.setItemHouseId(String.valueOf(id));\r
+ result.setItemHouseId(String.valueOf(rundownID));\r
result.setItemTitle(String.format("%s %s %s", start, name, channel));\r
\r
StringBuilder sb = new StringBuilder();\r
import org.apache.logging.log4j.LogManager;\r
import org.apache.logging.log4j.Logger;\r
\r
-import user.commons.MediaCubeMarker;\r
import user.jobengine.server.IJobEngine;\r
import user.jobengine.server.IJobRuntime;\r
\r
\r
@StepEntry\r
public Object[] execute(IJobEngine jobEngine, IJobRuntime jobRuntime) throws InterruptedException {\r
- MediaCubeMarker toMarker = new MediaCubeMarker("vasary@gmail.com");\r
- logger.info(toMarker, "A folyamat elindult");\r
- logger.warn(toMarker, "Egy WARN");\r
- logger.error(toMarker, "Egy ERROR");\r
- logger.warn(toMarker, "Egy WARN {}", "paraméterrel");\r
- logger.error(toMarker, "Egy ERROR {}", "paraméterrel");\r
+ logger.info(jobRuntime.getMarker(), "A folyamat elindult");\r
+ logger.warn(jobRuntime.getMarker(), "Egy WARN");\r
+ logger.error(jobRuntime.getMarker(), "Egy ERROR");\r
+ // logger.warn(toMarker, "Egy WARN {}", "paraméterrel");\r
+ // logger.error(toMarker, "Egy ERROR {}", "paraméterrel");\r
for (int i = 0; i < count; i++) {\r
if (!canContinue())\r
break;\r
jobRuntime.incrementProgress((i + 1) * count);\r
Thread.sleep(100);\r
- logger.info(getMarker(), "Log markered" + i);\r
+ logger.info(jobRuntime.getMarker(), "Log markered" + i);\r
+ // logger.info(toMarker, "Log markered" + i);\r
//logger.info("Progress {}", jobRuntime.getProgress());\r
}\r
- logger.info(toMarker, "A folyamat véget ért");\r
+ logger.info(jobRuntime.getFinishMarker(), "A folyamat véget ért");\r
//logger.info(LogCollector.FINISH);\r
return null;\r
}\r
import java.util.List;\r
import java.util.Map;\r
import java.util.Set;\r
-import java.util.concurrent.ConcurrentHashMap;\r
\r
import javax.swing.event.EventListenerList;\r
import javax.ws.rs.client.Entity;\r
import javax.ws.rs.core.MediaType;\r
import javax.ws.rs.core.Response;\r
\r
+import org.apache.commons.lang.StringUtils;\r
import org.apache.logging.log4j.LogManager;\r
import org.apache.logging.log4j.Logger;\r
import org.jboss.resteasy.client.jaxrs.BasicAuthentication;\r
import com.ibm.nosql.json.api.DB;\r
import com.ibm.nosql.json.api.DBCollection;\r
import com.ibm.nosql.json.api.DBCursor;\r
-import com.ibm.nosql.json.api.QueryBuilder;\r
-import com.ibm.nosql.json.api.WriteResult;\r
\r
import user.commons.CalendarUtils;\r
import user.commons.ListUtils;\r
import user.commons.octopus.IOctopusAPI;\r
import user.commons.remotestore.IProgressEventListener;\r
import user.commons.remotestore.ProgressEvent;\r
+import user.jobengine.server.steps.MetadataTypeDetector.MetadataType;\r
\r
public class OctopusDataMiner {\r
+ private static final Logger logger = LogManager.getLogger();\r
private static final String ARCHIVED = "archived";\r
private static final String FILTER = "filter";\r
- private static final String _TMP = "_tmp";\r
- private static final Logger logger = LogManager.getLogger();\r
private static final String LINEFEED = "\r\n";\r
private static final String SIMPLE_LINEFEED = "\n";\r
private static final String SAVING_STORY_ID = "Saving story {}";\r
- // private static final String SAVING_RUNDOWN = "Saving rundown : {} {}";\r
- // private static final String CHECKING_RUNDOWN = "Checking Rundown {} ({}/{})";\r
- private static final String FIELDS_STORYFOLDER_STORIES = "stories,Story.modified,Story.name,Story.id,Story.mosObjects,Story.script,Story.type,Story.format,Story.customColumns,CustomColumn.label,CustomColumn.value";\r
- private static final String FIELDS_RUNDOWN_STORIES = "slugs,Slug.story,Slug.position,Story.name,Story.id,Story.modified,Story.mosObjects,Story.script,Story.type,Story.format,Story.customColumns,CustomColumn.label,CustomColumn.value";\r
+ private static final String FIELDS_STORIES = "id,name,modified,type,format,mosObjects,script,customColumns,CustomColumn.label,CustomColumn.value,reporters,User.longName,User.userName";\r
private static final String FIELDS_RUNDOWN = "id,name,modified,scheduledStart,channel,Channel.name,rundownType,RundownType.name";\r
private static final String FIELDS_RUNDOWN_STORYIDS = "id,slugs,Slug.storyId,Slug.position";\r
private static final String FIELDS_STORYFOLDER = "id,name,modified";\r
private static final String OCTOPUS_DEVICE_NAME = "Octopus-Device-Name";\r
private static final String OCTOPUS_DEVICE_ID = "Octopus-Device-Id";\r
private static final String FIELDS = "fields";\r
- private static final String CHECKING_STORY_FOLDER = "Checking StoryFolder %s (%d/%d)";\r
private static final String EXIT = "Exit";\r
private static final String RESULT = "result";\r
private static final String STORY_FOLDER = "StoryFolder";\r
private static final String FINISHED = "Finished";\r
private static final String STARTING = "Starting";\r
private static final String MOSOBJECT = "Bejátszó: ";\r
+ private static final Object STORY = "Story";\r
\r
private DB db;\r
private ResteasyWebTarget webTarget;\r
private Map<Long, Object> currentRundowns;\r
private Map<Long, Object> currentFolder;\r
private Map<Long, Object> currentStories;\r
- private String RUNDOWN_COLLECTION;\r
- private String FOLDER_COLLECTION;\r
- private String STORY_COLLECTION;\r
private boolean includeArchived;\r
private Calendar zeroDate = CalendarUtils.createCalendar(2017, 11, 4);\r
+ private int objectCount;\r
+ private int currentObjectIndex;\r
\r
public OctopusDataMiner() {\r
db = NoSQLUtils.getNoSQLDB();\r
List<BasicDBObject> stories = NoSQLUtils.asList(storyFolderWithStoryIds, IOctopusAPI.STORIES);\r
if (stories == null)\r
continue;\r
- if (!folderIDs.contains(storyFolderId))\r
- folderIDs.add(storyFolderId);\r
+\r
+ folderIDs.add(storyFolderId);\r
+\r
long position = 1;\r
for (BasicDBObject story : stories) {\r
long storyId = story.getLong(IOctopusAPI.ID);\r
\r
+ storyIDs.add(storyId);\r
+\r
BasicDBList references = result.get(storyId);\r
if (references == null) {\r
references = new BasicDBList();\r
if (slugs == null)\r
continue;\r
\r
- if (!rundownIDs.contains(rundownId))\r
- rundownIDs.add(rundownId);\r
+ rundownIDs.add(rundownId);\r
\r
for (BasicDBObject slug : slugs) {\r
if (!slug.containsKey(IOctopusAPI.STORYID))\r
continue;\r
long storyId = slug.getLong(IOctopusAPI.STORYID);\r
+\r
+ storyIDs.add(storyId);\r
+\r
BasicDBList references = result.get(storyId);\r
if (references == null) {\r
references = new BasicDBList();\r
}\r
\r
public void clear() {\r
- db.getCollection(RUNDOWN_COLLECTION).remove();\r
- db.getCollection(STORY_COLLECTION).remove();\r
- db.getCollection(FOLDER_COLLECTION).remove();\r
- db.getCollection(IOctopusAPI.TIME_COLLECTION_NAME).remove();\r
- }\r
-\r
- private String concatParentsToStoryFolder(BasicDBObject actual, String name) {\r
-\r
- String fields = "name,id,parent";\r
- Response response = query("StoryFolder/" + actual.getLong("id"), fields).get();\r
- String json = response.readEntity(String.class);\r
- BasicDBObject resultObject = (BasicDBObject) JSONUtil.jsonToDbObject(json);\r
- BasicDBObject res = (BasicDBObject) resultObject.get("result");\r
- BasicDBObject parent = (BasicDBObject) res.get("parent");\r
- if (parent == null || parent.isEmpty())\r
- return name;\r
- String parentName = parent.getString(IOctopusAPI.NAME);\r
- String actualName = actual.getString(IOctopusAPI.NAME);\r
- String newName = String.format("%s/%s", parentName, actualName);\r
- return concatParentsToStoryFolder(parent, newName);\r
+ db.getCollection(IOctopusAPI.RUNDOWN_COLLECTION).remove();\r
+ db.getCollection(IOctopusAPI.FOLDER_COLLECTION).remove();\r
+ db.getCollection(IOctopusAPI.STORY_COLLECTION).remove();\r
}\r
\r
private void deleteDiff(Set<Long> oldIDs, Set<Long> newIDs, String collectionName) {\r
}\r
}\r
\r
- private void deleteOrphanRundowns() {\r
- try {\r
- DBCollection collection = db.getCollection(RUNDOWN_COLLECTION);\r
- BasicDBObject query = (BasicDBObject) QueryBuilder.start().put(IOctopusAPI.ID).notIn(storyRundowns.keySet().toArray()).get();\r
- WriteResult res = collection.remove(query);\r
- logger.trace(String.format("Deleted orphan rundowns: %d", res.getN()));\r
- } catch (Exception e) {\r
- logger.error(e);\r
- }\r
- }\r
-\r
- private void deleteOrphanStories() {\r
- try {\r
- DBCollection collection = db.getCollection(STORY_COLLECTION);\r
- BasicDBObject query = (BasicDBObject) QueryBuilder.start().put("id").notIn(storyIDs.toArray()).get();\r
- WriteResult res = collection.remove(query);\r
- logger.trace(String.format("Deleted orphan stories: %d", res.getN()));\r
- } catch (Exception e) {\r
- logger.error(e);\r
- }\r
- }\r
-\r
- private void deleteOrphanStoryFolders() {\r
- try {\r
- DBCollection collection = db.getCollection(RUNDOWN_COLLECTION);\r
- BasicDBObject query = (BasicDBObject) QueryBuilder.start().put(IOctopusAPI.ID).notIn(storyStoryFolders.keySet().toArray()).get();\r
- WriteResult res = collection.remove(query);\r
- logger.trace(String.format("Deleted orphan rundowns: %d", res.getN()));\r
- } catch (Exception e) {\r
- logger.error(e);\r
- }\r
- }\r
-\r
- private void ensureIndexes() {\r
- DBCollection collection = db.getCollection(FOLDER_COLLECTION);\r
- if (collection.count() == 0)\r
- collection.ensureIndex(IOctopusAPI.ID);\r
- collection = db.getCollection(RUNDOWN_COLLECTION);\r
- if (collection.count() == 0) {\r
- collection.ensureIndex(IOctopusAPI.ID);\r
- collection.ensureIndex(IOctopusAPI.SCHEDULED_START);\r
- }\r
- collection = db.getCollection(STORY_COLLECTION);\r
- if (collection.count() == 0)\r
- collection.ensureIndex(IOctopusAPI.ID);\r
- }\r
-\r
public void execute(boolean includeArchived) throws Exception {\r
this.includeArchived = includeArchived;\r
logger.trace(STARTING);\r
-\r
- // String MAIN_RUNDOWN_COLLECTION = IOctopusAPI.RUNDOWN_COLLECTION;\r
- // String MAIN_FOLDER_COLLECTION = IOctopusAPI.FOLDER_COLLECTION;\r
- // String MAIN_STORY_COLLECTION = IOctopusAPI.STORY_COLLECTION;\r
-\r
- // RUNDOWN_COLLECTION = MAIN_RUNDOWN_COLLECTION + _TMP;\r
- // FOLDER_COLLECTION = MAIN_FOLDER_COLLECTION + _TMP;\r
- // STORY_COLLECTION = MAIN_STORY_COLLECTION + _TMP;\r
- //\r
- // try {\r
- // db.getCollection(RUNDOWN_COLLECTION).drop();\r
- // db.getCollection(FOLDER_COLLECTION).drop();\r
- // db.getCollection(STORY_COLLECTION).drop();\r
- // } catch (Exception e) {\r
- // logger.catching(e);\r
- // throw e;\r
- // }\r
-\r
- RUNDOWN_COLLECTION = IOctopusAPI.RUNDOWN_COLLECTION;\r
- FOLDER_COLLECTION = IOctopusAPI.FOLDER_COLLECTION;\r
- STORY_COLLECTION = IOctopusAPI.STORY_COLLECTION;\r
-\r
- //NoSQLUtils.asList((BasicDBList)\r
- currentRundowns = getCurrentIDs(RUNDOWN_COLLECTION);\r
- currentFolder = getCurrentIDs(FOLDER_COLLECTION);\r
- currentStories = getCurrentIDs(STORY_COLLECTION);\r
+ //current = korábbi szinkronizálás\r
+ currentRundowns = getCurrentIDs(IOctopusAPI.RUNDOWN_COLLECTION);\r
+ currentFolder = getCurrentIDs(IOctopusAPI.FOLDER_COLLECTION);\r
+ currentStories = getCurrentIDs(IOctopusAPI.STORY_COLLECTION);\r
\r
BasicDBList rundowns = null;\r
BasicDBList storyFolders = null;\r
throw e;\r
}\r
\r
- processRundowns(rundowns);\r
- processStoryFolders(storyFolders);\r
-\r
- deleteDiff(currentRundowns.keySet(), rundownIDs, RUNDOWN_COLLECTION);\r
- deleteDiff(currentFolder.keySet(), folderIDs, FOLDER_COLLECTION);\r
- deleteDiff(currentStories.keySet(), storyIDs, STORY_COLLECTION);\r
- // deleteOrphanRundowns();\r
- // deleteOrphanStoryFolders();\r
- // deleteOrphanStories();\r
-\r
- //a sorrend fontos !\r
- // updateDiff(MAIN_STORY_COLLECTION, STORY_COLLECTION, IOctopusAPI.ID);\r
- // updateDiff(MAIN_RUNDOWN_COLLECTION, RUNDOWN_COLLECTION, IOctopusAPI.ID);\r
- // updateDiff(MAIN_FOLDER_COLLECTION, FOLDER_COLLECTION, IOctopusAPI.ID);\r
- //\r
- // updateDeleteDiff(MAIN_RUNDOWN_COLLECTION, RUNDOWN_COLLECTION, IOctopusAPI.ID);\r
- // updateDeleteDiff(MAIN_FOLDER_COLLECTION, FOLDER_COLLECTION, IOctopusAPI.ID);\r
- // updateDeleteDiff(MAIN_STORY_COLLECTION, STORY_COLLECTION, IOctopusAPI.ID);\r
-\r
- // try {\r
- // db.getCollection(RUNDOWN_COLLECTION).rename(MAIN_RUNDOWN_COLLECTION, true);\r
- // db.getCollection(FOLDER_COLLECTION).rename(MAIN_FOLDER_COLLECTION, true);\r
- // db.getCollection(STORY_COLLECTION).rename(MAIN_STORY_COLLECTION, true);\r
- // } catch (Exception e) {\r
- // logger.catching(e);\r
- // throw e;\r
- // }\r
-\r
- // logger.info("Activate");\r
+ objectCount = rundownIDs.size() + folderIDs.size() + storyIDs.size();\r
\r
- // try {\r
- // long ts = new Date().getTime();\r
- // db.getCollection(RUNDOWN_COLLECTION).exportFile(String.format("/opt/mediacube-test/log/%s-%s.js", RUNDOWN_COLLECTION, ts));\r
- // db.getCollection(FOLDER_COLLECTION).exportFile(String.format("/opt/mediacube-test/log/%s-%s.js", FOLDER_COLLECTION, ts));\r
- // db.getCollection(STORY_COLLECTION).exportFile(String.format("/opt/mediacube-test/log/%s-%s.js", STORY_COLLECTION, ts));\r
- // } catch (Exception e) {\r
- // logger.catching(e);\r
- // throw e;\r
- // }\r
+ storeStories();\r
+ storeRundowns(rundowns);\r
+ storeStoryFolders(storyFolders);\r
\r
+ deleteDiff(currentRundowns.keySet(), rundownIDs, IOctopusAPI.RUNDOWN_COLLECTION);\r
+ deleteDiff(currentFolder.keySet(), folderIDs, IOctopusAPI.FOLDER_COLLECTION);\r
+ deleteDiff(currentStories.keySet(), storyIDs, IOctopusAPI.STORY_COLLECTION);\r
logger.trace(FINISHED);\r
- //throw new Exception("Mérés");\r
}\r
\r
private String extractContent(BasicDBObject content) {\r
List<BasicDBObject> mosObjects = NoSQLUtils.asList(story, IOctopusAPI.MOS_OBJECTS);\r
if (mosObjects == null)\r
return null;\r
+\r
+ Map<String, String> mosLabels = extractScriptMosObjectIDs(story);\r
BasicDBList result = null;\r
for (BasicDBObject mosObject : mosObjects) {\r
if (!mosObject.containsKey(IOctopusAPI.MOS_ID))\r
String mosId = mosObject.getString(IOctopusAPI.MOS_ID);\r
if (!IOctopusAPI.NEXIO_MOS.equals(mosId))\r
continue;\r
+ String objId = mosObject.getString(IOctopusAPI.OBJ_ID);\r
+ if (objId == null)\r
+ continue;\r
+\r
+ MetadataType metadataType = MetadataTypeDetector.GuessMetadataType(objId);\r
+ if (!MetadataType.OCTOPUSPLACEHOLDER.equals(metadataType) && !MetadataType.OCTOPUSSTORY.equals(metadataType))\r
+ continue;\r
if (result == null)\r
result = new BasicDBList();\r
+\r
+ String label = mosLabels.get(objId);\r
+ if (StringUtils.isNotBlank(label))\r
+ mosObject.append(IOctopusAPI.LABEL, label);\r
result.add(mosObject);\r
}\r
return result;\r
return sb.length() == 0 ? null : sb.toString();\r
}\r
\r
+ private Map<String, String> extractScriptMosObjectIDs(BasicDBObject story) {\r
+ BasicDBObject script = NoSQLUtils.asDBObject(story, IOctopusAPI.SCRIPT);\r
+ if (script == null || script.isEmpty())\r
+ return null;\r
+\r
+ List<BasicDBObject> body = NoSQLUtils.asList(script, IOctopusAPI.BODY);\r
+ if (body == null || body.size() == 0)\r
+ return null;\r
+\r
+ Map<String, String> result = new HashMap<>();\r
+\r
+ for (BasicDBObject bodyItem : body) {\r
+ String label = bodyItem.getString(IOctopusAPI.LABEL);\r
+ if (StringUtils.isBlank(label))\r
+ continue;\r
+ BasicDBObject clip = NoSQLUtils.asDBObject(bodyItem, IOctopusAPI.CLIP);\r
+ if (clip == null)\r
+ continue;\r
+ BasicDBObject obj = NoSQLUtils.asDBObject(clip, IOctopusAPI.OBJECT);\r
+ if (obj == null)\r
+ continue;\r
+ String mosId = obj.getString(IOctopusAPI.MOS_ID);\r
+ if (!IOctopusAPI.NEXIO_MOS.equals(mosId))\r
+ continue;\r
+ String objId = obj.getString(IOctopusAPI.OBJ_ID);\r
+ if (objId == null)\r
+ continue;\r
+ result.put(mosId, label);\r
+\r
+ }\r
+ return result;\r
+ }\r
+\r
+ private void fireProgressEvent() {\r
+ currentObjectIndex++;\r
+ logger.info("currentObjectIndex {}", currentObjectIndex);\r
+ int progress = currentObjectIndex * 100 / objectCount;\r
+ if (progress != progressEvent.getProgress()) {\r
+ progressEvent.setProgress(progress);\r
+ fireProgressEvent(progressEvent);\r
+ }\r
+ }\r
+\r
private void fireProgressEvent(ProgressEvent evt) {\r
logger.debug("Progress changed to " + evt.getProgress() + "%");\r
if (progressListenerList == null)\r
return result;\r
}\r
\r
- private void processRundowns(BasicDBList rundowns) throws Exception {\r
- if (rundowns == null || rundowns.size() == 0) {\r
- progressEvent.setProgress(50);\r
- fireProgressEvent(progressEvent);\r
- } else {\r
- logger.info("Process rundowns");\r
- try {\r
- storeRundowns(rundowns, null);\r
- } catch (Exception e) {\r
- logger.catching(e);\r
- throw e;\r
- }\r
- }\r
- }\r
-\r
- private void processStoryFolders(BasicDBList storyFolders) throws Exception {\r
- if (storyFolders == null || storyFolders.size() == 0) {\r
- progressEvent.setProgress(100);\r
- fireProgressEvent(progressEvent);\r
- } else {\r
- logger.info("Process story folders");\r
- try {\r
- storeStoryFolders(storyFolders, null);\r
- } catch (Exception e) {\r
- logger.catching(e);\r
- throw e;\r
- }\r
- }\r
- }\r
-\r
private Builder query(String path, String fields) {\r
ResteasyWebTarget target = webTarget.path(path).queryParam(FIELDS, fields);\r
Builder result = target.request().header(OCTOPUS_DEVICE_ID, apiUser).header(OCTOPUS_DEVICE_NAME, apiPwd);\r
return result;\r
}\r
\r
+ private BasicDBObject queryStory(long storyID) {\r
+ logger.trace(ENTER);\r
+ BasicDBObject result = null;\r
+ Response response = query(String.format("%s/%d", STORY, storyID), FIELDS_STORIES).get();\r
+ String json = response.readEntity(String.class);\r
+ BasicDBObject resultObject = (BasicDBObject) JSONUtil.jsonToDbObject(json);\r
+ if (resultObject == null)\r
+ logger.error("Story {} is not available", storyID);\r
+ else\r
+ result = NoSQLUtils.asDBObject(resultObject, RESULT);\r
+ logger.trace(EXIT);\r
+ return result;\r
+ }\r
+\r
private BasicDBObject queryStoryFolder(BasicDBObject storyFolder, String fields) {\r
logger.trace(ENTER);\r
BasicDBObject result = null;\r
logger.trace(ENTER);\r
String name = rundown.containsKey(IOctopusAPI.NAME) ? rundown.getString(IOctopusAPI.NAME) : null;\r
logger.debug("Storing rundown {} {}", name, rundown.get(IOctopusAPI.SCHEDULED_START));\r
- BasicDBObject rundownWithStories = queryRundown(rundown, FIELDS_RUNDOWN_STORIES);\r
- if (rundownWithStories != null) {\r
- Date scheduledStart = toDate(rundown, IOctopusAPI.SCHEDULED_START);\r
- if (scheduledStart != null && scheduledStart.after(zeroDate.getTime())) {\r
- BasicDBList stories = NoSQLUtils.asDBList(rundownWithStories, IOctopusAPI.SLUGS);\r
- if (stories != null)\r
- storeRundownStories(stories);\r
- rundown.put(IOctopusAPI.SCHEDULED_START, toDate(rundown, IOctopusAPI.SCHEDULED_START));\r
- rundown.put(IOctopusAPI.MODIFIED, toDate(rundown, IOctopusAPI.MODIFIED));\r
- DBCollection collection = db.getCollection(RUNDOWN_COLLECTION);\r
- long rundownID = NoSQLUtils.asLong(rundown, IOctopusAPI.ID);\r
- setObjectID(currentRundowns, rundownID, rundown);\r
- collection.save(rundown);\r
- }\r
+ Date scheduledStart = toDate(rundown, IOctopusAPI.SCHEDULED_START);\r
+ if (scheduledStart != null && scheduledStart.after(zeroDate.getTime())) {\r
+ rundown.put(IOctopusAPI.SCHEDULED_START, toDate(rundown, IOctopusAPI.SCHEDULED_START));\r
+ rundown.put(IOctopusAPI.MODIFIED, toDate(rundown, IOctopusAPI.MODIFIED));\r
+ DBCollection collection = db.getCollection(IOctopusAPI.RUNDOWN_COLLECTION);\r
+ long rundownID = NoSQLUtils.asLong(rundown, IOctopusAPI.ID);\r
+ setObjectID(currentRundowns, rundownID, rundown);\r
+ collection.save(rundown);\r
}\r
- logger.trace(EXIT);\r
+ logger.trace(ENTER);\r
}\r
\r
- private void storeRundowns(BasicDBList rundowns, Date lastUpdateTime) {\r
+ private void storeRundowns(BasicDBList rundowns) {\r
logger.trace(ENTER);\r
List<BasicDBObject> rundownsList = NoSQLUtils.asList(rundowns);\r
- int idx = 1;\r
for (BasicDBObject rundown : rundownsList) {\r
- //logger.info(CHECKING_RUNDOWN, rundown.getLong(IOctopusAPI.ID), rundownsList.size(), idx);\r
- storeRundown(rundown);\r
- int progress = idx * 50 / rundownsList.size();\r
- if (progress - progressEvent.getProgress() > 0) {\r
- progressEvent.setProgress(progress);\r
- fireProgressEvent(progressEvent);\r
+ try {\r
+ storeRundown(rundown);\r
+ } catch (Exception e) {\r
+ logger.catching(e);\r
+ throw e;\r
}\r
-\r
- idx++;\r
+ fireProgressEvent();\r
}\r
- logger.trace(EXIT);\r
+ logger.trace(ENTER);\r
}\r
\r
- private void storeRundownStories(BasicDBList slugs) {\r
+ private void storeStories() {\r
logger.trace(ENTER);\r
- List<BasicDBObject> slugsList = NoSQLUtils.asList(slugs);\r
- for (BasicDBObject slug : slugsList) {\r
- if (slug.containsKey(IOctopusAPI.STORY))\r
- storeStory((BasicDBObject) slug.get(IOctopusAPI.STORY));\r
+ for (long storyID : storyIDs) {\r
+ try {\r
+ BasicDBObject story = queryStory(storyID);\r
+ if (story != null)\r
+ storeStory(story);\r
+ } catch (Exception e) {\r
+ logger.catching(e);\r
+ throw e;\r
+ }\r
+ fireProgressEvent();\r
}\r
- logger.trace(EXIT);\r
+ logger.trace(ENTER);\r
}\r
\r
private void storeStory(BasicDBObject story) {\r
return;\r
}\r
long storyID = story.getLong(IOctopusAPI.ID);\r
- boolean isContains = storyIDs.contains(storyID);\r
- if (isContains)\r
- return;\r
- storyIDs.add(storyID);\r
+\r
story.put(IOctopusAPI.MODIFIED, toDate(story, IOctopusAPI.MODIFIED));\r
BasicDBList rundownRef = storyRundowns.get(storyID);\r
BasicDBList storyFolderRef = storyStoryFolders.get(storyID);\r
BasicDBList modifiedMOS = extractRelevantMOSObjects(story);\r
-\r
- DBCollection collection = db.getCollection(STORY_COLLECTION);\r
+ DBCollection collection = db.getCollection(IOctopusAPI.STORY_COLLECTION);\r
\r
if (rundownRef != null)\r
story.put(IOctopusAPI.REF_RUNDOWN, rundownRef);\r
logger.trace(EXIT);\r
}\r
\r
- private void storeStoryFolder(BasicDBObject storyFolder, Date lastUpdateTime) {\r
+ private void storeStoryFolder(BasicDBObject storyFolder) {\r
logger.trace(ENTER);\r
- BasicDBObject storyFoldersWithStories = queryStoryFolder(storyFolder, FIELDS_STORYFOLDER_STORIES);\r
- if (storyFoldersWithStories != null) {\r
- BasicDBList stories = NoSQLUtils.asDBList(storyFoldersWithStories, IOctopusAPI.STORIES);\r
- if (stories != null)\r
- storeStoryFolderStories(stories);\r
- storyFolder.put(IOctopusAPI.MODIFIED, toDate(storyFolder, IOctopusAPI.MODIFIED));\r
- DBCollection collection = db.getCollection(FOLDER_COLLECTION);\r
- String name = storyFolder.getString(IOctopusAPI.NAME);\r
- logger.debug("Storing story folder {}", name);\r
- long folderID = NoSQLUtils.asLong(storyFolder, IOctopusAPI.ID);\r
- setObjectID(currentFolder, folderID, storyFolder);\r
- collection.save(storyFolder);\r
- }\r
+ storyFolder.put(IOctopusAPI.MODIFIED, toDate(storyFolder, IOctopusAPI.MODIFIED));\r
+ DBCollection collection = db.getCollection(IOctopusAPI.FOLDER_COLLECTION);\r
+ String name = storyFolder.getString(IOctopusAPI.NAME);\r
+ logger.debug("Storing story folder {}", name);\r
+ long folderID = NoSQLUtils.asLong(storyFolder, IOctopusAPI.ID);\r
+ setObjectID(currentFolder, folderID, storyFolder);\r
+ collection.save(storyFolder);\r
logger.trace(EXIT);\r
}\r
\r
- private void storeStoryFolders(BasicDBList storyFolders, Date lastUpdateTime) {\r
+ private void storeStoryFolders(BasicDBList storyFolders) {\r
logger.trace(ENTER);\r
List<BasicDBObject> storyFolderList = NoSQLUtils.asList(storyFolders);\r
- int idx = 1;\r
for (BasicDBObject storyFolder : storyFolderList) {\r
- logger.debug(String.format(CHECKING_STORY_FOLDER, storyFolder.getLong(IOctopusAPI.ID), storyFolderList.size(), idx));\r
- storeStoryFolder(storyFolder, lastUpdateTime);\r
- int progress = 50 + (idx * 50 / storyFolderList.size());\r
- if (progress - progressEvent.getProgress() > 0) {\r
- progressEvent.setProgress(progress);\r
- fireProgressEvent(progressEvent);\r
+ try {\r
+ storeStoryFolder(storyFolder);\r
+ } catch (Exception e) {\r
+ logger.catching(e);\r
+ throw e;\r
}\r
- idx++;\r
+ fireProgressEvent();\r
}\r
logger.trace(EXIT);\r
}\r
\r
- private void storeStoryFolderStories(BasicDBList stories) {\r
- logger.trace(ENTER);\r
- List<BasicDBObject> list = NoSQLUtils.asList(stories);\r
- for (BasicDBObject story : list)\r
- storeStory(story);\r
- logger.trace(EXIT);\r
- }\r
-\r
private Date toDate(BasicDBObject obj, String name) {\r
Date result = null;\r
if (obj.containsKey(name)) {\r
return result;\r
}\r
\r
- private void updateDeleteDiff(String oldCollectionName, String newCollectionName, String idFieldName) {\r
- DBCollection oldCollection = db.getCollection(oldCollectionName);\r
- DBCollection newCollection = db.getCollection(newCollectionName);\r
- DBCursor oldCollectionCursor = oldCollection.find(new BasicDBObject(), new BasicDBObject(idFieldName, 1));\r
- if (!oldCollectionCursor.hasNext()) {\r
- logger.error("{} collection is empty", newCollectionName);\r
- return;\r
- }\r
- List<BasicDBObject> oldItems = ListUtils.cast(oldCollectionCursor.toArray());\r
-\r
- DBCursor newCollectionCursor = newCollection.find();\r
- ConcurrentHashMap<Long, BasicDBObject> newItems = null;\r
- if (newCollectionCursor.hasNext()) {\r
- List<BasicDBObject> newList = ListUtils.cast(newCollectionCursor.toArray());\r
- newItems = ListUtils.map(newList, item -> item.getLong(IOctopusAPI.ID));\r
- }\r
- if (newItems == null)\r
- newItems = new ConcurrentHashMap<>();\r
-\r
- for (BasicDBObject oldItem : oldItems) {\r
- if (oldItem == null) {\r
- logger.error("Item is null");\r
- continue;\r
- }\r
- if (!oldItem.containsKey(idFieldName)) {\r
- logger.error("{} is null", idFieldName);\r
- continue;\r
- }\r
- long id = oldItem.getLong(idFieldName);\r
- BasicDBObject newItem = newItems.get(id);\r
- if (newItem == null) {\r
- //remove\r
- logger.info("Deleting {}", oldItem.toPrettyString(null));\r
- oldCollection.remove(new BasicDBObject(idFieldName, id));\r
- }\r
- }\r
- }\r
-\r
- private void updateDiff(String oldCollectionName, String newCollectionName, String idFieldName) {\r
- DBCollection oldCollection = db.getCollection(oldCollectionName);\r
- DBCollection newCollection = db.getCollection(newCollectionName);\r
- DBCursor newCollectionCursor = newCollection.find();\r
- if (!newCollectionCursor.hasNext()) {\r
- logger.error("{} collection is empty", newCollectionName);\r
- return;\r
- }\r
-\r
- List<BasicDBObject> newItems = ListUtils.cast(newCollectionCursor.toArray());\r
-\r
- DBCursor oldCollectionCursor = oldCollection.find();\r
- ConcurrentHashMap<Long, BasicDBObject> oldItems = null;\r
- if (oldCollectionCursor.hasNext()) {\r
- List<BasicDBObject> oldList = ListUtils.cast(oldCollectionCursor.toArray());\r
- oldItems = ListUtils.map(oldList, item -> item.getLong(IOctopusAPI.ID));\r
- }\r
- if (oldItems == null)\r
- oldItems = new ConcurrentHashMap<>();\r
- for (BasicDBObject newItem : newItems) {\r
- if (newItem == null) {\r
- logger.error("Item is null");\r
- continue;\r
- }\r
- if (!newItem.containsKey(idFieldName)) {\r
- logger.error("{} is null", idFieldName);\r
- continue;\r
- }\r
- long id = newItem.getLong(idFieldName);\r
- BasicDBObject oldItem = oldItems.get(id);\r
- boolean save = false;\r
- if (oldItem == null) {\r
- //save\r
- save = true;\r
- } else {\r
- //compare\r
- Object oldDbId = oldItem.getID();\r
- newItem.remove(IOctopusAPI._ID);\r
- oldItem.remove(IOctopusAPI._ID);\r
-\r
- if (!newItem.equals(oldItem)) {\r
- //save\r
- newItem.put(IOctopusAPI._ID, oldDbId);\r
- save = true;\r
- }\r
- }\r
-\r
- if (save) {\r
- logger.info("Saving to {} item {}", oldCollectionName, id);\r
- oldCollection.save(newItem);\r
- }\r
- }\r
- }\r
-\r
}\r
--- /dev/null
+package user.jobengine.server.steps;\r
+\r
+import java.util.Calendar;\r
+import java.util.Date;\r
+import java.util.HashMap;\r
+import java.util.HashSet;\r
+import java.util.List;\r
+import java.util.Map;\r
+import java.util.Set;\r
+import java.util.concurrent.ConcurrentHashMap;\r
+\r
+import javax.swing.event.EventListenerList;\r
+import javax.ws.rs.client.Entity;\r
+import javax.ws.rs.client.Invocation.Builder;\r
+import javax.ws.rs.core.MediaType;\r
+import javax.ws.rs.core.Response;\r
+\r
+import org.apache.logging.log4j.LogManager;\r
+import org.apache.logging.log4j.Logger;\r
+import org.jboss.resteasy.client.jaxrs.BasicAuthentication;\r
+import org.jboss.resteasy.client.jaxrs.ResteasyClient;\r
+import org.jboss.resteasy.client.jaxrs.ResteasyClientBuilder;\r
+import org.jboss.resteasy.client.jaxrs.ResteasyWebTarget;\r
+import org.joda.time.DateTime;\r
+\r
+import com.ibm.nosql.json.JSONUtil;\r
+import com.ibm.nosql.json.api.BasicDBList;\r
+import com.ibm.nosql.json.api.BasicDBObject;\r
+import com.ibm.nosql.json.api.DB;\r
+import com.ibm.nosql.json.api.DBCollection;\r
+import com.ibm.nosql.json.api.DBCursor;\r
+import com.ibm.nosql.json.api.QueryBuilder;\r
+import com.ibm.nosql.json.api.WriteResult;\r
+\r
+import user.commons.CalendarUtils;\r
+import user.commons.ListUtils;\r
+import user.commons.nosql.NoSQLUtils;\r
+import user.commons.octopus.IOctopusAPI;\r
+import user.commons.remotestore.IProgressEventListener;\r
+import user.commons.remotestore.ProgressEvent;\r
+\r
+public class OctopusDataMinerStable {\r
+ private static final String ARCHIVED = "archived";\r
+ private static final String FILTER = "filter";\r
+ private static final String _TMP = "_tmp";\r
+ private static final Logger logger = LogManager.getLogger();\r
+ private static final String LINEFEED = "\r\n";\r
+ private static final String SIMPLE_LINEFEED = "\n";\r
+ private static final String SAVING_STORY_ID = "Saving story {}";\r
+ // private static final String SAVING_RUNDOWN = "Saving rundown : {} {}";\r
+ // private static final String CHECKING_RUNDOWN = "Checking Rundown {} ({}/{})";\r
+ private static final String FIELDS_STORYFOLDER_STORIES = "stories,Story.modified,Story.name,Story.id,Story.mosObjects,Story.script,Story.type,Story.format,Story.customColumns,CustomColumn.label,CustomColumn.value";\r
+ private static final String FIELDS_RUNDOWN_STORIES = "slugs,Slug.story,Slug.position,Story.name,Story.id,Story.modified,Story.mosObjects,Story.script,Story.type,Story.format,Story.customColumns,CustomColumn.label,CustomColumn.value";\r
+ private static final String FIELDS_RUNDOWN = "id,name,modified,scheduledStart,channel,Channel.name,rundownType,RundownType.name";\r
+ private static final String FIELDS_RUNDOWN_STORYIDS = "id,slugs,Slug.storyId,Slug.position";\r
+ private static final String FIELDS_STORYFOLDER = "id,name,modified";\r
+ private static final String FIELDS_STORYFOLDER_STORYIDS = "id,stories,Story.id";\r
+ private static final String RUNDOWN = "Rundown";\r
+ private static final String OCTOPUS_DEVICE_NAME = "Octopus-Device-Name";\r
+ private static final String OCTOPUS_DEVICE_ID = "Octopus-Device-Id";\r
+ private static final String FIELDS = "fields";\r
+ private static final String CHECKING_STORY_FOLDER = "Checking StoryFolder %s (%d/%d)";\r
+ private static final String EXIT = "Exit";\r
+ private static final String RESULT = "result";\r
+ private static final String STORY_FOLDER = "StoryFolder";\r
+ private static final String ENTER = "Enter";\r
+ private static final String FINISHED = "Finished";\r
+ private static final String STARTING = "Starting";\r
+ private static final String MOSOBJECT = "Bejátszó: ";\r
+\r
+ private DB db;\r
+ private ResteasyWebTarget webTarget;\r
+ private String apiUser;\r
+ private String apiPwd;\r
+ private HashSet<Long> storyIDs = new HashSet<>();\r
+ private HashSet<Long> folderIDs = new HashSet<>();\r
+ private HashSet<Long> rundownIDs = new HashSet<>();\r
+ private EventListenerList progressListenerList;\r
+ private ProgressEvent progressEvent = new ProgressEvent(this, 0);\r
+ private Map<Long, BasicDBList> storyRundowns;\r
+ private Map<Long, BasicDBList> storyStoryFolders;\r
+ private Map<Long, Object> currentRundowns;\r
+ private Map<Long, Object> currentFolder;\r
+ private Map<Long, Object> currentStories;\r
+ private String RUNDOWN_COLLECTION;\r
+ private String FOLDER_COLLECTION;\r
+ private String STORY_COLLECTION;\r
+ private boolean includeArchived;\r
+ private Calendar zeroDate = CalendarUtils.createCalendar(2017, 11, 4);\r
+\r
+ public OctopusDataMinerStable() {\r
+ db = NoSQLUtils.getNoSQLDB();\r
+\r
+ String apiAddress = System.getProperty("jobengine.octopus.api.address");\r
+ apiUser = System.getProperty("jobengine.octopus.api.user");\r
+ apiPwd = System.getProperty("jobengine.octopus.api.password");\r
+\r
+ // ResteasyClient client = new ResteasyClientBuilder().register(JacksonJsonProvider.class).build();\r
+ ResteasyClient client = new ResteasyClientBuilder().build();\r
+ webTarget = client.target(apiAddress).register(new BasicAuthentication(apiUser, apiPwd));\r
+ }\r
+\r
+ public void addProgressListener(IProgressEventListener listener) {\r
+ if (progressListenerList == null)\r
+ progressListenerList = new EventListenerList();\r
+ progressListenerList.add(IProgressEventListener.class, listener);\r
+ }\r
+\r
+ private Map<Long, BasicDBList> buildFolderReferences(BasicDBList storyFolders) {\r
+ Map<Long, BasicDBList> result = new HashMap<>();\r
+ List<BasicDBObject> storyFolderList = NoSQLUtils.asList(storyFolders);\r
+ for (BasicDBObject storyFolder : storyFolderList) {\r
+ if (storyFolder == null || !storyFolder.containsKey(IOctopusAPI.ID))\r
+ continue;\r
+ BasicDBObject storyFolderWithStoryIds = queryStoryFolder(storyFolder, FIELDS_STORYFOLDER_STORYIDS);\r
+ long storyFolderId = storyFolderWithStoryIds.getLong(IOctopusAPI.ID);\r
+ List<BasicDBObject> stories = NoSQLUtils.asList(storyFolderWithStoryIds, IOctopusAPI.STORIES);\r
+ if (stories == null)\r
+ continue;\r
+ if (!folderIDs.contains(storyFolderId))\r
+ folderIDs.add(storyFolderId);\r
+ long position = 1;\r
+ for (BasicDBObject story : stories) {\r
+ long storyId = story.getLong(IOctopusAPI.ID);\r
+\r
+ BasicDBList references = result.get(storyId);\r
+ if (references == null) {\r
+ references = new BasicDBList();\r
+ result.put(storyId, references);\r
+ }\r
+ references.add(new BasicDBObject(IOctopusAPI.ID, storyFolderId).append(IOctopusAPI.POSITION, position++));\r
+ }\r
+ }\r
+ return result;\r
+ }\r
+\r
+ private Map<Long, BasicDBList> buildRundownReferences(BasicDBList rundowns) {\r
+ Map<Long, BasicDBList> result = new HashMap<>();\r
+ List<BasicDBObject> rundownsList = NoSQLUtils.asList(rundowns);\r
+ for (BasicDBObject rundown : rundownsList) {\r
+ if (rundown == null || !rundown.containsKey(IOctopusAPI.ID))\r
+ continue;\r
+ BasicDBObject rundownWithStoryids = queryRundown(rundown, FIELDS_RUNDOWN_STORYIDS);\r
+ long rundownId = rundownWithStoryids.getLong(IOctopusAPI.ID);\r
+ List<BasicDBObject> slugs = NoSQLUtils.asList(rundownWithStoryids, IOctopusAPI.SLUGS);\r
+ if (slugs == null)\r
+ continue;\r
+\r
+ if (!rundownIDs.contains(rundownId))\r
+ rundownIDs.add(rundownId);\r
+\r
+ for (BasicDBObject slug : slugs) {\r
+ if (!slug.containsKey(IOctopusAPI.STORYID))\r
+ continue;\r
+ long storyId = slug.getLong(IOctopusAPI.STORYID);\r
+ BasicDBList references = result.get(storyId);\r
+ if (references == null) {\r
+ references = new BasicDBList();\r
+ result.put(storyId, references);\r
+ }\r
+ long position = slug.getLong(IOctopusAPI.POSITION);\r
+ if (slug.containsKey(IOctopusAPI.POSITION))\r
+ position = slug.getLong(IOctopusAPI.POSITION);\r
+ references.add(new BasicDBObject(IOctopusAPI.ID, rundownId).append(IOctopusAPI.POSITION, position));\r
+ }\r
+ }\r
+ return result;\r
+ }\r
+\r
+ public void clear() {\r
+ db.getCollection(RUNDOWN_COLLECTION).remove();\r
+ db.getCollection(STORY_COLLECTION).remove();\r
+ db.getCollection(FOLDER_COLLECTION).remove();\r
+ db.getCollection(IOctopusAPI.TIME_COLLECTION_NAME).remove();\r
+ }\r
+\r
+ private String concatParentsToStoryFolder(BasicDBObject actual, String name) {\r
+\r
+ String fields = "name,id,parent";\r
+ Response response = query("StoryFolder/" + actual.getLong("id"), fields).get();\r
+ String json = response.readEntity(String.class);\r
+ BasicDBObject resultObject = (BasicDBObject) JSONUtil.jsonToDbObject(json);\r
+ BasicDBObject res = (BasicDBObject) resultObject.get("result");\r
+ BasicDBObject parent = (BasicDBObject) res.get("parent");\r
+ if (parent == null || parent.isEmpty())\r
+ return name;\r
+ String parentName = parent.getString(IOctopusAPI.NAME);\r
+ String actualName = actual.getString(IOctopusAPI.NAME);\r
+ String newName = String.format("%s/%s", parentName, actualName);\r
+ return concatParentsToStoryFolder(parent, newName);\r
+ }\r
+\r
+ private void deleteDiff(Set<Long> oldIDs, Set<Long> newIDs, String collectionName) {\r
+ if (oldIDs == null || oldIDs.size() == 0)\r
+ return;\r
+ if (newIDs != null && newIDs.size() > 0)\r
+ oldIDs.removeAll(newIDs);\r
+ DBCollection collection = db.getCollection(collectionName);\r
+ for (long id : oldIDs) {\r
+ logger.info("Deleting {} from {}", id, collectionName);\r
+ collection.remove(new BasicDBObject(IOctopusAPI.ID, id));\r
+ }\r
+ }\r
+\r
+ private void deleteOrphanRundowns() {\r
+ try {\r
+ DBCollection collection = db.getCollection(RUNDOWN_COLLECTION);\r
+ BasicDBObject query = (BasicDBObject) QueryBuilder.start().put(IOctopusAPI.ID).notIn(storyRundowns.keySet().toArray()).get();\r
+ WriteResult res = collection.remove(query);\r
+ logger.trace(String.format("Deleted orphan rundowns: %d", res.getN()));\r
+ } catch (Exception e) {\r
+ logger.error(e);\r
+ }\r
+ }\r
+\r
+ private void deleteOrphanStories() {\r
+ try {\r
+ DBCollection collection = db.getCollection(STORY_COLLECTION);\r
+ BasicDBObject query = (BasicDBObject) QueryBuilder.start().put("id").notIn(storyIDs.toArray()).get();\r
+ WriteResult res = collection.remove(query);\r
+ logger.trace(String.format("Deleted orphan stories: %d", res.getN()));\r
+ } catch (Exception e) {\r
+ logger.error(e);\r
+ }\r
+ }\r
+\r
+ private void deleteOrphanStoryFolders() {\r
+ try {\r
+ DBCollection collection = db.getCollection(RUNDOWN_COLLECTION);\r
+ BasicDBObject query = (BasicDBObject) QueryBuilder.start().put(IOctopusAPI.ID).notIn(storyStoryFolders.keySet().toArray()).get();\r
+ WriteResult res = collection.remove(query);\r
+ logger.trace(String.format("Deleted orphan rundowns: %d", res.getN()));\r
+ } catch (Exception e) {\r
+ logger.error(e);\r
+ }\r
+ }\r
+\r
+ private void ensureIndexes() {\r
+ DBCollection collection = db.getCollection(FOLDER_COLLECTION);\r
+ if (collection.count() == 0)\r
+ collection.ensureIndex(IOctopusAPI.ID);\r
+ collection = db.getCollection(RUNDOWN_COLLECTION);\r
+ if (collection.count() == 0) {\r
+ collection.ensureIndex(IOctopusAPI.ID);\r
+ collection.ensureIndex(IOctopusAPI.SCHEDULED_START);\r
+ }\r
+ collection = db.getCollection(STORY_COLLECTION);\r
+ if (collection.count() == 0)\r
+ collection.ensureIndex(IOctopusAPI.ID);\r
+ }\r
+\r
+ public void execute(boolean includeArchived) throws Exception {\r
+ this.includeArchived = includeArchived;\r
+ logger.trace(STARTING);\r
+\r
+ // String MAIN_RUNDOWN_COLLECTION = IOctopusAPI.RUNDOWN_COLLECTION;\r
+ // String MAIN_FOLDER_COLLECTION = IOctopusAPI.FOLDER_COLLECTION;\r
+ // String MAIN_STORY_COLLECTION = IOctopusAPI.STORY_COLLECTION;\r
+\r
+ // RUNDOWN_COLLECTION = MAIN_RUNDOWN_COLLECTION + _TMP;\r
+ // FOLDER_COLLECTION = MAIN_FOLDER_COLLECTION + _TMP;\r
+ // STORY_COLLECTION = MAIN_STORY_COLLECTION + _TMP;\r
+ //\r
+ // try {\r
+ // db.getCollection(RUNDOWN_COLLECTION).drop();\r
+ // db.getCollection(FOLDER_COLLECTION).drop();\r
+ // db.getCollection(STORY_COLLECTION).drop();\r
+ // } catch (Exception e) {\r
+ // logger.catching(e);\r
+ // throw e;\r
+ // }\r
+\r
+ RUNDOWN_COLLECTION = IOctopusAPI.RUNDOWN_COLLECTION;\r
+ FOLDER_COLLECTION = IOctopusAPI.FOLDER_COLLECTION;\r
+ STORY_COLLECTION = IOctopusAPI.STORY_COLLECTION;\r
+\r
+ //current = korábbi szinkronizálás\r
+ currentRundowns = getCurrentIDs(RUNDOWN_COLLECTION);\r
+ currentFolder = getCurrentIDs(FOLDER_COLLECTION);\r
+ currentStories = getCurrentIDs(STORY_COLLECTION);\r
+\r
+ BasicDBList rundowns = null;\r
+ BasicDBList storyFolders = null;\r
+ try {\r
+ rundowns = queryBuildRefRundowns();\r
+ storyFolders = queryBuildRefFolders();\r
+ } catch (Exception e) {\r
+ logger.catching(e);\r
+ throw e;\r
+ }\r
+\r
+ processRundowns(rundowns);\r
+ processStoryFolders(storyFolders);\r
+\r
+ deleteDiff(currentRundowns.keySet(), rundownIDs, RUNDOWN_COLLECTION);\r
+ deleteDiff(currentFolder.keySet(), folderIDs, FOLDER_COLLECTION);\r
+ deleteDiff(currentStories.keySet(), storyIDs, STORY_COLLECTION);\r
+ // deleteOrphanRundowns();\r
+ // deleteOrphanStoryFolders();\r
+ // deleteOrphanStories();\r
+\r
+ //a sorrend fontos !\r
+ // updateDiff(MAIN_STORY_COLLECTION, STORY_COLLECTION, IOctopusAPI.ID);\r
+ // updateDiff(MAIN_RUNDOWN_COLLECTION, RUNDOWN_COLLECTION, IOctopusAPI.ID);\r
+ // updateDiff(MAIN_FOLDER_COLLECTION, FOLDER_COLLECTION, IOctopusAPI.ID);\r
+ //\r
+ // updateDeleteDiff(MAIN_RUNDOWN_COLLECTION, RUNDOWN_COLLECTION, IOctopusAPI.ID);\r
+ // updateDeleteDiff(MAIN_FOLDER_COLLECTION, FOLDER_COLLECTION, IOctopusAPI.ID);\r
+ // updateDeleteDiff(MAIN_STORY_COLLECTION, STORY_COLLECTION, IOctopusAPI.ID);\r
+\r
+ // try {\r
+ // db.getCollection(RUNDOWN_COLLECTION).rename(MAIN_RUNDOWN_COLLECTION, true);\r
+ // db.getCollection(FOLDER_COLLECTION).rename(MAIN_FOLDER_COLLECTION, true);\r
+ // db.getCollection(STORY_COLLECTION).rename(MAIN_STORY_COLLECTION, true);\r
+ // } catch (Exception e) {\r
+ // logger.catching(e);\r
+ // throw e;\r
+ // }\r
+\r
+ // logger.info("Activate");\r
+\r
+ // try {\r
+ // long ts = new Date().getTime();\r
+ // db.getCollection(RUNDOWN_COLLECTION).exportFile(String.format("/opt/mediacube-test/log/%s-%s.js", RUNDOWN_COLLECTION, ts));\r
+ // db.getCollection(FOLDER_COLLECTION).exportFile(String.format("/opt/mediacube-test/log/%s-%s.js", FOLDER_COLLECTION, ts));\r
+ // db.getCollection(STORY_COLLECTION).exportFile(String.format("/opt/mediacube-test/log/%s-%s.js", STORY_COLLECTION, ts));\r
+ // } catch (Exception e) {\r
+ // logger.catching(e);\r
+ // throw e;\r
+ // }\r
+\r
+ logger.trace(FINISHED);\r
+ //throw new Exception("Mérés");\r
+ }\r
+\r
+ private String extractContent(BasicDBObject content) {\r
+ String scriptContent = "";\r
+ if (!content.containsKey(IOctopusAPI.TYPE))\r
+ return scriptContent;\r
+ String type = content.getString(IOctopusAPI.TYPE);\r
+ switch (type) {\r
+ case IOctopusAPI.TEXT: {\r
+ if (content.containsKey(IOctopusAPI.TEXT)) {\r
+ String text = content.getString(IOctopusAPI.TEXT);\r
+ if (text != null)\r
+ scriptContent += String.format("%s%s", text.replaceAll(SIMPLE_LINEFEED, LINEFEED), LINEFEED);\r
+ }\r
+ break;\r
+ }\r
+ case IOctopusAPI.MOS: {\r
+ BasicDBObject mosObject = NoSQLUtils.asDBObject(content, IOctopusAPI.OBJECT);\r
+ if (mosObject != null && !mosObject.isEmpty()) {\r
+ if (mosObject.containsKey(IOctopusAPI.OBJ_ID)) {\r
+ String objID = mosObject.getString(IOctopusAPI.OBJ_ID);\r
+ scriptContent += String.format("%s %s%s", MOSOBJECT, objID, LINEFEED);\r
+ }\r
+ }\r
+ break;\r
+ }\r
+ default: {\r
+ if (content.containsKey(IOctopusAPI.CONTENT)) {\r
+ List<BasicDBObject> innerContents = NoSQLUtils.asList(content, IOctopusAPI.CONTENT);\r
+ if (innerContents != null) {\r
+ for (BasicDBObject actualInnerContent : innerContents) {\r
+ if (actualInnerContent != null && actualInnerContent.isEmpty())\r
+ scriptContent += extractContent(actualInnerContent);\r
+ }\r
+ }\r
+ }\r
+ break;\r
+ }\r
+ }\r
+ return scriptContent;\r
+ }\r
+\r
+ private String extractCustomColumnValue(String columnName, BasicDBObject story) {\r
+ List<BasicDBObject> customColumns = NoSQLUtils.asList(story, IOctopusAPI.CUSTOM_COLUMNS);\r
+ if (customColumns == null)\r
+ return null;\r
+ String result = null;\r
+ for (BasicDBObject customColumn : customColumns) {\r
+ if (!customColumn.containsKey(IOctopusAPI.LABEL))\r
+ continue;\r
+ String currentName = customColumn.getString(IOctopusAPI.LABEL);\r
+ if (currentName == null)\r
+ continue;\r
+ if (!currentName.toLowerCase().equals(columnName.toLowerCase()))\r
+ continue;\r
+ if (!customColumn.containsKey(IOctopusAPI.VALUE))\r
+ continue;\r
+ result = customColumn.getString(IOctopusAPI.VALUE);\r
+ break;\r
+ }\r
+ return result;\r
+ }\r
+\r
+ private BasicDBList extractRelevantMOSObjects(BasicDBObject story) {\r
+ List<BasicDBObject> mosObjects = NoSQLUtils.asList(story, IOctopusAPI.MOS_OBJECTS);\r
+ if (mosObjects == null)\r
+ return null;\r
+ BasicDBList result = null;\r
+ for (BasicDBObject mosObject : mosObjects) {\r
+ if (!mosObject.containsKey(IOctopusAPI.MOS_ID))\r
+ continue;\r
+ String mosId = mosObject.getString(IOctopusAPI.MOS_ID);\r
+ if (!IOctopusAPI.NEXIO_MOS.equals(mosId))\r
+ continue;\r
+ if (result == null)\r
+ result = new BasicDBList();\r
+ result.add(mosObject);\r
+ }\r
+ return result;\r
+ }\r
+\r
+ private String extractScriptContent(BasicDBObject story) {\r
+ BasicDBObject script = NoSQLUtils.asDBObject(story, IOctopusAPI.SCRIPT);\r
+ if (script == null || script.isEmpty())\r
+ return null;\r
+\r
+ List<BasicDBObject> body = NoSQLUtils.asList(script, IOctopusAPI.BODY);\r
+ if (body == null || body.size() == 0)\r
+ return null;\r
+\r
+ StringBuilder sb = new StringBuilder();\r
+\r
+ for (BasicDBObject bodyItem : body) {\r
+ if (bodyItem.containsKey(IOctopusAPI.LABEL)) {\r
+ sb.append(bodyItem.getString(IOctopusAPI.LABEL));\r
+ sb.append(LINEFEED);\r
+ }\r
+ List<BasicDBObject> contents = NoSQLUtils.asList(bodyItem, IOctopusAPI.CONTENT);\r
+ if (contents == null)\r
+ continue;\r
+ for (BasicDBObject content : contents) {\r
+ sb.append(extractContent(content));\r
+ }\r
+ }\r
+ return sb.length() == 0 ? null : sb.toString();\r
+ }\r
+\r
+ private void fireProgressEvent(ProgressEvent evt) {\r
+ logger.debug("Progress changed to " + evt.getProgress() + "%");\r
+ if (progressListenerList == null)\r
+ return;\r
+ Object[] listeners = progressListenerList.getListenerList();\r
+ for (int i = 0; i < listeners.length; i += 2) {\r
+ if (listeners[i] == IProgressEventListener.class)\r
+ ((IProgressEventListener) listeners[i + 1]).progressChanged(evt);\r
+ }\r
+ }\r
+\r
+ public Map<Long, Object> getCurrentIDs(String name) {\r
+ Map<Long, Object> result = new HashMap<>();\r
+\r
+ DBCursor cursor = db.getCollection(name).find(new BasicDBObject(), new BasicDBObject(IOctopusAPI.ID, 1));\r
+ if (cursor.hasNext()) {\r
+ List<BasicDBObject> objects = ListUtils.cast(cursor.toArray());\r
+ for (BasicDBObject obj : objects) {\r
+ if (obj == null)\r
+ continue;\r
+ long id = NoSQLUtils.asLong(obj, IOctopusAPI.ID);\r
+ if (id == 0)\r
+ continue;\r
+ Object _id = obj.getID();\r
+ if (_id == null)\r
+ continue;\r
+ result.put(id, _id);\r
+ }\r
+ }\r
+ return result;\r
+ }\r
+\r
+ private void processRundowns(BasicDBList rundowns) throws Exception {\r
+ if (rundowns == null || rundowns.size() == 0) {\r
+ progressEvent.setProgress(50);\r
+ fireProgressEvent(progressEvent);\r
+ } else {\r
+ logger.info("Process rundowns");\r
+ try {\r
+ storeRundowns(rundowns, null);\r
+ } catch (Exception e) {\r
+ logger.catching(e);\r
+ throw e;\r
+ }\r
+ }\r
+ }\r
+\r
+ private void processStoryFolders(BasicDBList storyFolders) throws Exception {\r
+ if (storyFolders == null || storyFolders.size() == 0) {\r
+ progressEvent.setProgress(100);\r
+ fireProgressEvent(progressEvent);\r
+ } else {\r
+ logger.info("Process story folders");\r
+ try {\r
+ storeStoryFolders(storyFolders, null);\r
+ } catch (Exception e) {\r
+ logger.catching(e);\r
+ throw e;\r
+ }\r
+ }\r
+ }\r
+\r
+ private Builder query(String path, String fields) {\r
+ ResteasyWebTarget target = webTarget.path(path).queryParam(FIELDS, fields);\r
+ Builder result = target.request().header(OCTOPUS_DEVICE_ID, apiUser).header(OCTOPUS_DEVICE_NAME, apiPwd);\r
+ return result;\r
+ }\r
+\r
+ private BasicDBList queryBuildRefFolders() {\r
+ BasicDBList storyFolders;\r
+ logger.info("Fetch story folders");\r
+ storyFolders = queryStoryFolders();\r
+ logger.info("Fetch folder story references");\r
+ storyStoryFolders = buildFolderReferences(storyFolders);\r
+ return storyFolders;\r
+ }\r
+\r
+ private BasicDBList queryBuildRefRundowns() {\r
+ BasicDBList rundowns;\r
+ logger.info("Fetch rundowns");\r
+ rundowns = queryRundowns();\r
+ logger.info("Fetch rundown story references");\r
+ storyRundowns = buildRundownReferences(rundowns);\r
+ return rundowns;\r
+ }\r
+\r
+ private BasicDBObject queryRundown(BasicDBObject rundown, String fields) {\r
+ logger.trace(ENTER);\r
+ BasicDBObject result = null;\r
+ long id = NoSQLUtils.asLong(rundown, IOctopusAPI.ID);\r
+ Response response = query(String.format("%s/%d", RUNDOWN, id), fields).get();\r
+ String json = response.readEntity(String.class);\r
+ BasicDBObject resultObject = (BasicDBObject) JSONUtil.jsonToDbObject(json);\r
+ if (resultObject == null)\r
+ logger.error("Rundown {} {} is not available", id, rundown.getString(IOctopusAPI.NAME));\r
+ else\r
+ result = NoSQLUtils.asDBObject(resultObject, RESULT);\r
+ logger.trace(EXIT);\r
+ return result;\r
+ }\r
+\r
+ private BasicDBList queryRundowns() {\r
+ logger.trace(ENTER);\r
+ BasicDBList result = null;\r
+ Builder query = query(RUNDOWN, FIELDS_RUNDOWN);\r
+ Response response = null;\r
+ if (includeArchived) {\r
+ response = query.post(Entity.entity(new BasicDBObject(FILTER, new BasicDBObject(ARCHIVED, true)).toString(), MediaType.APPLICATION_JSON));\r
+ } else\r
+ response = query.get();\r
+\r
+ String json = response.readEntity(String.class);\r
+ BasicDBObject resultObject = (BasicDBObject) JSONUtil.jsonToDbObject(json);\r
+ if (resultObject != null)\r
+ result = NoSQLUtils.asDBList(resultObject, RESULT);\r
+ logger.trace(EXIT);\r
+ return result;\r
+ }\r
+\r
+ private BasicDBObject queryStoryFolder(BasicDBObject storyFolder, String fields) {\r
+ logger.trace(ENTER);\r
+ BasicDBObject result = null;\r
+ long id = NoSQLUtils.asLong(storyFolder, IOctopusAPI.ID);\r
+ Response response = query(String.format("%s/%d", STORY_FOLDER, id), fields).get();\r
+ String json = response.readEntity(String.class);\r
+ BasicDBObject resultObject = (BasicDBObject) JSONUtil.jsonToDbObject(json);\r
+ if (resultObject == null)\r
+ logger.error("StoryFolder {} {} is not available", id, storyFolder.getString(IOctopusAPI.NAME));\r
+ else\r
+ result = NoSQLUtils.asDBObject(resultObject, RESULT);\r
+ logger.trace(EXIT);\r
+ return result;\r
+ }\r
+\r
+ private BasicDBList queryStoryFolders() {\r
+ logger.trace(ENTER);\r
+ BasicDBList result = null;\r
+ Response response = query(STORY_FOLDER, FIELDS_STORYFOLDER).get();\r
+ String json = response.readEntity(String.class);\r
+ BasicDBObject resultObject = (BasicDBObject) JSONUtil.jsonToDbObject(json);\r
+ if (resultObject != null)\r
+ result = NoSQLUtils.asDBList(resultObject, RESULT);\r
+\r
+ // /* teszt */\r
+ // List<BasicDBObject> list = NoSQLUtils.asList(result);\r
+ // for (BasicDBObject actual : list) {\r
+ // String fullName = concatParentsToStoryFolder(actual, actual.getString(IOctopusAPI.NAME));\r
+ // //logger.info("Checking StoryFolder {}", fullName);\r
+ // actual.remove(IOctopusAPI.NAME);\r
+ // actual.append(IOctopusAPI.NAME, fullName);\r
+ // }\r
+ //\r
+ logger.trace(EXIT);\r
+ return result;\r
+ }\r
+\r
+ public void removeProgressListener(IProgressEventListener listener) {\r
+ progressListenerList.remove(IProgressEventListener.class, listener);\r
+ }\r
+\r
+ void setObjectID(Map<Long, Object> current, long id, BasicDBObject objectToSave) {\r
+ Object _id = current.get(id);\r
+ if (_id == null)\r
+ return;\r
+ objectToSave.put(IOctopusAPI._ID, _id);\r
+ }\r
+\r
+ private void storeRundown(BasicDBObject rundown) {\r
+ logger.trace(ENTER);\r
+ String name = rundown.containsKey(IOctopusAPI.NAME) ? rundown.getString(IOctopusAPI.NAME) : null;\r
+ logger.debug("Storing rundown {} {}", name, rundown.get(IOctopusAPI.SCHEDULED_START));\r
+ BasicDBObject rundownWithStories = queryRundown(rundown, FIELDS_RUNDOWN_STORIES);\r
+ if (rundownWithStories != null) {\r
+ Date scheduledStart = toDate(rundown, IOctopusAPI.SCHEDULED_START);\r
+ if (scheduledStart != null && scheduledStart.after(zeroDate.getTime())) {\r
+ BasicDBList stories = NoSQLUtils.asDBList(rundownWithStories, IOctopusAPI.SLUGS);\r
+ if (stories != null)\r
+ storeRundownStories(stories);\r
+ rundown.put(IOctopusAPI.SCHEDULED_START, toDate(rundown, IOctopusAPI.SCHEDULED_START));\r
+ rundown.put(IOctopusAPI.MODIFIED, toDate(rundown, IOctopusAPI.MODIFIED));\r
+ DBCollection collection = db.getCollection(RUNDOWN_COLLECTION);\r
+ long rundownID = NoSQLUtils.asLong(rundown, IOctopusAPI.ID);\r
+ setObjectID(currentRundowns, rundownID, rundown);\r
+ collection.save(rundown);\r
+ }\r
+ }\r
+ logger.trace(EXIT);\r
+ }\r
+\r
+ private void storeRundowns(BasicDBList rundowns, Date lastUpdateTime) {\r
+ logger.trace(ENTER);\r
+ List<BasicDBObject> rundownsList = NoSQLUtils.asList(rundowns);\r
+ int idx = 1;\r
+ for (BasicDBObject rundown : rundownsList) {\r
+ //logger.info(CHECKING_RUNDOWN, rundown.getLong(IOctopusAPI.ID), rundownsList.size(), idx);\r
+ storeRundown(rundown);\r
+ int progress = idx * 50 / rundownsList.size();\r
+ if (progress - progressEvent.getProgress() > 0) {\r
+ progressEvent.setProgress(progress);\r
+ fireProgressEvent(progressEvent);\r
+ }\r
+\r
+ idx++;\r
+ }\r
+ logger.trace(EXIT);\r
+ }\r
+\r
+ private void storeRundownStories(BasicDBList slugs) {\r
+ logger.trace(ENTER);\r
+ List<BasicDBObject> slugsList = NoSQLUtils.asList(slugs);\r
+ for (BasicDBObject slug : slugsList) {\r
+ if (slug.containsKey(IOctopusAPI.STORY))\r
+ storeStory((BasicDBObject) slug.get(IOctopusAPI.STORY));\r
+ }\r
+ logger.trace(EXIT);\r
+ }\r
+\r
+ private void storeStory(BasicDBObject story) {\r
+ logger.trace(ENTER);\r
+ if (!story.containsKey(IOctopusAPI.ID)) {\r
+ logger.error("Missing id in story {}", story.toPrettyString(null));\r
+ return;\r
+ }\r
+ long storyID = story.getLong(IOctopusAPI.ID);\r
+ boolean isContains = storyIDs.contains(storyID);\r
+ if (isContains)\r
+ return;\r
+ storyIDs.add(storyID);\r
+ story.put(IOctopusAPI.MODIFIED, toDate(story, IOctopusAPI.MODIFIED));\r
+ BasicDBList rundownRef = storyRundowns.get(storyID);\r
+ BasicDBList storyFolderRef = storyStoryFolders.get(storyID);\r
+ BasicDBList modifiedMOS = extractRelevantMOSObjects(story);\r
+\r
+ DBCollection collection = db.getCollection(STORY_COLLECTION);\r
+\r
+ if (rundownRef != null)\r
+ story.put(IOctopusAPI.REF_RUNDOWN, rundownRef);\r
+ if (storyFolderRef != null)\r
+ story.put(IOctopusAPI.REF_STORYFOLDER, storyFolderRef);\r
+ String scriptContent = extractScriptContent(story);\r
+ story.put(IOctopusAPI.SCRIPT_CONTENT, scriptContent);\r
+ if (modifiedMOS == null || modifiedMOS.isEmpty()) {\r
+ if (story.containsKey(IOctopusAPI.MOS_OBJECTS))\r
+ story.remove(IOctopusAPI.MOS_OBJECTS);\r
+ } else\r
+ story.put(IOctopusAPI.MOS_OBJECTS, modifiedMOS);\r
+\r
+ String parentStoryId = extractCustomColumnValue(IOctopusAPI.PARENT_STORY_ID, story);\r
+ if (parentStoryId == null) {\r
+ logger.warn("Missing {} in story", IOctopusAPI.PARENT_STORY_ID, story.toPrettyString(null));\r
+ story.append(IOctopusAPI.PARENT_STORY_ID, storyID);\r
+ } else\r
+ story.append(IOctopusAPI.PARENT_STORY_ID, parentStoryId);\r
+ logger.debug(SAVING_STORY_ID, storyID);\r
+ setObjectID(currentStories, storyID, story);\r
+ collection.save(story);\r
+ logger.trace(EXIT);\r
+ }\r
+\r
+ private void storeStoryFolder(BasicDBObject storyFolder, Date lastUpdateTime) {\r
+ logger.trace(ENTER);\r
+ BasicDBObject storyFoldersWithStories = queryStoryFolder(storyFolder, FIELDS_STORYFOLDER_STORIES);\r
+ if (storyFoldersWithStories != null) {\r
+ BasicDBList stories = NoSQLUtils.asDBList(storyFoldersWithStories, IOctopusAPI.STORIES);\r
+ if (stories != null)\r
+ storeStoryFolderStories(stories);\r
+ storyFolder.put(IOctopusAPI.MODIFIED, toDate(storyFolder, IOctopusAPI.MODIFIED));\r
+ DBCollection collection = db.getCollection(FOLDER_COLLECTION);\r
+ String name = storyFolder.getString(IOctopusAPI.NAME);\r
+ logger.debug("Storing story folder {}", name);\r
+ long folderID = NoSQLUtils.asLong(storyFolder, IOctopusAPI.ID);\r
+ setObjectID(currentFolder, folderID, storyFolder);\r
+ collection.save(storyFolder);\r
+ }\r
+ logger.trace(EXIT);\r
+ }\r
+\r
+ private void storeStoryFolders(BasicDBList storyFolders, Date lastUpdateTime) {\r
+ logger.trace(ENTER);\r
+ List<BasicDBObject> storyFolderList = NoSQLUtils.asList(storyFolders);\r
+ int idx = 1;\r
+ for (BasicDBObject storyFolder : storyFolderList) {\r
+ logger.debug(String.format(CHECKING_STORY_FOLDER, storyFolder.getLong(IOctopusAPI.ID), storyFolderList.size(), idx));\r
+ storeStoryFolder(storyFolder, lastUpdateTime);\r
+ int progress = 50 + (idx * 50 / storyFolderList.size());\r
+ if (progress - progressEvent.getProgress() > 0) {\r
+ progressEvent.setProgress(progress);\r
+ fireProgressEvent(progressEvent);\r
+ }\r
+ idx++;\r
+ }\r
+ logger.trace(EXIT);\r
+ }\r
+\r
+ private void storeStoryFolderStories(BasicDBList stories) {\r
+ logger.trace(ENTER);\r
+ List<BasicDBObject> list = NoSQLUtils.asList(stories);\r
+ for (BasicDBObject story : list)\r
+ storeStory(story);\r
+ logger.trace(EXIT);\r
+ }\r
+\r
+ private Date toDate(BasicDBObject obj, String name) {\r
+ Date result = null;\r
+ if (obj.containsKey(name)) {\r
+ String dt = obj.getString(name);\r
+ if (dt != null) {\r
+ // create jodatime from date\r
+ DateTime jdt = new DateTime(dt);\r
+ result = jdt.toDate();\r
+ }\r
+ }\r
+ return result;\r
+ }\r
+\r
+ private void updateDeleteDiff(String oldCollectionName, String newCollectionName, String idFieldName) {\r
+ DBCollection oldCollection = db.getCollection(oldCollectionName);\r
+ DBCollection newCollection = db.getCollection(newCollectionName);\r
+ DBCursor oldCollectionCursor = oldCollection.find(new BasicDBObject(), new BasicDBObject(idFieldName, 1));\r
+ if (!oldCollectionCursor.hasNext()) {\r
+ logger.error("{} collection is empty", newCollectionName);\r
+ return;\r
+ }\r
+ List<BasicDBObject> oldItems = ListUtils.cast(oldCollectionCursor.toArray());\r
+\r
+ DBCursor newCollectionCursor = newCollection.find();\r
+ ConcurrentHashMap<Long, BasicDBObject> newItems = null;\r
+ if (newCollectionCursor.hasNext()) {\r
+ List<BasicDBObject> newList = ListUtils.cast(newCollectionCursor.toArray());\r
+ newItems = ListUtils.map(newList, item -> item.getLong(IOctopusAPI.ID));\r
+ }\r
+ if (newItems == null)\r
+ newItems = new ConcurrentHashMap<>();\r
+\r
+ for (BasicDBObject oldItem : oldItems) {\r
+ if (oldItem == null) {\r
+ logger.error("Item is null");\r
+ continue;\r
+ }\r
+ if (!oldItem.containsKey(idFieldName)) {\r
+ logger.error("{} is null", idFieldName);\r
+ continue;\r
+ }\r
+ long id = oldItem.getLong(idFieldName);\r
+ BasicDBObject newItem = newItems.get(id);\r
+ if (newItem == null) {\r
+ //remove\r
+ logger.info("Deleting {}", oldItem.toPrettyString(null));\r
+ oldCollection.remove(new BasicDBObject(idFieldName, id));\r
+ }\r
+ }\r
+ }\r
+\r
+ private void updateDiff(String oldCollectionName, String newCollectionName, String idFieldName) {\r
+ DBCollection oldCollection = db.getCollection(oldCollectionName);\r
+ DBCollection newCollection = db.getCollection(newCollectionName);\r
+ DBCursor newCollectionCursor = newCollection.find();\r
+ if (!newCollectionCursor.hasNext()) {\r
+ logger.error("{} collection is empty", newCollectionName);\r
+ return;\r
+ }\r
+\r
+ List<BasicDBObject> newItems = ListUtils.cast(newCollectionCursor.toArray());\r
+\r
+ DBCursor oldCollectionCursor = oldCollection.find();\r
+ ConcurrentHashMap<Long, BasicDBObject> oldItems = null;\r
+ if (oldCollectionCursor.hasNext()) {\r
+ List<BasicDBObject> oldList = ListUtils.cast(oldCollectionCursor.toArray());\r
+ oldItems = ListUtils.map(oldList, item -> item.getLong(IOctopusAPI.ID));\r
+ }\r
+ if (oldItems == null)\r
+ oldItems = new ConcurrentHashMap<>();\r
+ for (BasicDBObject newItem : newItems) {\r
+ if (newItem == null) {\r
+ logger.error("Item is null");\r
+ continue;\r
+ }\r
+ if (!newItem.containsKey(idFieldName)) {\r
+ logger.error("{} is null", idFieldName);\r
+ continue;\r
+ }\r
+ long id = newItem.getLong(idFieldName);\r
+ BasicDBObject oldItem = oldItems.get(id);\r
+ boolean save = false;\r
+ if (oldItem == null) {\r
+ //save\r
+ save = true;\r
+ } else {\r
+ //compare\r
+ Object oldDbId = oldItem.getID();\r
+ newItem.remove(IOctopusAPI._ID);\r
+ oldItem.remove(IOctopusAPI._ID);\r
+\r
+ if (!newItem.equals(oldItem)) {\r
+ //save\r
+ newItem.put(IOctopusAPI._ID, oldDbId);\r
+ save = true;\r
+ }\r
+ }\r
+\r
+ if (save) {\r
+ logger.info("Saving to {} item {}", oldCollectionName, id);\r
+ oldCollection.save(newItem);\r
+ }\r
+ }\r
+ }\r
+\r
+}\r
--- /dev/null
+package user.commons;\r
+\r
+@SuppressWarnings("serial")\r
+public class MediaCubeFinishMarker extends MediaCubeMarker {\r
+\r
+ public MediaCubeFinishMarker(MediaCubeMarker parent) {\r
+ setSessionID(parent.getSessionID());\r
+ }\r
+\r
+ public MediaCubeFinishMarker(MediaCubeMarker parent, String to) {\r
+ super(to);\r
+ setSessionID(parent.getSessionID());\r
+ }\r
+}\r
public class MediaCubeMarker extends Log4jMarker {\r
private static final String MEDIACUBE = "MEDIACUBE";\r
private String to;\r
+ private String sessionID;\r
+\r
+ public MediaCubeMarker() {\r
+ super(MEDIACUBE);\r
+ }\r
\r
public MediaCubeMarker(String to) {\r
super(MEDIACUBE);\r
this.to = to;\r
}\r
\r
+ public String getSessionID() {\r
+ return sessionID;\r
+ }\r
+\r
public String getTo() {\r
return to;\r
}\r
\r
+ public void setSessionID(String sessionID) {\r
+ this.sessionID = sessionID;\r
+ }\r
+\r
public void setTo(String to) {\r
this.to = to;\r
}\r
\r
+ // @Override\r
+ // public boolean equals(Object o) {\r
+ // if (o == null || !o.getClass().equals(MediaCubeMarker.class))\r
+ // return false;\r
+ //\r
+ //\r
+ // return true;\r
+ // }\r
+\r
}\r
static final String MOS_OBJECTS = "mosObjects";\r
static final String NAME = "name";\r
static final String LABEL = "label";\r
+ static final String CLIP = "clip";\r
static final String OBJ_ID = "objId";\r
static final String OBJECT = "object";\r
static final String MOS = "mos";\r
public List<DBObject> getStories() {\r
List<DBObject> result = null;\r
DBCollection collection = db.getCollection(STORY_COLLECTION);\r
- DBCursor find = collection.find();\r
+ DBCursor find = collection.find().limit(1000);\r
if (find.hasNext())\r
result = find.toArray();\r
return result;\r
eclipse.preferences.version=1\r
encoding//pages/login.zul=UTF-8\r
+encoding//resources/i3-label_hu.properties=UTF-8\r
encoding//src/user/jobengine/server/JobRuntime.java=UTF-8\r
<?xml version="1.0" encoding="UTF-8"?>\r
<?init class="user.jobengine.zk.util.AuthInitiator"?>\r
-<zk xmlns:w="http://www.zkoss.org/2005/zk/client" >\r
- <style src="/css/archivum.css" />\r
+<zk xmlns:w="http://www.zkoss.org/2005/zk/client">\r
+ <style src="/css/archivum.css" />\r
<zscript><
https://git.useribm.hu / mediacube.git / commitdiff