From: Vásáry Dániel Date: Wed, 6 Dec 2017 15:51:53 +0000 (+0000) Subject: git-tfs-id: [http://tfs.userrendszerhaz.hu:8080/tfs/DefaultCollection]$/MediaCube... X-Git-Url: http://git.useribm.hu/?a=commitdiff_plain;h=ec5edd00f96830403996482dc168bcc2b5c0da96;p=mediacube.git git-tfs-id: [tfs.userrendszerhaz.hu:8080/tfs/DefaultCollection]$/MediaCube;C30792 --- diff --git a/client/Maestro/Configuration/ACTIVE-configuration-vezerlo.json b/client/Maestro/Configuration/-ACTIVE-configuration-vezerlo.json similarity index 100% rename from client/Maestro/Configuration/ACTIVE-configuration-vezerlo.json rename to client/Maestro/Configuration/-ACTIVE-configuration-vezerlo.json diff --git a/client/Maestro/Configuration/-configuration-studio.json b/client/Maestro/Configuration/configuration-studio.json similarity index 100% rename from client/Maestro/Configuration/-configuration-studio.json rename to client/Maestro/Configuration/configuration-studio.json diff --git a/client/Maestro/Installer/installforge-installer-project.ifp b/client/Maestro/Installer/installforge-installer-project.ifp index e88bb6a8..abebe935 100644 Binary files a/client/Maestro/Installer/installforge-installer-project.ifp and b/client/Maestro/Installer/installforge-installer-project.ifp differ diff --git a/client/Maestro/Maestro.csproj b/client/Maestro/Maestro.csproj index 5cc53d91..c21783fc 100644 --- a/client/Maestro/Maestro.csproj +++ b/client/Maestro/Maestro.csproj @@ -252,7 +252,7 @@ Always - + Always @@ -324,7 +324,7 @@ Always - + Always diff --git a/client/Maestro/MaestroForm.Metadata.cs b/client/Maestro/MaestroForm.Metadata.cs index 9ad1a3a3..23c11a6d 100644 --- a/client/Maestro/MaestroForm.Metadata.cs +++ b/client/Maestro/MaestroForm.Metadata.cs @@ -170,12 +170,16 @@ namespace Maestro { if (movieSegments == null) movieSegments = new BindingList(); bool readOnly = true; - if (ArchiveMetadata == null) - MsgBox.Error("A metaadatok nem elérhetők, a szegmens definiálás nem engedélyezett."); - else { + if (ArchiveMetadata == null) { + ArchiveMetadata = GetArchiveMetadata(); + if (ArchiveMetadata == null) + MsgBox.Error("A metaadatok nem elérhetők, a szegmens definiálás nem engedélyezett."); + else + readOnly = ArchiveMetadata.ok; + } else readOnly = ArchiveMetadata.ok; + if (readOnly && ArchiveMetadata != null) MsgBox.Warning("Az anyag már el van fogadva, a szegmens módosítás nem kerül mentésre. A szegmensek módosításához vissza kell vonni az anyag elfogadását az adástervező rendszerben."); - } OpenFile(SelectedSource.FileInfo, true, readOnly); UpdateDefineSegmentsStatus(); } diff --git a/client/Maestro/Properties/AssemblyInfo.cs b/client/Maestro/Properties/AssemblyInfo.cs index d6d602d5..fab38224 100644 --- a/client/Maestro/Properties/AssemblyInfo.cs +++ b/client/Maestro/Properties/AssemblyInfo.cs @@ -32,5 +32,5 @@ using System.Runtime.InteropServices; // You can specify all the values or you can default the Build and Revision Numbers // by using the '*' as shown below: // [assembly: AssemblyVersion("1.0.*")] -[assembly: AssemblyVersion("2.0.6.5")] -[assembly: AssemblyFileVersion("2.0.6.5")] +[assembly: AssemblyVersion("2.0.6.6")] +[assembly: AssemblyFileVersion("2.0.6.6")] diff --git a/client/Maestro/Sources/NexioRESTSource.cs b/client/Maestro/Sources/NexioRESTSource.cs index da2220c5..2bed6e43 100644 --- a/client/Maestro/Sources/NexioRESTSource.cs +++ b/client/Maestro/Sources/NexioRESTSource.cs @@ -23,6 +23,7 @@ namespace Maestro.Sources { private const string EXTAGENCY = "extagency"; private const string DURATION = "duration"; private const string MODIFIED = "modifiedtimestamp"; + private const string MXFEXT = ".MXF"; private List acceptableAgencies = new List(); private List cache; private IMessageBus messageBus; @@ -175,32 +176,38 @@ namespace Maestro.Sources { } private void OnResponse(object sender, MessageEventArgs e) { - JObject resultObject = JObject.Parse(e.Data); - JToken jToken = resultObject.GetValue(NexioAPI.DATA); + try { + JObject resultObject = JObject.Parse(e.Data); + JToken jToken = resultObject.GetValue(NexioAPI.DATA); - if (!jToken.HasValues) - return; - JToken d = JToken.Parse(jToken.ToString()); - string action = resultObject.GetValue(NexioAPI.ACTION).ToString(); - switch (action) { - case NexioAPI.LIST: - OnList(d); - break; - case NexioAPI.CREATE: - if (!initialized) - return; - OnCreated(d); - break; - case NexioAPI.DELETE: - if (!initialized) - return; - OnDeleted(d); - break; - case NexioAPI.UPDATE: - if (!initialized) - return; - OnChanged(d); - break; + if (!jToken.HasValues) + return; + JToken d = JToken.Parse(jToken.ToString()); + string action = resultObject.GetValue(NexioAPI.ACTION).ToString(); + switch (action) { + case NexioAPI.LIST: + OnList(d); + break; + case NexioAPI.CREATE: + if (!initialized) + return; + OnCreated(d); + break; + case NexioAPI.DELETE: + if (!initialized) + return; + OnDeleted(d); + break; + case NexioAPI.UPDATE: + if (!initialized) + return; + OnChanged(d); + break; + } + } + catch (Exception ex) { + logger.Error(ex); + MsgBox.Error(ex.Message); } } @@ -213,15 +220,19 @@ namespace Maestro.Sources { } private void SetItem(ref NexioSourceItem item, JToken token) { - item.ID = token[ID].ToString(); - item.Name = token[LONGNAMEID].ToString() + ".MXF"; - item.Agency = token[EXTAGENCY]?.ToString(); - item.Created = DateTime.Parse(token[RECORDDATE]?.ToString()); - item.Modified = DateTime.Parse(token[MODIFIED]?.ToString()); - int d = token.Value(DURATION); - if (d < 0) - d = d * -1; - item.Duration = new Timecode(d).ToString(); + try { + item.ID = token[ID].ToString(); + item.Name = token[LONGNAMEID].ToString() + MXFEXT; + item.Agency = token[EXTAGENCY]?.ToString(); + item.Created = DateTime.Parse(token[RECORDDATE]?.ToString()); + item.Modified = DateTime.Parse(token[MODIFIED]?.ToString()); + int d = token.Value(DURATION); + item.Duration = new Timecode(d).ToString(); + } + catch (Exception e) { + logger.Error(e.Message); + } + } @@ -235,7 +246,7 @@ namespace Maestro.Sources { Where(x => x.CanHandle(acceptableAgencies)). OrderByDescending(i => i.Created). ToList().ForEach(i => Add(i)); - initialized = true; + initialized = true; }); } diff --git a/server/-configuration/log4j2.xml b/server/-configuration/log4j2.xml index 9e0ab538..f6606e91 100644 --- a/server/-configuration/log4j2.xml +++ b/server/-configuration/log4j2.xml @@ -57,6 +57,6 @@ - + \ No newline at end of file diff --git a/server/-configuration/run-mediacube-server.launch b/server/-configuration/run-mediacube-server.launch index 2f966b04..fd9c1e1d 100644 --- a/server/-configuration/run-mediacube-server.launch +++ b/server/-configuration/run-mediacube-server.launch @@ -19,7 +19,7 @@ - + diff --git a/server/-configuration/scheduledjobs.json b/server/-configuration/scheduledjobs.json index a430fef5..7a8ea8cc 100644 --- a/server/-configuration/scheduledjobs.json +++ b/server/-configuration/scheduledjobs.json @@ -125,7 +125,7 @@ { "active": false, "name" : "Teszt folyamat", - "template": "fake.xml", + "template": "fake-noparams.xml", "executeimmediate": false, "cronexpression": "0/10 * * ? * *", "parameters": [ {"name": "itemID", "value": 1, "type": "java.lang.Long"} ] diff --git a/server/user.jobengine.executors/src/user/jobengine/server/steps/FakeStep.java b/server/user.jobengine.executors/src/user/jobengine/server/steps/FakeStep.java index 9e7a831c..fd7c139f 100644 --- a/server/user.jobengine.executors/src/user/jobengine/server/steps/FakeStep.java +++ b/server/user.jobengine.executors/src/user/jobengine/server/steps/FakeStep.java @@ -12,20 +12,21 @@ public class FakeStep extends JobStep { @StepEntry public Object[] execute(long itemID, IJobEngine jobEngine, IJobRuntime jobRuntime) throws Exception { - throw new Exception("42"); - // try { - // for (int i = 0; i < count; i++) { - // if (!canContinue()) - // break; - // jobRuntime.incrementProgress((i + 1) * count); - // Thread.sleep(1000); - // //logger.info(getMarker(), "Log markered"); - // //logger.info("Progress {}", jobRuntime.getProgress()); - // } - // } catch (Exception e) { - // logger.error(e.getMessage()); - // } - //return null; + // throw new Exception("42"); + Object[] result = { 1 }; + try { + for (int i = 0; i < count; i++) { + if (!canContinue()) + break; + jobRuntime.incrementProgress((i + 1) * count); + Thread.sleep(1000); + //logger.info(getMarker(), "Log markered"); + //logger.info("Progress {}", jobRuntime.getProgress()); + } + } catch (Exception e) { + logger.error(e.getMessage()); + } + return result; } } diff --git a/server/user.jobengine.executors/src/user/jobengine/server/steps/SyncOCTOPUSDataStep.java b/server/user.jobengine.executors/src/user/jobengine/server/steps/SyncOCTOPUSDataStep.java index 19a59c35..07510c89 100644 --- a/server/user.jobengine.executors/src/user/jobengine/server/steps/SyncOCTOPUSDataStep.java +++ b/server/user.jobengine.executors/src/user/jobengine/server/steps/SyncOCTOPUSDataStep.java @@ -24,10 +24,10 @@ public class SyncOCTOPUSDataStep extends JobStep { } @StepEntry - public Object[] execute(boolean forceFull, IJobEngine jobEngine, final IJobRuntime jobRuntime) { + public Object[] execute(boolean forceFull, IJobEngine jobEngine, final IJobRuntime jobRuntime) throws Exception { OctopusDataMiner dataMiner = null; try { - jobRuntime.incrementProgress(10); + //jobRuntime.incrementProgress(10); progressListener = createListener(jobRuntime); dataMiner = new OctopusDataMiner(); dataMiner.addProgressListener(progressListener); diff --git a/server/user.jobengine.osgi.commons/src/user/commons/nexio/NexioClipEventDispatcher.java b/server/user.jobengine.osgi.commons/src/user/commons/nexio/NexioClipEventDispatcher.java index 712fff3e..86c3b18f 100644 --- a/server/user.jobengine.osgi.commons/src/user/commons/nexio/NexioClipEventDispatcher.java +++ b/server/user.jobengine.osgi.commons/src/user/commons/nexio/NexioClipEventDispatcher.java @@ -38,8 +38,10 @@ import user.commons.remotestore.IProgressEventListener; */ public class NexioClipEventDispatcher implements ClipEventListener { + private static final Logger logger = LogManager.getLogger(); + private static final String NEXIO_USE_MOS_GATEWAY = "nexio.useMOSGateway"; private static final String ID = "id"; private static final String VIDEO_FORMAT = "videoformat"; private static final String VIDEO_BITRATE = "videobitrate"; @@ -128,6 +130,9 @@ public class NexioClipEventDispatcher implements ClipEventListener { BasicDBObject jsClip = new BasicDBObject(); jsClip.put(ID, evt.getClip().getId().get()); //Unique internal ID removeDeleted(jsClip); + } else if (evt.getEventType() == ClipEventType.LISTENER_RESTART_NEEDED) { + shutdown(); + startup(); } } catch (Exception exc) { logger.error("", exc); @@ -174,11 +179,6 @@ public class NexioClipEventDispatcher implements ClipEventListener { } catch (Exception e) { // System.out.println(e); } - try { - jsClip.put(FILESIZE, clip.getFileSize()); - } catch (Exception e) { - // System.out.println(e); - } try { String agency = new String(clip.getExtendedField(GetExtendedFieldCommand.FN_AGENCY), INexioAPI.ENCODING); jsClip.put(EXTAGENCY, agency); @@ -190,9 +190,6 @@ public class NexioClipEventDispatcher implements ClipEventListener { } catch (Exception e) { // System.out.println(e); } - //jsClip.put(RECORDDATE, new DateTime(rs.getString(i++)).toDate()); - //jsClip.put(EXTAGENCY, rs.getString(i++)); - //jsClip.put(START, rs.getString(i++)); //System.out.println(String.format("ID = '%s'\tXID = '%s'\tDuration = '%s'\tModified = '%s'\tVideoFormat = '%s'\tVideoBitrate = '%s'", id, xid, duration, modifiedTimestamp.getTime(), videoFormat, videoBitrate)); return jsClip; } @@ -221,7 +218,7 @@ public class NexioClipEventDispatcher implements ClipEventListener { logger.debug("Found NEXIO clip: {} XID IS MISSING!!", clip.getId()); } } catch (Exception e) { - logger.error("", e); + logger.catching(e); } } logger.info("NEXIO metadata import completed."); @@ -367,7 +364,8 @@ public class NexioClipEventDispatcher implements ClipEventListener { if (!clip.equals(clipForID)) { collection.save(clip); onUpdate(clip); - } + } else + logger.warn("Not found {}, XID based clip is {}", id, clipForXID.toPrettyString(null)); } } } @@ -404,17 +402,24 @@ public class NexioClipEventDispatcher implements ClipEventListener { logger.catching(e); return; } - logger.info("NEXIO server is connected."); + logger.info("NEXIO server connected"); this.progressListenerList = new EventListenerList(); this.dropAllClipsFromMongo(); this.ensureIndexes(); - + boolean[] useMOSGateway = { false }; + try { + useMOSGateway[0] = Boolean.parseBoolean(System.getProperty(NEXIO_USE_MOS_GATEWAY)); + } finally { + } executor.submit(() -> { try { - //1. kezdeti masolas - copyClipsIntoMongo(); - //2. esemeny feliratkozas + if (useMOSGateway[0]) { + NexioDataMiner dm = new NexioDataMiner(); + dm.transferClips(); + } else { + copyClipsIntoMongo(); + } NexioClipEventDispatcher.this.controller.getMediabase().startMediaListener(); NexioClipEventDispatcher.this.controller.getMediabase().getMediaListener().addClipEventListener(NexioClipEventDispatcher.this); } catch (Exception e) { diff --git a/server/user.jobengine.osgi.commons/src/user/commons/nexio/NexioDataMiner.java b/server/user.jobengine.osgi.commons/src/user/commons/nexio/NexioDataMiner.java index b937f65f..b6c4a6b8 100644 --- a/server/user.jobengine.osgi.commons/src/user/commons/nexio/NexioDataMiner.java +++ b/server/user.jobengine.osgi.commons/src/user/commons/nexio/NexioDataMiner.java @@ -15,7 +15,6 @@ import javax.swing.event.EventListenerList; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; -import org.joda.time.DateTime; import org.osgi.framework.Bundle; import org.osgi.framework.BundleContext; import org.osgi.framework.FrameworkUtil; @@ -43,19 +42,32 @@ public class NexioDataMiner implements Runnable, INexioChangeListener { private static final String EXTAGENCY = "extagency"; private static final String RECORDDATE = "recorddate"; private static final String LONGNAMEID = "longnameid"; + private static final String ID = "id"; public static final String CLIP_COLLECTION_NAME = "nexioclips"; + private static final String VIDEOBITRATE = "videobitrate"; + private static final String VIDEOFORMAT = "videoformat"; + private static final String KILLDATE = "killdate"; private static String url, user, pass; - private EventListenerList progressListenerList; - private ProgressEvent progressEvent = new ProgressEvent(this, 0); - private INexioChangeListener nexioChangeListener; - private DB db; - static { url = System.getProperty(JOBENGINE_NEXIO_DB_URL); user = System.getProperty(JOBENGINE_NEXIO_DB_USER); pass = System.getProperty(JOBENGINE_NEXIO_DB_PASSWORD); } + private static Connection getSQLConnection() throws ClassNotFoundException, SQLException { + Connection result = null; + Class.forName(SQLSERVER_JDBC_SQL_SERVER_DRIVER); + result = DriverManager.getConnection(url, user, pass); + return result; + } + + private EventListenerList progressListenerList; + private ProgressEvent progressEvent = new ProgressEvent(this, 0); + + private INexioChangeListener nexioChangeListener; + + private DB db; + public NexioDataMiner() { this.progressListenerList = new EventListenerList(); db = NoSQLUtils.getNoSQLDB(); @@ -66,15 +78,40 @@ public class NexioDataMiner implements Runnable, INexioChangeListener { progressListenerList.add(IProgressEventListener.class, listener); } - public void removeProgressListener(IProgressEventListener listener) { - progressListenerList.remove(IProgressEventListener.class, listener); + private void ensureIndexes() { + DBCollection collection = db.getCollection(CLIP_COLLECTION_NAME); + if (collection.count() == 0) + collection.ensureIndex(LONGNAMEID); } - private static Connection getSQLConnection() throws ClassNotFoundException, SQLException { - Connection result = null; - Class.forName(SQLSERVER_JDBC_SQL_SERVER_DRIVER); - result = DriverManager.getConnection(url, user, pass); - return result; + private void fireProgressEvent(ProgressEvent evt) { + logger.debug("Progress changed to " + evt.getProgress() + "%"); + Object[] listeners = progressListenerList.getListenerList(); + for (int i = 0; i < listeners.length; i += 2) { + if (listeners[i] == IProgressEventListener.class) + ((IProgressEventListener) listeners[i + 1]).progressChanged(evt); + } + } + + @Override + public void onCreate(BasicDBObject data) { + logger.debug("onCreate clip: " + data); + if (nexioChangeListener != null) + nexioChangeListener.onCreate(data); + } + + @Override + public void onDelete(BasicDBObject data) { + logger.debug("onDelete clip: " + data); + if (nexioChangeListener != null) + nexioChangeListener.onDelete(data); + } + + @Override + public void onUpdate(BasicDBObject data) { + logger.debug("onUpdate clip: " + data); + if (nexioChangeListener != null) + nexioChangeListener.onUpdate(data); } public Map queryClips(Connection con) { @@ -82,19 +119,21 @@ public class NexioDataMiner implements Runnable, INexioChangeListener { int count = queryClipsCount(con); logger.debug("Clips count is {}", count); - String querySQL = "SELECT c.longnameid, CONVERT(NVARCHAR(30), c.recorddate, 126) as recordate, c.extagency, CONVERT(NVARCHAR(30), cd.modifiedtimestamp, 126) as modifiedtimestamp, cd.start, cd.duration FROM ClipsTbl c LEFT OUTER JOIN ClipsDomainsTbl cd ON (cd.longnameid = c.longnameid)"; + String querySQL = "SELECT cd.id, c.longnameid, c.recorddate, c.extagency, cd.id, cd.modifiedtimestamp, cd.duration, cd.videoformat, cd.videobitrate FROM ClipsTbl c LEFT OUTER JOIN ClipsDomainsTbl cd ON (cd.longnameid = c.longnameid)"; logger.debug("Executing statement: {}", querySQL); try (PreparedStatement stmt = con.prepareStatement(querySQL); ResultSet rs = stmt.executeQuery();) { int idx = 1; while (rs.next()) { - int i = 1; BasicDBObject clip = new BasicDBObject(); - clip.put(LONGNAMEID, rs.getString(i++)); - clip.put(RECORDDATE, new DateTime(rs.getString(i++)).toDate()); - clip.put(EXTAGENCY, rs.getString(i++)); - clip.put(MODIFIEDTIMESTAMP, new DateTime(rs.getString(i++)).toDate()); - clip.put(START, rs.getString(i++)); - clip.put(DURATION, rs.getLong(i++)); + clip.put(ID, rs.getString(ID)); + clip.put(LONGNAMEID, rs.getString(LONGNAMEID)); + clip.put(RECORDDATE, rs.getDate(RECORDDATE).getTime()); + clip.put(EXTAGENCY, rs.getString(EXTAGENCY)); + clip.put(MODIFIEDTIMESTAMP, rs.getDate(MODIFIEDTIMESTAMP).getTime()); + clip.put(KILLDATE, rs.getDate(KILLDATE).getTime()); + clip.put(VIDEOBITRATE, rs.getString(VIDEOBITRATE)); + clip.put(VIDEOFORMAT, rs.getString(VIDEOFORMAT)); + clip.put(DURATION, rs.getLong(DURATION)); result.put(clip.getString(LONGNAMEID), clip); int progress = idx * 50 / count; if (progress - progressEvent.getProgress() > 0) { @@ -121,15 +160,44 @@ public class NexioDataMiner implements Runnable, INexioChangeListener { return count; } - private void fireProgressEvent(ProgressEvent evt) { - logger.debug("Progress changed to " + evt.getProgress() + "%"); - Object[] listeners = progressListenerList.getListenerList(); - for (int i = 0; i < listeners.length; i += 2) { - if (listeners[i] == IProgressEventListener.class) - ((IProgressEventListener) listeners[i + 1]).progressChanged(evt); + private Map readClips() { + Map result = new HashMap<>(); + DBCollection collection = db.getCollection(NexioDataMiner.CLIP_COLLECTION_NAME); + DBCursor find = collection.find(); + if (find.hasNext()) { + List clips = ListUtils.cast(find.toArray()); + result = ListUtils.map(clips, item -> item.getString(LONGNAMEID)); + for (BasicDBObject clip : clips) + result.put(clip.getString(LONGNAMEID), clip); + } + return result; + } + + private void removeDeleted(Map remoteClips, Map localClips) { + Set remoteNames = new HashSet<>(remoteClips.keySet()); + Set localNames = new HashSet<>(localClips.keySet()); + localNames.removeAll(remoteNames); + if (localNames.isEmpty()) + logger.debug("Nothing to remove"); + + DBCollection collection = db.getCollection(NexioDataMiner.CLIP_COLLECTION_NAME); + for (String name : localNames) { + logger.debug("Removing {}", name); + + BasicDBObject clip = localClips.get(name); + collection.remove(clip); + onDelete(clip); } } + public void removeProgressListener(IProgressEventListener listener) { + progressListenerList.remove(IProgressEventListener.class, listener); + } + + public void reset() { + db.getCollection(CLIP_COLLECTION_NAME).drop(); + } + @Override public void run() { try (Connection con = getSQLConnection()) { @@ -151,8 +219,7 @@ public class NexioDataMiner implements Runnable, INexioChangeListener { if (bundle != null) { BundleContext bundleContext = bundle.getBundleContext(); if (bundleContext != null) { - ServiceReference reference = bundleContext - .getServiceReference(INexioChangeListener.class); + ServiceReference reference = bundleContext.getServiceReference(INexioChangeListener.class); if (reference != null) { INexioChangeListener service = bundleContext.getService(reference); setNexioChangeListener(service); @@ -162,55 +229,8 @@ public class NexioDataMiner implements Runnable, INexioChangeListener { } } - private void ensureIndexes() { - DBCollection collection = db.getCollection(CLIP_COLLECTION_NAME); - if (collection.count() == 0) - collection.ensureIndex(LONGNAMEID); - } - - private void removeDeleted(Map remoteClips, Map localClips) { - Set remoteNames = new HashSet<>(remoteClips.keySet()); - Set localNames = new HashSet<>(localClips.keySet()); - localNames.removeAll(remoteNames); - if (localNames.isEmpty()) - logger.debug("Nothing to remove"); - - DBCollection collection = db.getCollection(NexioDataMiner.CLIP_COLLECTION_NAME); - for (String name : localNames) { - logger.debug("Removing {}", name); - - BasicDBObject clip = localClips.get(name); - collection.remove(clip); - onDelete(clip); - } - } - - private Map readClips() { - Map result = new HashMap<>(); - DBCollection collection = db.getCollection(NexioDataMiner.CLIP_COLLECTION_NAME); - DBCursor find = collection.find(); - if (find.hasNext()) { - List clips = ListUtils.cast(find.toArray()); - result = ListUtils.map(clips, item -> item.getString(LONGNAMEID)); - for (BasicDBObject clip : clips) - result.put(clip.getString(LONGNAMEID), clip); - } - return result; - } - - private void storeClips(Map remoteClips, Map localClips) { - logger.trace("Enter"); - int idx = 1; - for (BasicDBObject clip : remoteClips.values()) { - storeClip(clip, localClips); - int progress = 50 + (idx * 50 / remoteClips.size()); - if (progress - progressEvent.getProgress() > 0) { - progressEvent.setProgress(progress); - fireProgressEvent(progressEvent); - } - idx++; - } - logger.trace("Exit"); + public void setNexioChangeListener(INexioChangeListener nexioChangeListener) { + this.nexioChangeListener = nexioChangeListener; } private void storeClip(BasicDBObject clip, Map localClips) { @@ -233,33 +253,55 @@ public class NexioDataMiner implements Runnable, INexioChangeListener { logger.trace("Exit"); } - public void reset() { - db.getCollection(CLIP_COLLECTION_NAME).drop(); - } - - public void setNexioChangeListener(INexioChangeListener nexioChangeListener) { - this.nexioChangeListener = nexioChangeListener; - } - - @Override - public void onCreate(BasicDBObject data) { - logger.debug("onCreate clip: " + data); - if (nexioChangeListener != null) - nexioChangeListener.onCreate(data); + private void storeClips(Map remoteClips, Map localClips) { + logger.trace("Enter"); + int idx = 1; + for (BasicDBObject clip : remoteClips.values()) { + storeClip(clip, localClips); + int progress = 50 + (idx * 50 / remoteClips.size()); + if (progress - progressEvent.getProgress() > 0) { + progressEvent.setProgress(progress); + fireProgressEvent(progressEvent); + } + idx++; + } + logger.trace("Exit"); } - @Override - public void onDelete(BasicDBObject data) { - logger.debug("onDelete clip: " + data); - if (nexioChangeListener != null) - nexioChangeListener.onDelete(data); + public void transferClips() { + try (Connection con = getSQLConnection()) { + DBCollection collection = db.getCollection(CLIP_COLLECTION_NAME); + collection.drop(); + transferClips(con, collection); + } catch (Exception e) { + logger.error(e); + } } - @Override - public void onUpdate(BasicDBObject data) { - logger.debug("onUpdate clip: " + data); - if (nexioChangeListener != null) - nexioChangeListener.onUpdate(data); + private void transferClips(Connection con, DBCollection collection) { + logger.info("Transfering clips from NEXIO MOS Gateway {}", con.toString()); + String querySQL = "SELECT cd.id, c.longnameid, c.recorddate, c.extagency, c.killdate, cd.modifiedtimestamp, cd.duration, cd.videoformat, cd.videobitrate FROM ClipsTbl c LEFT OUTER JOIN ClipsDomainsTbl cd ON (cd.longnameid = c.longnameid)"; + logger.debug("Executing statement: {}", querySQL); + int count = 0; + try (PreparedStatement stmt = con.prepareStatement(querySQL); ResultSet rs = stmt.executeQuery();) { + while (rs.next()) { + BasicDBObject clip = new BasicDBObject(); + clip.put(ID, rs.getString(ID)); + clip.put(LONGNAMEID, rs.getString(LONGNAMEID)); + clip.put(EXTAGENCY, rs.getString(EXTAGENCY)); + clip.put(VIDEOBITRATE, rs.getString(VIDEOBITRATE)); + clip.put(VIDEOFORMAT, rs.getString(VIDEOFORMAT)); + clip.put(DURATION, rs.getLong(DURATION)); + clip.put(MODIFIEDTIMESTAMP, rs.getTimestamp(MODIFIEDTIMESTAMP)); + clip.put(RECORDDATE, rs.getTimestamp(RECORDDATE).getTime()); + clip.put(KILLDATE, rs.getTimestamp(KILLDATE)); + collection.insert(clip); + count++; + } + logger.info("Transfered {} clips", count); + } catch (Exception e) { + logger.error(e); + } } } diff --git a/server/user.jobengine.osgi.commons/src/user/commons/nexio/api/ClipEventType.java b/server/user.jobengine.osgi.commons/src/user/commons/nexio/api/ClipEventType.java index ad96dbb9..1a23e647 100644 --- a/server/user.jobengine.osgi.commons/src/user/commons/nexio/api/ClipEventType.java +++ b/server/user.jobengine.osgi.commons/src/user/commons/nexio/api/ClipEventType.java @@ -2,14 +2,13 @@ package user.commons.nexio.api; public enum ClipEventType { - CLIP_ADDED, - CLIP_DELETED; + LISTENER_RESTART_NEEDED, CLIP_ADDED, CLIP_DELETED; - public String value() { - return name(); - } + public static ClipEventType fromValue(String v) { + return valueOf(v); + } - public static ClipEventType fromValue(String v) { - return valueOf(v); - } + public String value() { + return name(); + } } diff --git a/server/user.jobengine.osgi.commons/src/user/commons/nexio/api/MediaListener.java b/server/user.jobengine.osgi.commons/src/user/commons/nexio/api/MediaListener.java index 84356e03..e9c321d7 100644 --- a/server/user.jobengine.osgi.commons/src/user/commons/nexio/api/MediaListener.java +++ b/server/user.jobengine.osgi.commons/src/user/commons/nexio/api/MediaListener.java @@ -2,7 +2,6 @@ package user.commons.nexio.api; import java.util.ArrayList; -import org.apache.commons.io.output.ByteArrayOutputStream; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; @@ -47,6 +46,7 @@ public class MediaListener implements Runnable { static private final int NN_ID_ADDED = 0; static private final int NN_ID_DELETED = 1; + static private final int NN_LISTENER_RESTART = 2; public static String bytesToHex(byte[] bytes) { char[] hexChars = new char[bytes.length * 2]; @@ -67,6 +67,8 @@ public class MediaListener implements Runnable { private Connection connection = null; + private ArrayList notificationList = new ArrayList(); + //KONSTRUKTOR public MediaListener(Mediabase _mediaBase, String _nexioHost, int _nexioPort) { this.mediaBase = _mediaBase; @@ -79,6 +81,30 @@ public class MediaListener implements Runnable { this.clipEventListeners.add(listener); } + private byte[] collectNotifications(boolean dontAnswer) throws Exception { + + //kov. notificationnek megfelelo listazas kerese + if (notificationList.size() > 0 && !dontAnswer) { + handleNotification(notificationList.get(0)); + } + + //tovabbi notification-ok gyujtese + //CHANGE_NOTIFICATION_RESP (BLOCKING READ!) ******** + byte[] buffer = null; + for (buffer = readBytes(2); CHANGE_NOTIFICATION_RESP_S.equals(bytesToHex(buffer)); buffer = readBytes(2)) { + buffer = readBytes(2); + int notNumber = buffer[1]; + notificationList.add(notNumber); + logger.debug("Nexio MediaListener notification: CFA6" + bytesToHex(buffer)); + + //csak igy jon meg a kesobb kezelt LIST_FIRST_ID_LIST + if (notificationList.size() == 1 && !dontAnswer) { + handleNotification(notNumber); + } + } + return buffer; + } + private void createNexioConnection() throws Exception { this.connection = TCPConnection.getInstance(this.nexioHost, this.nexioPort); logger.info("Nexio MediaListener is connected to NEXIO " + this.nexioHost + " : " + this.nexioPort); @@ -92,28 +118,42 @@ public class MediaListener implements Runnable { } else if (notificationNumber == NN_ID_DELETED) { Clip deletedClip = new ClipImpl(mediaBase.getProtocol(), id); evt = new ClipEvent(deletedClip, ClipEventType.CLIP_DELETED); + } else if (notificationNumber == NN_LISTENER_RESTART) { + evt = new ClipEvent(null, ClipEventType.LISTENER_RESTART_NEEDED); } for (ClipEventListener cel : this.clipEventListeners) { cel.clipEventPerformed(evt); } } - private byte[] readBytesOLD(int numberOfBytes) throws Exception { - try { - byte[] ret = new byte[numberOfBytes]; - this.connection.read(ret, 0, numberOfBytes); - return ret; - } catch (Exception e) { - restartListener(); - throw e; + private void handleNotification(int notificationNumber) throws Exception { + //LIST_FIRST_ID_LIST + /* + 0x01 Main ID Handle List D8 4C ID + 0x02 ID Handles Added List D8 4C ID + 0x03 ID Handles Deleted List D8 4C ID + 0x11 Main Extended ID List DF 4C BC XID * + 0x12 Extended IDs Added List DF 4C BC XID * + 0x13 Extended IDs Deleted List DF 4C BC XID * + 0x21 Main ID Handle/Extended ID List DF 4C BC ID XID * + 0x22 ID Handles/Extended IDs Added List DF 4C BC ID XID * + 0x23 ID Handles/Extended IDs Deleted List DF 4C BC ID XID * + */ + if (notificationNumber == NN_ID_ADDED) { + writeBytes(LIST_FIRST_ID_LIST_ADDED); + } else if (notificationNumber == NN_ID_DELETED) { + //SEND ACK + writeBytes(ACK); + writeBytes(LIST_FIRST_ID_LIST_DELETED); } + } private byte[] readBytes(int numberOfBytes) throws Exception { byte[] ret = new byte[numberOfBytes]; try { - for(int c=0; c notificationList = new ArrayList(); - - @Override public void run() { try { @@ -157,19 +206,18 @@ public class MediaListener implements Runnable { byte[] buffer = readBytes(2); logger.info("Nexio MediaListener is waiting for events.."); - while (isRunning) { try { //BLOKKOLVA addig olvassa fel es gyujti a notification-oket amig mas adat nem jon. //az elso alkalommal amikor nem notification-t kap, visszater. buffer = collectNotifications(false); - + //LIST_FIRST_ID_LIST response if (LIST_FIRST_ID_LIST_FOUND.equals(bytesToHex(buffer))) { int notificationNumber = notificationList.get(0); notificationList.remove(0); - + //read first id buffer = readBytes(8); String stringId = new String(buffer); @@ -179,7 +227,8 @@ public class MediaListener implements Runnable { //read next ids writeBytes(LIST_NEXT_ID_LIST); //for (buffer = readBytes(2); LIST_NEXT_ID_LIST_FOUND.equals(bytesToHex(buffer)) && isRunning; buffer = readBytes(2)) { - for (buffer = collectNotifications(true); LIST_NEXT_ID_LIST_FOUND.equals(bytesToHex(buffer)) && isRunning; buffer = collectNotifications(true)) { + for (buffer = collectNotifications(true); LIST_NEXT_ID_LIST_FOUND.equals(bytesToHex(buffer)) + && isRunning; buffer = collectNotifications(true)) { byte[] idBuffer = readBytes(8); stringId = new String(idBuffer); @@ -191,12 +240,11 @@ public class MediaListener implements Runnable { logger.debug("Nexio MediaListener end of id list: " + bytesToHex(buffer)); //SEND ACK writeBytes(ACK); - - }else - if (LIST_FIRST_ID_LIST_NOT_FOUND.equals(bytesToHex(buffer))) { + + } else if (LIST_FIRST_ID_LIST_NOT_FOUND.equals(bytesToHex(buffer))) { notificationList.remove(0); logger.debug("Nexio MediaListener LIST_FIRST_ID_LIST ID not found!"); - }else{ + } else { logger.debug("Nexio MediaListener unknown answer has arrived for LIST_FIRST_ID_LIST: " + bytesToHex(buffer)); } } catch (Exception exc) { @@ -209,56 +257,6 @@ public class MediaListener implements Runnable { } } - - private byte[] collectNotifications(boolean dontAnswer) throws Exception{ - - //kov. notificationnek megfelelo listazas kerese - if(notificationList.size() > 0 && !dontAnswer){ - handleNotification(notificationList.get(0)); - } - - //tovabbi notification-ok gyujtese - //CHANGE_NOTIFICATION_RESP (BLOCKING READ!) ******** - byte[] buffer = null; - for(buffer = readBytes(2); CHANGE_NOTIFICATION_RESP_S.equals(bytesToHex(buffer)); buffer = readBytes(2)){ - buffer = readBytes(2); - int notNumber = buffer[1]; - notificationList.add(notNumber); - logger.debug("Nexio MediaListener notification: CFA6" + bytesToHex(buffer)); - - //csak igy jon meg a kesobb kezelt LIST_FIRST_ID_LIST - if(notificationList.size() == 1 && !dontAnswer){ - handleNotification(notNumber); - } - } - return buffer; - } - - - private void handleNotification(int notificationNumber) throws Exception{ - //LIST_FIRST_ID_LIST - /* - 0x01 Main ID Handle List D8 4C ID - 0x02 ID Handles Added List D8 4C ID - 0x03 ID Handles Deleted List D8 4C ID - 0x11 Main Extended ID List DF 4C BC XID * - 0x12 Extended IDs Added List DF 4C BC XID * - 0x13 Extended IDs Deleted List DF 4C BC XID * - 0x21 Main ID Handle/Extended ID List DF 4C BC ID XID * - 0x22 ID Handles/Extended IDs Added List DF 4C BC ID XID * - 0x23 ID Handles/Extended IDs Deleted List DF 4C BC ID XID * - */ - if (notificationNumber == NN_ID_ADDED) { - writeBytes(LIST_FIRST_ID_LIST_ADDED); - }else - if (notificationNumber == NN_ID_DELETED) { - //SEND ACK - writeBytes(ACK); - writeBytes(LIST_FIRST_ID_LIST_DELETED); - } - - } - public void startListener() { try { createNexioConnection(); diff --git a/server/user.jobengine.osgi.commons/src/user/commons/nosql/NoSQLUtils.java b/server/user.jobengine.osgi.commons/src/user/commons/nosql/NoSQLUtils.java index e81b8f63..1649db72 100644 --- a/server/user.jobengine.osgi.commons/src/user/commons/nosql/NoSQLUtils.java +++ b/server/user.jobengine.osgi.commons/src/user/commons/nosql/NoSQLUtils.java @@ -15,7 +15,7 @@ import com.ibm.nosql.json.api.NoSQLClient; public class NoSQLUtils { private static final Logger logger = LogManager.getLogger(); public static final String Url, User, Pass; - + public static DB noSQLClient = null; static { Url = System.getProperty("jobengine.nosql.db.url"); User = System.getProperty("jobengine.nosql.db.user"); @@ -67,17 +67,13 @@ public class NoSQLUtils { } public static DB getNoSQLDB() { - DB result = null; try { - result = NoSQLClient.getDB(NoSQLUtils.Url, NoSQLUtils.User, NoSQLUtils.Pass); + if (noSQLClient == null) + noSQLClient = NoSQLClient.getDB(NoSQLUtils.Url, NoSQLUtils.User, NoSQLUtils.Pass); } catch (Exception e) { - logger.error(e); + logger.catching(e); } - return result; - } - - public static DB getNoSQLDB(String url, String user, String pass) { - return NoSQLClient.getDB(Url, User, Pass); + return noSQLClient; } } diff --git a/server/user.jobengine.osgi.commons/src/user/commons/octopus/IOctopusAPI.java b/server/user.jobengine.osgi.commons/src/user/commons/octopus/IOctopusAPI.java index 69d3e8d4..d8a8baec 100644 --- a/server/user.jobengine.osgi.commons/src/user/commons/octopus/IOctopusAPI.java +++ b/server/user.jobengine.osgi.commons/src/user/commons/octopus/IOctopusAPI.java @@ -12,7 +12,7 @@ public interface IOctopusAPI { static final String TIME_COLLECTION_NAME = "time"; static final String LASTUPDATE_TIME = "lastUpdateTime"; static final String NEXIO_MOS = "NEXIO.MOS"; - static final String PARENT_STORY_ID = "parentStoryId"; + static final String PARENT_STORY_ID = "ParentStoryId"; static final String VALUE = "value"; static final String CUSTOM_COLUMNS = "customColumns"; static final String MOS_ID = "mosId"; diff --git a/server/user.jobengine.osgi.commons/src/user/commons/octopus/OctopusDataMiner.java b/server/user.jobengine.osgi.commons/src/user/commons/octopus/OctopusDataMiner.java index 48eedfac..87a3fcb0 100644 --- a/server/user.jobengine.osgi.commons/src/user/commons/octopus/OctopusDataMiner.java +++ b/server/user.jobengine.osgi.commons/src/user/commons/octopus/OctopusDataMiner.java @@ -5,6 +5,7 @@ import java.util.HashMap; import java.util.HashSet; import java.util.List; import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; import javax.swing.event.EventListenerList; import javax.ws.rs.client.Invocation.Builder; @@ -28,12 +29,13 @@ import com.ibm.nosql.json.api.DBObject; import com.ibm.nosql.json.api.QueryBuilder; import com.ibm.nosql.json.api.WriteResult; +import user.commons.ListUtils; import user.commons.nosql.NoSQLUtils; import user.commons.remotestore.IProgressEventListener; import user.commons.remotestore.ProgressEvent; public class OctopusDataMiner implements Runnable { - private static final String _TMP = "_tmp1"; + private static final String _TMP = "_tmp"; private static final Logger logger = LogManager.getLogger(); private static final String LINEFEED = "\r\n"; private static final String SIMPLE_LINEFEED = "\n"; @@ -74,6 +76,10 @@ public class OctopusDataMiner implements Runnable { private String FOLDER_COLLECTION = IOctopusAPI.FOLDER_COLLECTION; private String STORY_COLLECTION = IOctopusAPI.STORY_COLLECTION; + private Map newRundowns = new HashMap<>(); + private Map newStoryFolders = new HashMap<>(); + private Map newStories = new HashMap<>(); + public OctopusDataMiner() { db = NoSQLUtils.getNoSQLDB(); @@ -203,6 +209,44 @@ public class OctopusDataMiner implements Runnable { return concatParentsToStoryFolder(parent, newName); } + private void deleteDiff(String oldCollectionName, String newCollectionName, String idFieldName) { + DBCollection oldCollection = db.getCollection(oldCollectionName); + DBCollection newCollection = db.getCollection(newCollectionName); + DBCursor oldCollectionCursor = newCollection.find(new BasicDBObject(), new BasicDBObject(idFieldName, 1)); + if (!oldCollectionCursor.hasNext()) { + logger.error("{} collection is empty", newCollectionName); + return; + } + List oldItems = ListUtils.cast(oldCollectionCursor.toArray()); + + DBCursor newCollectionCursor = oldCollection.find(); + ConcurrentHashMap newItems = null; + if (newCollectionCursor.hasNext()) { + List newList = ListUtils.cast(newCollectionCursor.toArray()); + newItems = ListUtils.map(newList, item -> item.getLong(IOctopusAPI.ID)); + } + if (newItems == null) + newItems = new ConcurrentHashMap<>(); + + for (BasicDBObject oldItem : oldItems) { + if (oldItem == null) { + logger.error("Item is null"); + continue; + } + if (!oldItem.containsKey(idFieldName)) { + logger.error("{} is null", idFieldName); + continue; + } + long id = oldItem.getLong(idFieldName); + BasicDBObject newItem = newItems.get(id); + if (newItem == null) { + //remove + logger.info("Deleting {}", oldItem.toPrettyString(null)); + oldCollection.remove(new BasicDBObject(idFieldName, id)); + } + } + } + private void deleteOrphanStories() { try { DBCollection collection = db.getCollection(STORY_COLLECTION); @@ -228,18 +272,20 @@ public class OctopusDataMiner implements Runnable { collection.ensureIndex(IOctopusAPI.ID); } - public void execute() { + public void execute() throws Exception { logger.trace(STARTING); RUNDOWN_COLLECTION = IOctopusAPI.RUNDOWN_COLLECTION + _TMP; FOLDER_COLLECTION = IOctopusAPI.FOLDER_COLLECTION + _TMP; STORY_COLLECTION = IOctopusAPI.STORY_COLLECTION + _TMP; - // ResteasyWebTarget target = webTarget.path(RUNDOWN); - // Builder result = target.request().header(OCTOPUS_DEVICE_ID, apiUser).header(OCTOPUS_DEVICE_NAME, apiPwd); - // Response r = result.get(); - // String x = r.readEntity(String.class); - // logger.info(x); - // return; + try { + db.getCollection(RUNDOWN_COLLECTION).drop(); + db.getCollection(FOLDER_COLLECTION).drop(); + db.getCollection(STORY_COLLECTION).drop(); + } catch (Exception e) { + logger.catching(e); + throw e; + } BasicDBList rundowns = null; BasicDBList storyFolders = null; @@ -252,60 +298,34 @@ public class OctopusDataMiner implements Runnable { throw e; } - if (rundowns == null || rundowns.size() == 0) { - progressEvent.setProgress(50); - fireProgressEvent(progressEvent); - } else { - logger.info("Process rundowns"); - try { - storeRundowns(rundowns, null); - } catch (Exception e) { - logger.catching(e); - throw e; - } - } - - if (storyFolders == null || storyFolders.size() == 0) { - progressEvent.setProgress(100); - fireProgressEvent(progressEvent); - } else { - logger.info("Process story folders"); - try { - storeStoryFolders(storyFolders, null); - } catch (Exception e) { - logger.catching(e); - throw e; - } - } - setLastUpdateTime(new Date()); - logger.info("Activate"); + processRundowns(rundowns); + processStoryFolders(storyFolders); - // db.getCollection(IOctopusAPI.RUNDOWN_COLLECTION).drop(); - try { - db.getCollection(RUNDOWN_COLLECTION).rename(IOctopusAPI.RUNDOWN_COLLECTION, true); - } catch (Exception e) { - logger.catching(e); - throw e; - } + //a sorrend fontos ! + updateDiff(IOctopusAPI.STORY_COLLECTION, STORY_COLLECTION, IOctopusAPI.ID); + updateDiff(IOctopusAPI.RUNDOWN_COLLECTION, RUNDOWN_COLLECTION, IOctopusAPI.ID); + updateDiff(IOctopusAPI.FOLDER_COLLECTION, FOLDER_COLLECTION, IOctopusAPI.ID); + deleteDiff(IOctopusAPI.RUNDOWN_COLLECTION, RUNDOWN_COLLECTION, IOctopusAPI.ID); + deleteDiff(IOctopusAPI.FOLDER_COLLECTION, FOLDER_COLLECTION, IOctopusAPI.ID); + deleteDiff(IOctopusAPI.STORY_COLLECTION, STORY_COLLECTION, IOctopusAPI.ID); - // db.getCollection(IOctopusAPI.FOLDER_COLLECTION).drop(); - try { - db.getCollection(FOLDER_COLLECTION).rename(IOctopusAPI.FOLDER_COLLECTION, true); - } catch (Exception e) { - logger.catching(e); - throw e; - } + //setLastUpdateTime(new Date()); + logger.info("Activate"); - // db.getCollection(IOctopusAPI.STORY_COLLECTION).drop(); - try { - db.getCollection(STORY_COLLECTION).rename(IOctopusAPI.STORY_COLLECTION, true); - } catch (Exception e) { - logger.catching(e); - throw e; - } + // db.getCollection(RUNDOWN_COLLECTION).rename(IOctopusAPI.RUNDOWN_COLLECTION, true); + // db.getCollection(FOLDER_COLLECTION).rename(IOctopusAPI.FOLDER_COLLECTION, true); + // db.getCollection(STORY_COLLECTION).rename(IOctopusAPI.STORY_COLLECTION, true); + logger.trace(FINISHED); - // logger.trace(FINISHED); + } + public void executetest() { + // ResteasyWebTarget target = webTarget.path(RUNDOWN); + // Builder result = target.request().header(OCTOPUS_DEVICE_ID, apiUser).header(OCTOPUS_DEVICE_NAME, apiPwd); + // Response r = result.get(); + // String x = r.readEntity(String.class); + // logger.info(x); + // return; } private String extractContent(BasicDBObject content) { @@ -387,14 +407,6 @@ public class OctopusDataMiner implements Runnable { return result; } - // private boolean isModified(Date date, BasicDBObject object, String name) { - // Date actualModifiedString = toDate(object, name); - // if (actualModifiedString == null) - // logger.trace(ACTUAL_MODIFIED_STRING_IS_NULL); - // int result = date.compareTo(actualModifiedString); - // return result <= 0; - // } - private String extractScriptContent(BasicDBObject story) { BasicDBObject script = NoSQLUtils.asDBObject(story, IOctopusAPI.SCRIPT); if (script == null || script.isEmpty()) @@ -441,6 +453,14 @@ public class OctopusDataMiner implements Runnable { return result; } + // private boolean isModified(Date date, BasicDBObject object, String name) { + // Date actualModifiedString = toDate(object, name); + // if (actualModifiedString == null) + // logger.trace(ACTUAL_MODIFIED_STRING_IS_NULL); + // int result = date.compareTo(actualModifiedString); + // return result <= 0; + // } + private boolean isModified(Date date, BasicDBObject object) { if (date == null) return true; @@ -449,6 +469,36 @@ public class OctopusDataMiner implements Runnable { return result <= 0; } + private void processRundowns(BasicDBList rundowns) throws Exception { + if (rundowns == null || rundowns.size() == 0) { + progressEvent.setProgress(50); + fireProgressEvent(progressEvent); + } else { + logger.info("Process rundowns"); + try { + storeRundowns(rundowns, null); + } catch (Exception e) { + logger.catching(e); + throw e; + } + } + } + + private void processStoryFolders(BasicDBList storyFolders) throws Exception { + if (storyFolders == null || storyFolders.size() == 0) { + progressEvent.setProgress(100); + fireProgressEvent(progressEvent); + } else { + logger.info("Process story folders"); + try { + storeStoryFolders(storyFolders, null); + } catch (Exception e) { + logger.catching(e); + throw e; + } + } + } + private Builder query(String path, String fields) { //logger.info("Class loader {}", getClass().getClassLoader()); // try { @@ -635,7 +685,7 @@ public class OctopusDataMiner implements Runnable { List rundownsList = NoSQLUtils.asList(rundowns); int idx = 1; for (BasicDBObject rundown : rundownsList) { - logger.info(CHECKING_RUNDOWN, rundown.getLong(IOctopusAPI.ID), rundownsList.size(), idx); + //logger.info(CHECKING_RUNDOWN, rundown.getLong(IOctopusAPI.ID), rundownsList.size(), idx); storeRundown(rundown, lastUpdateTime); int progress = idx * 50 / rundownsList.size(); if (progress - progressEvent.getProgress() > 0) { @@ -660,6 +710,10 @@ public class OctopusDataMiner implements Runnable { private void storeStory(BasicDBObject story, Date lastUpdateTime) { logger.trace(ENTER); + if (!story.containsKey(IOctopusAPI.ID)) { + logger.error("Missing id in story {}", story.toPrettyString(null)); + return; + } long storyID = story.getLong(IOctopusAPI.ID); boolean isContains = storyIDs.contains(storyID); if (isContains) @@ -717,9 +771,10 @@ public class OctopusDataMiner implements Runnable { story.put(IOctopusAPI.MOS_OBJECTS, modifiedMOS); String parentStoryId = extractCustomColumnValue(IOctopusAPI.PARENT_STORY_ID, story); - if (parentStoryId == null) + if (parentStoryId == null) { + logger.warn("Missing {} in story", IOctopusAPI.PARENT_STORY_ID, story.toPrettyString(null)); story.append(IOctopusAPI.PARENT_STORY_ID, storyID); - else + } else story.append(IOctopusAPI.PARENT_STORY_ID, parentStoryId); logger.debug(SAVING_STORY_ID, storyID); collection.save(story); @@ -782,4 +837,58 @@ public class OctopusDataMiner implements Runnable { return result; } + private void updateDiff(String oldCollectionName, String newCollectionName, String idFieldName) { + DBCollection oldCollection = db.getCollection(oldCollectionName); + DBCollection newCollection = db.getCollection(newCollectionName); + DBCursor newCollectionCursor = newCollection.find(); + if (!newCollectionCursor.hasNext()) { + logger.error("{} collection is empty", newCollectionName); + return; + } + + List newItems = ListUtils.cast(newCollectionCursor.toArray()); + + DBCursor oldCollectionCursor = oldCollection.find(); + ConcurrentHashMap oldItems = null; + if (oldCollectionCursor.hasNext()) { + List oldList = ListUtils.cast(oldCollectionCursor.toArray()); + oldItems = ListUtils.map(oldList, item -> item.getLong(IOctopusAPI.ID)); + } + if (oldItems == null) + oldItems = new ConcurrentHashMap<>(); + for (BasicDBObject newItem : newItems) { + if (newItem == null) { + logger.error("Item is null"); + continue; + } + if (!newItem.containsKey(idFieldName)) { + logger.error("{} is null", idFieldName); + continue; + } + long id = newItem.getLong(idFieldName); + BasicDBObject oldItem = oldItems.get(id); + boolean save = false; + if (oldItem == null) { + //save + save = true; + } else { + //compare + Object oldDbId = oldItem.getID(); + newItem.remove(IOctopusAPI._ID); + oldItem.remove(IOctopusAPI._ID); + + if (!newItem.equals(oldItem)) { + //save + newItem.put(IOctopusAPI._ID, oldDbId); + save = true; + } + } + + if (save) { + logger.info("Saving {}", newItem.toPrettyString(null)); + oldCollection.save(newItem); + } + } + } + } diff --git a/server/user.jobengine.osgi.commons/test/user/common/octopus/test/OctopusDataMinerTest.java b/server/user.jobengine.osgi.commons/test/user/common/octopus/test/OctopusDataMinerTest.java index eb4ba4c0..49c39444 100644 --- a/server/user.jobengine.osgi.commons/test/user/common/octopus/test/OctopusDataMinerTest.java +++ b/server/user.jobengine.osgi.commons/test/user/common/octopus/test/OctopusDataMinerTest.java @@ -48,7 +48,7 @@ public class OctopusDataMinerTest { private OctopusDataMiner sut; @Test - public void integrationExecute() { + public void integrationExecute() throws Exception { // fixture sut = new OctopusDataMiner(); // Exercise diff --git a/server/user.jobengine.osgi.services/deploy.launch b/server/user.jobengine.osgi.services/deploy.launch index 93a34309..94e5a916 100644 --- a/server/user.jobengine.osgi.services/deploy.launch +++ b/server/user.jobengine.osgi.services/deploy.launch @@ -1,7 +1,7 @@ - + diff --git a/server/user.jobengine.osgi.services/src/user/jobengine/osgi/ws/nexio/NexioWSServlet.java b/server/user.jobengine.osgi.services/src/user/jobengine/osgi/ws/nexio/NexioWSServlet.java index 24dced10..71709d09 100644 --- a/server/user.jobengine.osgi.services/src/user/jobengine/osgi/ws/nexio/NexioWSServlet.java +++ b/server/user.jobengine.osgi.services/src/user/jobengine/osgi/ws/nexio/NexioWSServlet.java @@ -34,13 +34,13 @@ public class NexioWSServlet extends WebSocketServlet implements INexioChangeList private NexioClipEventDispatcher dispatcher = null; public NexioWSServlet() { - logger.info("NexioAPIServlet created"); + logger.info("Created"); // messagePool = new NexioAPIServletMessagePool(3, sessions); instance = this; dispatcher = new NexioClipEventDispatcher(); dispatcher.setNexioChangeListener(this); - //dispatcher.startup(); - logger.info("NexioWSServlet has been initialized."); + dispatcher.startup(); + logger.info("Initialized"); /* BundleContext bundleContext = FrameworkUtil.getBundle(this.getClass()).getBundleContext(); if (bundleContext != null) {