From ce6b0dd897d8c762f2d05aa1667071cb0e4f6cbe Mon Sep 17 00:00:00 2001 From: =?utf8?q?V=C3=A1s=C3=A1ry=20D=C3=A1niel?= Date: Tue, 19 Dec 2017 22:37:19 +0000 Subject: [PATCH] git-tfs-id: [http://tfs.userrendszerhaz.hu:8080/tfs/DefaultCollection]$/MediaCube;C30814 --- .../debug-mediacube-remote.launch | 2 +- .../deploy-bkup-mediacube-modules.launch | 2 +- .../deploy-cluster-mediacube-modules.launch | 2 +- .../deploy-main-mediacube-modules.launch | 19 ++++ .../test-deploy-bkup-mediacube-modules.launch | 19 ++++ .../test-deploy-main-mediacube-modules.launch | 19 ++++ server/-product/log4j2.xml | 11 ++- server/-product/mediacube.product | 89 +++++++++---------- server/-product/pom.xml | 1 - .../config/scheduledjobs.json | 2 +- .../CopyForArchiveNEXIOMaterialsStep.java | 9 +- .../server/steps/OctopusDataMiner.java | 56 +++++++++--- .../src/user/commons/StoreUri.java | 5 ++ 13 files changed, 165 insertions(+), 71 deletions(-) create mode 100644 server/-configuration/deploy-main-mediacube-modules.launch create mode 100644 server/-configuration/test-deploy-bkup-mediacube-modules.launch create mode 100644 server/-configuration/test-deploy-main-mediacube-modules.launch diff --git a/server/-configuration/debug-mediacube-remote.launch b/server/-configuration/debug-mediacube-remote.launch index e7463db1..b715dd61 100644 --- a/server/-configuration/debug-mediacube-remote.launch +++ b/server/-configuration/debug-mediacube-remote.launch @@ -10,7 +10,7 @@ - + diff --git a/server/-configuration/deploy-bkup-mediacube-modules.launch b/server/-configuration/deploy-bkup-mediacube-modules.launch index 1388508f..0ed5d2e0 100644 --- a/server/-configuration/deploy-bkup-mediacube-modules.launch +++ b/server/-configuration/deploy-bkup-mediacube-modules.launch @@ -1,7 +1,7 @@ - + diff --git a/server/-configuration/deploy-cluster-mediacube-modules.launch b/server/-configuration/deploy-cluster-mediacube-modules.launch index 22037459..c02915a8 100644 --- a/server/-configuration/deploy-cluster-mediacube-modules.launch +++ b/server/-configuration/deploy-cluster-mediacube-modules.launch @@ -1,7 +1,7 @@ - + diff --git a/server/-configuration/deploy-main-mediacube-modules.launch b/server/-configuration/deploy-main-mediacube-modules.launch new file mode 100644 index 00000000..908437a2 --- /dev/null +++ b/server/-configuration/deploy-main-mediacube-modules.launch @@ -0,0 +1,19 @@ + + + + + + + + + + + + + + + + + + + diff --git a/server/-configuration/test-deploy-bkup-mediacube-modules.launch b/server/-configuration/test-deploy-bkup-mediacube-modules.launch new file mode 100644 index 00000000..ed06cb2f --- /dev/null +++ b/server/-configuration/test-deploy-bkup-mediacube-modules.launch @@ -0,0 +1,19 @@ + + + + + + + + + + + + + + + + + + + diff --git a/server/-configuration/test-deploy-main-mediacube-modules.launch b/server/-configuration/test-deploy-main-mediacube-modules.launch new file mode 100644 index 00000000..533ba6cf --- /dev/null +++ b/server/-configuration/test-deploy-main-mediacube-modules.launch @@ -0,0 +1,19 @@ + + + + + + + + + + + + + + + + + + + diff --git a/server/-product/log4j2.xml b/server/-product/log4j2.xml index 91345a08..08bb456e 100644 --- a/server/-product/log4j2.xml +++ b/server/-product/log4j2.xml @@ -21,13 +21,20 @@ - + + + + + + + + diff --git a/server/-product/mediacube.product b/server/-product/mediacube.product index 3762d6c7..82fbbf90 100644 --- a/server/-product/mediacube.product +++ b/server/-product/mediacube.product @@ -9,53 +9,52 @@ -consoleLog -console 5555 - -Xms512m --Xmx1024m --Dfile.encoding=UTF-8 --agentlib:jdwp=transport=dt_socket,server=y,suspend=n,address=8888 --Djavax.ws.rs.ext.RuntimeDelegate=org.jboss.resteasy.spi.ResteasyProviderFactory - -#System --Djetty.home=/opt/mediacube/configuration/jetty --Dlog4j.configurationFile=/opt/mediacube/configuration/log4j2.xml --Djobengine.jobsteps.root=/opt/mediacube/configuration/executors --Djobengine.jobtemplates.root=/opt/mediacube/configuration/jobtemplates --Djobengine.jobscheduling.config=/opt/mediacube/configuration/scheduledjobs.json - --Dorg.eclipse.equinox.http.jetty.http.port=80 --Dorg.eclipse.epp.logging.aeri.skipReports=true --Declipse.ignoreApp=true --Dosgi.noShutdown=true --Djobengine.loglevel=INFO --Djetty.etc.config.urls=etc/user-jetty.xml,etc/user-jetty-ssl.xml,etc/user-jetty-ssl-context.xml,etc/user-jetty-http.xml,etc/user-jetty-https.xml - - #Database --Djobengine.db.url=jdbc:db2://localhost:50000/mc:retrieveMessagesFromServerOnGetMessage=true; --Djobengine.db.user=db2admin --Djobengine.db.password=password --Djobengine.nosql.db.url=jdbc:db2://localhost:50000/mc:retrieveMessagesFromServerOnGetMessage=true; --Djobengine.nosql.db.user=db2admin --Djobengine.nosql.db.password=password --Djobengine.nosql.db.schema=test - -#Octopus --Djobengine.octopus.api.address=http://10.10.1.11/api/v1 --Djobengine.octopus.api.user=mam --Djobengine.octopus.api.password=napocska --Djobengine.octopus.rundowns.name=test-rundowns --Djobengine.octopus.storyfolders.name=test-storyfolders --Djobengine.octopus.stories.name=test-stories + -Xms512m +-Xmx1024m +-Dfile.encoding=UTF-8 +-agentlib:jdwp=transport=dt_socket,server=y,suspend=n,address=8888 +-Djavax.ws.rs.ext.RuntimeDelegate=org.jboss.resteasy.spi.ResteasyProviderFactory -#Selenio --Djobengine.selenio.address=http://10.10.1.71:44000/TranscodeMgrWS?wsdl --Djobengine.selenio.projectfilepath=\\10.10.1.71\Data\Blueprints\MP4_H264_AAC.zenium +#System +-Djetty.home=/opt/mediacube/configuration/jetty +-Dlog4j.configurationFile=/opt/mediacube/configuration/log4j2.xml +-Djobengine.jobsteps.root=/opt/mediacube/configuration/executors +-Djobengine.jobtemplates.root=/opt/mediacube/configuration/jobtemplates +-Djobengine.jobscheduling.config=/opt/mediacube/configuration/scheduledjobs.json -#Nexio --Djobengine.nexio.db.url=jdbc:sqlserver://10.10.1.59:1433;databaseName=NXDB; --Djobengine.nexio.db.user=sa --Djobengine.nexio.db.password=resolve --Dnexio.useMOSGateway=true --Dnexio.host=10.10.1.55 +-Dorg.eclipse.epp.logging.aeri.skipReports=true +-Declipse.ignoreApp=true +-Dosgi.noShutdown=true +-Djobengine.loglevel=INFO +-Djetty.etc.config.urls=etc/user-jetty.xml,etc/user-jetty-ssl.xml,etc/user-jetty-ssl-context.xml,etc/user-jetty-http.xml,etc/user-jetty-https.xml + + #Database +-Djobengine.db.url=jdbc:db2://10.10.1.27:50000/mc:retrieveMessagesFromServerOnGetMessage=true; +-Djobengine.db.user=db2admin +-Djobengine.db.password=password +-Djobengine.nosql.db.url=jdbc:db2://10.10.1.27:50000/mc:retrieveMessagesFromServerOnGetMessage=true; +-Djobengine.nosql.db.user=db2admin +-Djobengine.nosql.db.password=password +-Djobengine.nosql.db.schema=test + +#Octopus +-Djobengine.octopus.api.address=http://10.10.1.11/api/v1 +-Djobengine.octopus.api.user=mam +-Djobengine.octopus.api.password=napocska +-Djobengine.octopus.rundowns.name=test_rundowns +-Djobengine.octopus.storyfolders.name=test_storyfolders +-Djobengine.octopus.stories.name=test_stories + +#Selenio +-Djobengine.selenio.address=http://10.10.1.71:44000/TranscodeMgrWS?wsdl +-Djobengine.selenio.projectfilepath=\\10.10.1.71\Data\Blueprints\MP4_H264_AAC.zenium + +#Nexio +-Djobengine.nexio.db.url=jdbc:sqlserver://10.10.1.59:1433;databaseName=NXDB; +-Djobengine.nexio.db.user=sa +-Djobengine.nexio.db.password=resolve +-Dnexio.useMOSGateway=true +-Dnexio.host=10.10.1.55 -Dnexio.disable=true -XstartOnFirstThread -Dorg.eclipse.swt.internal.carbon.smallFonts diff --git a/server/-product/pom.xml b/server/-product/pom.xml index 3ab00a53..cc77af1a 100644 --- a/server/-product/pom.xml +++ b/server/-product/pom.xml @@ -314,6 +314,5 @@ ${project.build.directory}/products/MediaCube/linux/gtk/x86_64 - /opt/mediacube-test diff --git a/server/user.jobengine.executors/config/scheduledjobs.json b/server/user.jobengine.executors/config/scheduledjobs.json index 8e3af0f4..044007dd 100644 --- a/server/user.jobengine.executors/config/scheduledjobs.json +++ b/server/user.jobengine.executors/config/scheduledjobs.json @@ -10,7 +10,7 @@ ] }, { - "active": true, + "active": false, "name" : "OCTOPUS adatok szinkronizálása (archivált tükrök is)", "template": "sync-octopus.xml", "executeimmediate": true, diff --git a/server/user.jobengine.executors/src/user/jobengine/server/steps/CopyForArchiveNEXIOMaterialsStep.java b/server/user.jobengine.executors/src/user/jobengine/server/steps/CopyForArchiveNEXIOMaterialsStep.java index efcbb66a..262c54ff 100644 --- a/server/user.jobengine.executors/src/user/jobengine/server/steps/CopyForArchiveNEXIOMaterialsStep.java +++ b/server/user.jobengine.executors/src/user/jobengine/server/steps/CopyForArchiveNEXIOMaterialsStep.java @@ -27,7 +27,6 @@ import user.commons.nosql.NoSQLUtils; import user.commons.octopus.IOctopusAPI; import user.commons.octopus.OctopusAPI; import user.commons.remotestore.FtpDirectoryLister; -import user.commons.remotestore.RemoteFileHandler; import user.commons.remotestore.RemoteStoreProtocol; import user.jobengine.db.IItemManager; import user.jobengine.server.IJobEngine; @@ -165,8 +164,8 @@ public class CopyForArchiveNEXIOMaterialsStep extends JobStep { return null; } - sourceFtp = ((FtpDirectoryLister) RemoteFileHandler.createLister(sourceUri)).connect(); - targetFtp = ((FtpDirectoryLister) RemoteFileHandler.createLister(targetUri)).connect(); + sourceFtp = ((FtpDirectoryLister) sourceUri.getLister()).connect(); + targetFtp = ((FtpDirectoryLister) targetUri.getLister()).connect(); processRundowns(rundowns); if (sourceUri != null) sourceUri.cleanUp(); @@ -379,7 +378,7 @@ public class CopyForArchiveNEXIOMaterialsStep extends JobStep { if (!FTPReply.isPositiveCompletion(reply)) throw new Exception("!STAT"); - String replyText = sourceFtp.getReplyString(); + //String replyText = sourceFtp.getReplyString(); //logger.info("Status: {}", replyText); if (reply == 226) { break; @@ -398,7 +397,7 @@ public class CopyForArchiveNEXIOMaterialsStep extends JobStep { OutputStream outStream = null; if (targetUri != null) targetUri.cleanUp(); - targetFtp = ((FtpDirectoryLister) RemoteFileHandler.createLister(targetUri)).connect(); + targetFtp = ((FtpDirectoryLister) targetUri.getLister()).connect(); try { if (!targetFtp.changeWorkingDirectory(EscortFiles.STATUSFOLDER)) { targetFtp.makeDirectory(EscortFiles.STATUSFOLDER); diff --git a/server/user.jobengine.executors/src/user/jobengine/server/steps/OctopusDataMiner.java b/server/user.jobengine.executors/src/user/jobengine/server/steps/OctopusDataMiner.java index cc5aa35e..6a65db58 100644 --- a/server/user.jobengine.executors/src/user/jobengine/server/steps/OctopusDataMiner.java +++ b/server/user.jobengine.executors/src/user/jobengine/server/steps/OctopusDataMiner.java @@ -76,6 +76,9 @@ public class OctopusDataMiner { private ProgressEvent progressEvent = new ProgressEvent(this, 0); private Map storyRundowns; private Map storyStoryFolders; + private Map currentRundowns; + private Map currentFolder; + private Map currentStories; private String RUNDOWN_COLLECTION; private String FOLDER_COLLECTION; private String STORY_COLLECTION; @@ -248,9 +251,14 @@ public class OctopusDataMiner { RUNDOWN_COLLECTION = IOctopusAPI.RUNDOWN_COLLECTION; FOLDER_COLLECTION = IOctopusAPI.FOLDER_COLLECTION; STORY_COLLECTION = IOctopusAPI.STORY_COLLECTION; + + //NoSQLUtils.asList((BasicDBList) + currentRundowns = getCurrentIDs(RUNDOWN_COLLECTION); + currentFolder = getCurrentIDs(FOLDER_COLLECTION); + currentStories = getCurrentIDs(STORY_COLLECTION); + BasicDBList rundowns = null; BasicDBList storyFolders = null; - try { rundowns = queryBuildRefRundowns(); storyFolders = queryBuildRefFolders(); @@ -262,9 +270,9 @@ public class OctopusDataMiner { processRundowns(rundowns); processStoryFolders(storyFolders); - deleteOrphanRundowns(); - deleteOrphanStoryFolders(); - deleteOrphanStories(); + // deleteOrphanRundowns(); + // deleteOrphanStoryFolders(); + // deleteOrphanStories(); //a sorrend fontos ! // updateDiff(MAIN_STORY_COLLECTION, STORY_COLLECTION, IOctopusAPI.ID); @@ -416,6 +424,27 @@ public class OctopusDataMiner { } } + public Map getCurrentIDs(String name) { + Map result = new HashMap<>(); + + DBCursor cursor = db.getCollection(name).find(new BasicDBObject(), new BasicDBObject(IOctopusAPI.ID, 1)); + if (cursor.hasNext()) { + List objects = ListUtils.cast(cursor.toArray()); + for (BasicDBObject obj : objects) { + if (obj == null) + continue; + long id = NoSQLUtils.asLong(obj, IOctopusAPI.ID); + if (id == 0) + continue; + Object _id = obj.getID(); + if (_id == null) + continue; + result.put(id, _id); + } + } + return result; + } + private void processRundowns(BasicDBList rundowns) throws Exception { if (rundowns == null || rundowns.size() == 0) { progressEvent.setProgress(50); @@ -544,14 +573,11 @@ public class OctopusDataMiner { progressListenerList.remove(IProgressEventListener.class, listener); } - void setObjectID(DBCollection collection, BasicDBObject objectToSave) { - BasicDBObject obj = (BasicDBObject) collection.findOne(new BasicDBObject(IOctopusAPI.ID, NoSQLUtils.asLong(objectToSave, IOctopusAPI.ID))); - if (obj == null) - return; - Object id = obj.getID(); - if (id == null) + void setObjectID(Map current, long id, BasicDBObject objectToSave) { + Object _id = current.get(id); + if (_id == null) return; - objectToSave.put(IOctopusAPI._ID, id); + objectToSave.put(IOctopusAPI._ID, _id); } private void storeRundown(BasicDBObject rundown) { @@ -568,7 +594,8 @@ public class OctopusDataMiner { rundown.put(IOctopusAPI.SCHEDULED_START, toDate(rundown, IOctopusAPI.SCHEDULED_START)); rundown.put(IOctopusAPI.MODIFIED, toDate(rundown, IOctopusAPI.MODIFIED)); DBCollection collection = db.getCollection(RUNDOWN_COLLECTION); - setObjectID(collection, rundown); + long rundownID = NoSQLUtils.asLong(rundown, IOctopusAPI.ID); + setObjectID(currentRundowns, rundownID, rundown); collection.save(rundown); } } @@ -640,7 +667,7 @@ public class OctopusDataMiner { } else story.append(IOctopusAPI.PARENT_STORY_ID, parentStoryId); logger.debug(SAVING_STORY_ID, storyID); - setObjectID(collection, story); + setObjectID(currentStories, storyID, story); collection.save(story); logger.trace(EXIT); } @@ -656,7 +683,8 @@ public class OctopusDataMiner { DBCollection collection = db.getCollection(FOLDER_COLLECTION); String name = storyFolder.getString(IOctopusAPI.NAME); logger.debug("Storing story folder {}", name); - setObjectID(collection, storyFolder); + long folderID = NoSQLUtils.asLong(storyFolder, IOctopusAPI.ID); + setObjectID(currentFolder, folderID, storyFolder); collection.save(storyFolder); } logger.trace(EXIT); diff --git a/server/user.jobengine.osgi.commons/src/user/commons/StoreUri.java b/server/user.jobengine.osgi.commons/src/user/commons/StoreUri.java index ef1cac88..3b0daa20 100644 --- a/server/user.jobengine.osgi.commons/src/user/commons/StoreUri.java +++ b/server/user.jobengine.osgi.commons/src/user/commons/StoreUri.java @@ -186,6 +186,11 @@ public class StoreUri extends EntityBase implements Serializable { return filter; } + public IDirectoryLister getLister() throws Exception { + ensureLister(); + return lister; + } + public String getPassword() { return password; } -- 2.54.0