From 5441098927a68f9e42ba1dd70a05afb1e2229b2c Mon Sep 17 00:00:00 2001 From: "vasary.daniel" Date: Fri, 25 Jun 2021 13:37:45 +0000 Subject: [PATCH] git-tfs-id: [http://tfs.userrendszerhaz.hu:8080/tfs/DefaultCollection]$/MediaCube;C32275 --- .../settings/application.yaml | 3 + .../mediacube/executors/tests/Support.java | 184 ++++++++++++------ .../037_add_disableproxy_mediafile.sql | 18 ++ .../build-metadata.launch | 19 ++ 4 files changed, 169 insertions(+), 55 deletions(-) create mode 100644 server/user.jobengine.osgi.db/migrations/scripts/037_add_disableproxy_mediafile.sql create mode 100644 server/user.mediacube.metadata/build-metadata.launch diff --git a/server/hu.user.mediacube.executors.tests/settings/application.yaml b/server/hu.user.mediacube.executors.tests/settings/application.yaml index 39e7b952..93cf8eb7 100644 --- a/server/hu.user.mediacube.executors.tests/settings/application.yaml +++ b/server/hu.user.mediacube.executors.tests/settings/application.yaml @@ -35,6 +35,9 @@ services: rundowns-collection-name: rundowns stories-collection-name: stories folders-collection-name: storyfolders +# rundowns-collection-name: test-rundowns +# stories-collection-name: test-stories +# folders-collection-name: test-storyfolders jobs: validate-transfers: false copy-buffer-size: 32768 diff --git a/server/hu.user.mediacube.executors.tests/src/hu/user/mediacube/executors/tests/Support.java b/server/hu.user.mediacube.executors.tests/src/hu/user/mediacube/executors/tests/Support.java index 58d2470d..b334efae 100644 --- a/server/hu.user.mediacube.executors.tests/src/hu/user/mediacube/executors/tests/Support.java +++ b/server/hu.user.mediacube.executors.tests/src/hu/user/mediacube/executors/tests/Support.java @@ -12,7 +12,9 @@ import java.nio.file.Paths; import java.nio.file.attribute.BasicFileAttributes; import java.sql.Timestamp; import java.util.ArrayList; +import java.util.HashSet; import java.util.List; +import java.util.Set; import org.jboss.resteasy.client.jaxrs.ResteasyClient; import org.jboss.resteasy.client.jaxrs.ResteasyClientBuilder; @@ -26,12 +28,15 @@ import com.ibm.nosql.json.api.BasicDBObject; import com.ibm.nosql.json.api.DB; import com.ibm.nosql.json.api.DBCollection; import com.ibm.nosql.json.api.DBCursor; +import com.ibm.nosql.json.api.DBObject; import com.ibm.nosql.json.api.QueryBuilder; import user.commons.ListUtils; import user.commons.RemoteFile; import user.commons.StoreUri; import user.commons.nosql.NoSQLUtils; +import user.commons.octopus.IOctopusAPI; +import user.commons.octopus.OctopusAPI; import user.commons.remotestore.RemoteStoreProtocol; import user.jobengine.db.IItemManager; import user.jobengine.db.Item; @@ -53,23 +58,27 @@ public class Support { @BeforeClass static public void setUpConnection() throws TSMException { - // System.setProperty("jobengine.octopus.rundowns.name", "test_rundowns"); - // System.setProperty("jobengine.octopus.stories.name", "test_stories"); - // System.setProperty("jobengine.octopus.storyfolders.name", "test_storyfolders"); - // System.setProperty("jobengine.octopus.rundowns.name", "rundowns180620"); - // System.setProperty("jobengine.octopus.stories.name", "stories180620"); - // System.setProperty("jobengine.octopus.storyfolders.name", "storyfolders180620"); - System.setProperty("jobengine.octopus.rundowns.name", "rundowns181217"); - System.setProperty("jobengine.octopus.stories.name", "stories181217"); - System.setProperty("jobengine.octopus.storyfolders.name", "storyfolders181217"); - - // System.setProperty("jobengine.nosql.db.url", "jdbc:db2://10.11.1.90:50000/mc:retrieveMessagesFromServerOnGetMessage=true;"); - // System.setProperty("jobengine.nosql.db.user", "db2admin"); - // System.setProperty("jobengine.nosql.db.password", "password"); - System.setProperty("nexio.host", "10.10.1.55"); - // System.setProperty(ItemManager.DBURL, "jdbc:db2://10.228.212.42:50000/mc:retrieveMessagesFromServerOnGetMessage=true;"); - // System.setProperty(ItemManager.DBUSERNAME, "blobtest"); - // System.setProperty(ItemManager.DBPASSWORD, "blobtest"); + // System.setProperty("jobengine.octopus.rundowns.name", "test_rundowns"); + // System.setProperty("jobengine.octopus.stories.name", "test_stories"); + // System.setProperty("jobengine.octopus.storyfolders.name", + // "test_storyfolders"); + // System.setProperty("jobengine.octopus.rundowns.name", "rundowns180620"); + // System.setProperty("jobengine.octopus.stories.name", "stories180620"); + // System.setProperty("jobengine.octopus.storyfolders.name", + // "storyfolders180620"); +// System.setProperty("jobengine.octopus.rundowns.name", "rundowns181217"); +// System.setProperty("jobengine.octopus.stories.name", "stories181217"); +// System.setProperty("jobengine.octopus.storyfolders.name", "storyfolders181217"); + + // System.setProperty("jobengine.nosql.db.url", + // "jdbc:db2://10.11.1.90:50000/mc:retrieveMessagesFromServerOnGetMessage=true;"); + // System.setProperty("jobengine.nosql.db.user", "db2admin"); + // System.setProperty("jobengine.nosql.db.password", "password"); +// System.setProperty("nexio.host", "10.10.1.55"); + // System.setProperty(ItemManager.DBURL, + // "jdbc:db2://10.228.212.42:50000/mc:retrieveMessagesFromServerOnGetMessage=true;"); + // System.setProperty(ItemManager.DBUSERNAME, "blobtest"); + // System.setProperty(ItemManager.DBPASSWORD, "blobtest"); // System.setProperty(ItemManager.DBURL, manager = new ItemManager(); @@ -101,7 +110,7 @@ public class Support { } stream.close(); - //check_rd_process(allFiles, allStatus); + // check_rd_process(allFiles, allStatus); } catch (Exception e) { System.err.println(e.getMessage()); } @@ -117,7 +126,8 @@ public class Support { private void check_rd_collect_status(Path mediaFilePath, List allFiles) { File file = mediaFilePath.toFile(); - if (file.isDirectory() || (!file.getName().toUpperCase().endsWith(".JSON") && !file.getName().toUpperCase().endsWith(".KILLDATE"))) + if (file.isDirectory() || (!file.getName().toUpperCase().endsWith(".JSON") + && !file.getName().toUpperCase().endsWith(".KILLDATE"))) return; if (file.getName().toUpperCase().endsWith(".KILLDATE")) { @@ -143,7 +153,8 @@ public class Support { @Test public void getJSONFile() throws Exception { Path filePath = Paths.get("//10.11.1.90/root/mediacube/data/ARCHIVE/14-03379-0092-1_.mov"); - Path json = Paths.get(Paths.get(filePath.getParent().toString(), ".STATUS", filePath.getFileName().toString()) + ".json"); + Path json = Paths.get( + Paths.get(filePath.getParent().toString(), ".STATUS", filePath.getFileName().toString()) + ".json"); System.out.println(json); } @@ -153,37 +164,43 @@ public class Support { assertNotNull(media); } - // private void check_rd_process(List allFiles, List allStatus) { - // File mediaFile = mediaFilePath.toFile(); - // if (mediaFile.isDirectory() || !mediaFile.getName().toUpperCase().endsWith(".MXF")) - // return; - // Path status = Paths.get(mediaFilePath.getParent().toString(), ".STATUS", mediaFile.getName() + ".json"); - // if (!status.toFile().exists()) { - // System.out.println("Nincs JSON: " + mediaFile.getAbsolutePath()); - // } + // private void check_rd_process(List allFiles, List allStatus) + // { + // File mediaFile = mediaFilePath.toFile(); + // if (mediaFile.isDirectory() || + // !mediaFile.getName().toUpperCase().endsWith(".MXF")) + // return; + // Path status = Paths.get(mediaFilePath.getParent().toString(), ".STATUS", + // mediaFile.getName() + ".json"); + // if (!status.toFile().exists()) { + // System.out.println("Nincs JSON: " + mediaFile.getAbsolutePath()); + // } // - // if (mediaFile.length() == 0) { - // String houseid = mediaFile.getName().substring(0, mediaFile.getName().lastIndexOf(".")); - // final String idToCheck = houseid.substring(0, houseid.lastIndexOf("-")); - // MetadataType metadataType = MetadataTypeDetector.GuessMetadataType(idToCheck); - // if (metadataType == MetadataType.OctopusPlaceholder) { - // String query = String.format("select count(*) from mediafile where houseid like '%s%%'", idToCheck); - // IResultSetConsumer consumer = rs -> { - // if (rs.getInt(1) == 0) { + // if (mediaFile.length() == 0) { + // String houseid = mediaFile.getName().substring(0, + // mediaFile.getName().lastIndexOf(".")); + // final String idToCheck = houseid.substring(0, houseid.lastIndexOf("-")); + // MetadataType metadataType = + // MetadataTypeDetector.GuessMetadataType(idToCheck); + // if (metadataType == MetadataType.OctopusPlaceholder) { + // String query = String.format("select count(*) from mediafile where houseid + // like '%s%%'", idToCheck); + // IResultSetConsumer consumer = rs -> { + // if (rs.getInt(1) == 0) { // - // File f = check_rd_nonzero_fellow(idToCheck, allFiles); - // if (f == null) - // System.out.println("0 még sincs korábbi: " + mediaFile.getAbsolutePath()); + // File f = check_rd_nonzero_fellow(idToCheck, allFiles); + // if (f == null) + // System.out.println("0 még sincs korábbi: " + mediaFile.getAbsolutePath()); // - // } - // return false; - // }; - // manager.executeQuery(query, consumer, null); - // } + // } + // return false; + // }; + // manager.executeQuery(query, consumer, null); + // } // - // } + // } // - // } + // } private void rescue_archive(RescueNEXIOMaterials processor, String clipName, long duration) throws Exception { /* @@ -194,7 +211,7 @@ public class Support { RundownArchive ra = processor.processRundow(clipName, duration); if (ra != null) { System.out.println(ra.getItemTitle()); - //processor.copy(ra); + // processor.copy(ra); } else System.out.println("No rundown for " + clipName); @@ -210,7 +227,8 @@ public class Support { DBCursor cursor = collection.find(queryBuilder.get()); int count = 0; RescueNEXIOMaterials processor = new RescueNEXIOMaterials(); - processor.set(2098, "administrator", "system", "ftp://10.10.1.100/ARCHIVE/TEST", "mediacube", "Broadca5T", 21, "ARCHIVED", manager); + processor.set(2098, "administrator", "system", "ftp://10.10.1.100/ARCHIVE/TEST", "mediacube", "Broadca5T", 21, + "ARCHIVED", manager); List clips = ListUtils.cast(cursor.toArray()); for (BasicDBObject clip : clips) { @@ -232,24 +250,26 @@ public class Support { @Test public void test_PlanAirExtensions() throws Exception { - String actual = PlanAirExtensions.getMorpeusXML(manager, "jdbc:sqlserver://10.10.1.45;databaseName=PA_Echo;", "MAM", "Echotv.hu", "M009729A", "ISILON"); + String actual = PlanAirExtensions.getMorpeusXML(manager, "jdbc:sqlserver://10.10.1.45;databaseName=PA_Echo;", + "MAM", "Echotv.hu", "M009729A", "ISILON"); assertNotNull(actual); assertTrue(actual.length() > 0); } @Test public void test_ProjectCleanupMountedLocationStep() throws Exception { - //TODO + // TODO ProjectCleanupMountedLocationStep sut = new ProjectCleanupMountedLocationStep(); sut.execute("c://_video//projects", null, new JobRuntime()); } @Test public void test_Solr() throws Exception { - //http://lucene.apache.org/solr/guide/7_5/using-solrj.html#using-solrj + // http://lucene.apache.org/solr/guide/7_5/using-solrj.html#using-solrj final String solrUrl = "http://localhost:8983/solr"; - //return new HttpSolrClient.Builder(solrUrl).withConnectionTimeout(10000).withSocketTimeout(60000).build(); + // return new + // HttpSolrClient.Builder(solrUrl).withConnectionTimeout(10000).withSocketTimeout(60000).build(); ResteasyClient client = new ResteasyClientBuilder().build(); ResteasyWebTarget webTarget = client.target("httpp:"); @@ -278,7 +298,8 @@ public class Support { Item i = manager.createItem("Generic", "Trigger teszt2", "itemdesc2", "itemhouseid2"); Media m = manager.createMedia("Generic", "mediatitle2", "mediadesc2", "mediahouseid2"); i.appendMedia(m); - MediaFile mf = manager.createMediaFile("filename2", manager.getFileType("High-res"), manager.getSystemStore(false), m); + MediaFile mf = manager.createMediaFile("filename2", manager.getFileType("High-res"), + manager.getSystemStore(false), m); m.appendMediaFile(mf); manager.mergeItemStructure(i); @@ -326,7 +347,8 @@ public class Support { @Test public void testTSM() throws Exception { - //System.setProperty("PATH", "c:\\_workspace\\USER\\MediaCube\\server\\user.tsm.client\\apilib\\;%PATH%"); + // System.setProperty("PATH", + // "c:\\_workspace\\USER\\MediaCube\\server\\user.tsm.client\\apilib\\;%PATH%"); TSMClient c = null; try { final TSMClient client = new TSMClient("JOBENGINE"); @@ -346,7 +368,8 @@ public class Support { } if (objects != null && objects.length != 1) { - System.out.println(houseId + " count: " + objects.length + " archived: " + rs.getTimestamp("archived")); + System.out.println( + houseId + " count: " + objects.length + " archived: " + rs.getTimestamp("archived")); } else System.out.println(houseId); @@ -362,4 +385,55 @@ public class Support { } + @Test + public void test_check_archived_rundownstories() throws Exception { + OctopusAPI octopus = new OctopusAPI(); + List rundowns = octopus.getRundowns(null); + + System.out.println("Reading archive"); + Set archived = new HashSet<>(); + String sql = "select mediafilehouseid from vw_items where mediatype ='Hír bejátszó'"; + manager.executeQuery(sql, rs -> { + String id = rs.getString(1); + + try { + if (id.contains("-")) + id = id.substring(0, id.lastIndexOf("-")); + else + id = id.substring(0, id.length() - 4); + } catch (Exception e) { + System.out.println(e.getMessage()); + } + + archived.add(id); + // System.out.println(id); + return true; + }, null); + + System.out.println("Processing rundowns"); + for (DBObject o : rundowns) { + BasicDBObject bo = (BasicDBObject) o; + System.out.println(bo.getLong(IOctopusAPI.ID) + " | " + bo.getString(IOctopusAPI.SCHEDULED_START) + " | " + + bo.getString(IOctopusAPI.NAME)); + List stories = octopus.getRundownFullStories(bo.getLong(IOctopusAPI.ID)); + if (stories == null) + continue; + for (DBObject s : stories) { + BasicDBObject bs = (BasicDBObject) s; + List mosObjects = NoSQLUtils.asList(bs, IOctopusAPI.MOS_OBJECTS); + if (mosObjects == null) + continue; + for (BasicDBObject mos : mosObjects) { + String mosId = mos.getString(IOctopusAPI.OBJ_ID); + if (archived.contains(mosId)) + System.out.println(mosId); + else + System.out.println("Missing " + mosId); + } + + } + } + + } + } diff --git a/server/user.jobengine.osgi.db/migrations/scripts/037_add_disableproxy_mediafile.sql b/server/user.jobengine.osgi.db/migrations/scripts/037_add_disableproxy_mediafile.sql new file mode 100644 index 00000000..cf3050a4 --- /dev/null +++ b/server/user.jobengine.osgi.db/migrations/scripts/037_add_disableproxy_mediafile.sql @@ -0,0 +1,18 @@ +-- // Add filesize column to mediafile +-- Migration SQL that makes the change goes here. + +ALTER TABLE MEDIAFILE ADD COLUMN DISABLEPROXY CHARACTER(1) NOT NULL DEFAULT 'N' +@ +ALTER TABLE MEDIAFILE ADD CONSTRAINT CHK_MEDIAFILE_DISABLEPROXY CHECK (DISABLEPROXY in ('Y', 'N')) +@ +CALL SYSPROC.ADMIN_CMD ( 'REORG TABLE DB2ADMIN.MEDIAFILE' ) +@ + +-- //@UNDO +-- SQL to undo the change goes here. +ALTER TABLE MEDIAFILE DROP CONSTRAINT CHK_MEDIAFILE_DISABLEPROXY +@ +ALTER TABLE MEDIAFILE DROP COLUMN DISABLEPROXY +@ +CALL SYSPROC.ADMIN_CMD ( 'REORG TABLE DB2ADMIN.MEDIAFILE' ) +@ diff --git a/server/user.mediacube.metadata/build-metadata.launch b/server/user.mediacube.metadata/build-metadata.launch new file mode 100644 index 00000000..3d3875ac --- /dev/null +++ b/server/user.mediacube.metadata/build-metadata.launch @@ -0,0 +1,19 @@ + + + + + + + + + + + + + + + + + + + -- 2.54.0