From 493574dfd628f3e227e8d915feda0d914e6bed70 Mon Sep 17 00:00:00 2001 From: =?utf8?q?V=C3=A1s=C3=A1ry=20D=C3=A1niel?= Date: Wed, 21 Oct 2020 05:46:16 +0000 Subject: [PATCH] git-tfs-id: [http://tfs.userrendszerhaz.hu:8080/tfs/DefaultCollection]$/MediaCube;C31988 --- .../mediacube/executors/tests/SmallTests.java | 82 +++++++++++++++++++ .../server/steps/ArchiveListBuilderStep.java | 31 +------ .../server/steps/MetadataTransformStep.java | 24 +++++- .../commons/remotestore/TSMOutputStream.java | 6 +- .../scripts/024_create_pasa_table.sql | 44 ++++++++++ .../user/jobengine/search/MediaFinder.java | 9 +- .../user.jobengine.osgi.server/pages/menu.zul | 14 ++-- .../jobengine/server/steps/ArchiveItem.java | 67 +++++++++++++++ .../ws/mediacube/MediaCubeAPIWSSocket.java | 12 ++- .../user/tsm/client/TSMBufferedClient.java | 44 ++++++---- 10 files changed, 274 insertions(+), 59 deletions(-) create mode 100644 server/user.jobengine.osgi.db/migrations/scripts/024_create_pasa_table.sql diff --git a/server/hu.user.mediacube.executors.tests/src/hu/user/mediacube/executors/tests/SmallTests.java b/server/hu.user.mediacube.executors.tests/src/hu/user/mediacube/executors/tests/SmallTests.java index bb4be753..10a0267f 100644 --- a/server/hu.user.mediacube.executors.tests/src/hu/user/mediacube/executors/tests/SmallTests.java +++ b/server/hu.user.mediacube.executors.tests/src/hu/user/mediacube/executors/tests/SmallTests.java @@ -1,5 +1,7 @@ package hu.user.mediacube.executors.tests; +import static org.junit.Assert.assertNotNull; + import java.io.File; import java.nio.file.DirectoryStream; import java.nio.file.Files; @@ -29,7 +31,10 @@ import org.apache.commons.io.FilenameUtils; import org.apache.commons.lang.StringUtils; import org.junit.Test; +import com.ibm.nosql.json.api.BasicDBList; + import user.jobengine.db.Media; +import user.jobengine.server.steps.ArchiveItem; import user.jobengine.server.steps.EscortFiles; import user.jobengine.server.steps.FFMpeg; import user.jobengine.server.steps.TSMExtendedRetrieveStep; @@ -491,4 +496,81 @@ public class SmallTests { System.out.println(modifiedHours); System.out.println(DateTimeFormatter.ISO_LOCAL_DATE_TIME.format(ldt)); } + + //mv missing from tsm + @Test + public void test993() throws Exception { + //List missing = Files.readAllLines(Paths.get("/opt/mv-missing.txt")); + List missing = Files.readAllLines(Paths.get("/opt/mv-mc-all.txt")); + String tsm = new String(Files.readAllBytes(Paths.get("/opt/mv-tsm.txt"))); + String polc = new String(Files.readAllBytes(Paths.get("/opt/mv-polc.txt"))); + String rd = new String(Files.readAllBytes(Paths.get("/opt/mv-rundown.txt"))); + String om = new String(Files.readAllBytes(Paths.get("/opt/mv-omarchive.txt"))); + int fCount = 0; + int pCount = 0; + int oCount = 0; + + int promo = 0; + int ad = 0; + int material = 0; + int other = 0; + int inrd = 0; + + System.out.println("Start"); + for (String fileName : missing) { + String onlyId = fileName.trim(); + onlyId = onlyId.substring(0, onlyId.lastIndexOf(".")); + + boolean tsmContains = tsm.contains(fileName); + + if (tsmContains) + continue; + + boolean omContains = om.contains(fileName); + if (omContains) { + oCount++; + continue; + } + boolean polcContains = polc.contains(fileName); + if (polcContains) { + pCount++; + continue; + } + + fCount++; + System.out.println(fileName + " " + fCount); + + // if (rd.contains(onlyId)) + // inrd++; + + if (fileName.startsWith("R")) { + ad++; + } else if (fileName.startsWith("P")) + promo++; + else if (fileName.startsWith("M")) { + material++; + } else { + //System.out.println(onlyId); + other++; + } + } + System.out.println(String.format("TSM hiány %d, polcon van %d, omben van %d", fCount, pCount, oCount)); + System.out.println(String.format("Egyéb %d, Reklám %d, Promó %d, Anyag %d, Adásban volt %d", other, ad, promo, material, inrd)); + // Files.write(Paths.get("/opt/AMC/mcmissing.txt"), mcmissing); + } + + //archiveitem+tags + @Test + public void test994() throws Exception { + ArchiveItem archiveItem = ArchiveItem.fromFile(Paths.get("/opt1/.STATUS/0409_2032_UJ_OVI_OVB_MIX.MXF.json")); + assertNotNull(archiveItem.getTags()); + + BasicDBList tags = archiveItem.getTags(); + if (tags != null) { + for (Object tag : tags) { + System.out.println(String.valueOf(tag)); + } + } + + } } diff --git a/server/user.jobengine.executors/src/user/jobengine/server/steps/ArchiveListBuilderStep.java b/server/user.jobengine.executors/src/user/jobengine/server/steps/ArchiveListBuilderStep.java index cf973249..b2321829 100644 --- a/server/user.jobengine.executors/src/user/jobengine/server/steps/ArchiveListBuilderStep.java +++ b/server/user.jobengine.executors/src/user/jobengine/server/steps/ArchiveListBuilderStep.java @@ -14,10 +14,6 @@ import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.apache.logging.log4j.Marker; -import com.ibm.nosql.json.JSONUtil; -import com.ibm.nosql.json.api.BasicDBObject; - -import user.commons.nosql.NoSQLUtils; import user.jobengine.server.IJobEngine; import user.jobengine.server.IJobRuntime; @@ -43,32 +39,18 @@ public class ArchiveListBuilderStep extends JobStep { public static final String MEDIA_TYPE = "mediaType"; private static final String DURATION = "duration"; private static final String EXISTING_MEDIAID = "existingMediaId"; + private static final String TAGS = "tags"; private Marker marker; private ArchiveItem createArchiveItem(Path jsonFilePath, Path mediaFilePath, Path catchedFilePath) { ArchiveItem result = null; - byte[] readAllBytes = null; try { - readAllBytes = Files.readAllBytes(jsonFilePath); - BasicDBObject dbObject = (BasicDBObject) JSONUtil.jsonToDbObject(new String(readAllBytes)); - if (dbObject == null) - throw new NullPointerException("Can not parse JSON file: " + jsonFilePath); - result = new ArchiveItem(); - result.setItemHouseId(getMetadata(dbObject, ITEM_HOUSEID)); - result.setItemTitle(getMetadata(dbObject, ITEM_TITLE)); - result.setItemDescription(getMetadata(dbObject, ITEM_DESCRIPTION)); - result.setMediaHouseId(getMetadata(dbObject, MEDIA_HOUSEID)); - result.setMediaTitle(getMetadata(dbObject, MEDIA_TITLE)); - result.setMediaDescription(getMetadata(dbObject, MEDIA_DESCRIPTION)); - result.setMediaType(getMetadata(dbObject, MEDIA_TYPE)); + result = ArchiveItem.fromFile(jsonFilePath); result.setMediaFile(mediaFilePath.toString()); result.setCatchedFile(catchedFilePath.toString()); - result.setDuration(NoSQLUtils.asLong(dbObject, DURATION)); - result.setExistingMediaId(NoSQLUtils.asLong(dbObject, EXISTING_MEDIAID)); - } catch (IOException e) { + } catch (Exception e) { logger.catching(e); - return null; } return result; @@ -122,13 +104,6 @@ public class ArchiveListBuilderStep extends JobStep { return new Object[] { archiveList }; } - private String getMetadata(BasicDBObject dbObject, String fieldName) { - String result = null; - if (dbObject.containsKey(fieldName)) - result = dbObject.getString(fieldName); - return result; - } - private boolean processPathItem(Path mediaFilePath, final List archiveList) { File mediaFile = mediaFilePath.toFile(); diff --git a/server/user.jobengine.executors/src/user/jobengine/server/steps/MetadataTransformStep.java b/server/user.jobengine.executors/src/user/jobengine/server/steps/MetadataTransformStep.java index 4bb7992f..d7085875 100644 --- a/server/user.jobengine.executors/src/user/jobengine/server/steps/MetadataTransformStep.java +++ b/server/user.jobengine.executors/src/user/jobengine/server/steps/MetadataTransformStep.java @@ -14,6 +14,8 @@ import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.apache.logging.log4j.Marker; +import com.ibm.nosql.json.api.BasicDBList; + import user.jobengine.db.Item; import user.jobengine.db.ItemManager; import user.jobengine.db.ItemType; @@ -34,6 +36,23 @@ public class MetadataTransformStep extends JobStep { private ItemManager itemManager; private Marker marker; + private void addTags(ArchiveItem archiveItem, Media mediaCubeMedia) { + BasicDBList tags = archiveItem.getTags(); + if (tags != null) { + for (Object tag : tags) { + + try { + String tagText = String.valueOf(tag); + itemManager.addMediaTag(tagText, mediaCubeMedia.getId()); + System.out.println(); + + } catch (Exception e) { + logger.catching(e); + } + } + } + } + private void checkDuplicates(ArchiveItem archiveItem, String sourceFileName) throws Exception { if (itemManager.isMediaFileExists(sourceFileName)) { try { @@ -78,6 +97,7 @@ public class MetadataTransformStep extends JobStep { mediaCubeMedia = itemManager.createMedia(mediaType, archiveItem.getMediaTitle(), archiveItem.getMediaDescription(), archiveItem.getMediaHouseId()); mediaCubeMedia.setLength(archiveItem.getDuration()); mediaCubeItem.appendMedia(mediaCubeMedia); + return mediaCubeMedia; } @@ -96,7 +116,7 @@ public class MetadataTransformStep extends JobStep { jobRuntime.incrementProgress(50); String mediaType = getCreateType(archiveItem); mediaCubeMedia = createMedia(archiveItem, mediaCubeItem, mediaType); - //ha itemid 0 akkor merge, egyebkent media insert + //ha itemid 0 akkor merge, egyebkent media insert if (mediaCubeItem.getId() == 0) itemManager.mergeItemStructure(mediaCubeItem); @@ -105,6 +125,8 @@ public class MetadataTransformStep extends JobStep { mediaCubeMedia.add(); } + addTags(archiveItem, mediaCubeMedia); + } catch (Exception e) { logger.catching(e); String fileName = new File(archiveItem.getMediaFile()).getName(); diff --git a/server/user.jobengine.osgi.commons/src/user/commons/remotestore/TSMOutputStream.java b/server/user.jobengine.osgi.commons/src/user/commons/remotestore/TSMOutputStream.java index 5a8a2541..803ba10d 100644 --- a/server/user.jobengine.osgi.commons/src/user/commons/remotestore/TSMOutputStream.java +++ b/server/user.jobengine.osgi.commons/src/user/commons/remotestore/TSMOutputStream.java @@ -28,11 +28,11 @@ public class TSMOutputStream extends OutputStream { } @Override - public void close() { + public void close() throws IOException { try { - client.finishSend(); + client.commit(); } catch (Exception e) { - e.printStackTrace(); + throw new IOException(e); } client.disconnect(); } diff --git a/server/user.jobengine.osgi.db/migrations/scripts/024_create_pasa_table.sql b/server/user.jobengine.osgi.db/migrations/scripts/024_create_pasa_table.sql new file mode 100644 index 00000000..7b37a518 --- /dev/null +++ b/server/user.jobengine.osgi.db/migrations/scripts/024_create_pasa_table.sql @@ -0,0 +1,44 @@ +-- +-- Copyright 2010-2016 the original author or authors. +-- +-- Licensed under the Apache License, Version 2.0 (the "License"); +-- you may not use this file except in compliance with the License. +-- You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, software +-- distributed under the License is distributed on an "AS IS" BASIS, +-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +-- See the License for the specific language governing permissions and +-- limitations under the License. +-- + +-- // Create TAG TAG2MEDIA tables +-- Migration SQL that makes the change goes here. + +--ID;NAME;ONTSM;EXPIRATION;CREATED;PID;FILENAME;LASTMODIFIED;SIZE;PARTS;MEDIASTORE;NOF_FRAMES;ORIGMEDIASTORE;ORIGTSMFILENAME +--1937;DK_20027300_02;Y;2109-03-02 17:27:34.765;2009-08-05 16:59:05.39;1937;DK_20027300_02;2009-08-05 16:57:25.0;2430599168;1;user.pasa.server.MSFTPK2;;; + +CREATE TABLE PASA +( + ID BIGINT, + NAME VARCHAR(100), + ONTSM VARCHAR(1), + EXPIRATION TIMESTAMP, + CREATED TIMESTAMP, + PID BIGINT, + FILENAME VARCHAR(100), + LASTMODIFIED TIMESTAMP, + PARTS INT, + MEDIASTORE VARCHAR(100), + NOF_FRAMES VARCHAR(100), + ORIGMEDIASTORE VARCHAR(100), + ORIGTSMFILENAME VARCHAR(100) +)@ + +-- //@UNDO +-- SQL to undo the change goes here. + +DROP TABLE PASA +@ diff --git a/server/user.jobengine.osgi.db/src/user/jobengine/search/MediaFinder.java b/server/user.jobengine.osgi.db/src/user/jobengine/search/MediaFinder.java index 7f5dfb41..8d5078a8 100644 --- a/server/user.jobengine.osgi.db/src/user/jobengine/search/MediaFinder.java +++ b/server/user.jobengine.osgi.db/src/user/jobengine/search/MediaFinder.java @@ -66,8 +66,13 @@ public class MediaFinder implements IMediaFinder { if (criteria.endsWith("*")) criteria = criteria.replace("*", ""); criteria = criteria.trim(); - if (criteria != null && criteria.trim().length() > 0) - sql.append(" AND m.title LIKE('%" + criteria + "%')"); + if (criteria != null && criteria.trim().length() > 0) { + sql.append(" AND ("); + sql.append(" m.title LIKE('%" + criteria + "%')"); + sql.append(" OR i.title LIKE('%" + criteria + "%')"); + sql.append(" OR mf.relativepath LIKE('%" + criteria + "%')"); + sql.append(")"); + } } else { criteria = formatSearchValue(criteria); if (criteria != null && criteria.trim().length() > 0) diff --git a/server/user.jobengine.osgi.server/pages/menu.zul b/server/user.jobengine.osgi.server/pages/menu.zul index a25c2674..9586f5bb 100644 --- a/server/user.jobengine.osgi.server/pages/menu.zul +++ b/server/user.jobengine.osgi.server/pages/menu.zul @@ -55,20 +55,18 @@ - - - - - - - - + + + + + + diff --git a/server/user.jobengine.osgi.server/src/user/jobengine/server/steps/ArchiveItem.java b/server/user.jobengine.osgi.server/src/user/jobengine/server/steps/ArchiveItem.java index 76ece446..b1f2853c 100644 --- a/server/user.jobengine.osgi.server/src/user/jobengine/server/steps/ArchiveItem.java +++ b/server/user.jobengine.osgi.server/src/user/jobengine/server/steps/ArchiveItem.java @@ -1,10 +1,66 @@ package user.jobengine.server.steps; import java.io.File; +import java.io.IOException; import java.io.Serializable; +import java.nio.file.Files; +import java.nio.file.Path; + +import com.ibm.nosql.json.JSONUtil; +import com.ibm.nosql.json.api.BasicDBList; +import com.ibm.nosql.json.api.BasicDBObject; + +import user.commons.nosql.NoSQLUtils; public class ArchiveItem implements Serializable { private static final long serialVersionUID = -7278747969491719357L; + public static final String ITEM_TITLE = "itemTitle"; + public static final String ITEM_HOUSEID = "itemHouseId"; + public static final String ITEM_DESCRIPTION = "itemDescription"; + public static final String MEDIA_HOUSEID = "mediaHouseId"; + public static final String MEDIA_TITLE = "mediaTitle"; + public static final String MEDIA_DESCRIPTION = "mediaDescription"; + public static final String MEDIA_TYPE = "mediaType"; + private static final String DURATION = "duration"; + private static final String EXISTING_MEDIAID = "existingMediaId"; + private static final String TAGS = "tags"; + + public static ArchiveItem fromFile(Path jsonFilePath) throws IOException { + ArchiveItem result = null; + byte[] readAllBytes = null; + readAllBytes = Files.readAllBytes(jsonFilePath); + BasicDBObject dbObject = (BasicDBObject) JSONUtil.jsonToDbObject(new String(readAllBytes)); + if (dbObject == null) + throw new NullPointerException("Can not parse JSON file: " + jsonFilePath); + result = new ArchiveItem(); + result.setItemHouseId(getMetadata(dbObject, ITEM_HOUSEID)); + result.setItemTitle(getMetadata(dbObject, ITEM_TITLE)); + result.setItemDescription(getMetadata(dbObject, ITEM_DESCRIPTION)); + result.setMediaHouseId(getMetadata(dbObject, MEDIA_HOUSEID)); + result.setMediaTitle(getMetadata(dbObject, MEDIA_TITLE)); + result.setMediaDescription(getMetadata(dbObject, MEDIA_DESCRIPTION)); + result.setMediaType(getMetadata(dbObject, MEDIA_TYPE)); + result.setDuration(NoSQLUtils.asLong(dbObject, DURATION)); + result.setExistingMediaId(NoSQLUtils.asLong(dbObject, EXISTING_MEDIAID)); + result.setTags(getMetadatas(dbObject, TAGS)); + + return result; + } + + private static String getMetadata(BasicDBObject dbObject, String fieldName) { + String result = null; + if (dbObject.containsKey(fieldName)) + result = dbObject.getString(fieldName); + return result; + } + + private static BasicDBList getMetadatas(BasicDBObject dbObject, String fieldName) { + BasicDBList result = null; + if (dbObject.containsKey(fieldName)) + result = (BasicDBList) dbObject.get(fieldName); + return result; + } + private String itemTitle; private String itemHouseId; private String itemDescription; @@ -17,6 +73,8 @@ public class ArchiveItem implements Serializable { private long duration; private long existingMediaId; + private BasicDBList tags; + public String getCatchedFile() { return catchedFile; } @@ -61,6 +119,10 @@ public class ArchiveItem implements Serializable { return mediaType; } + public BasicDBList getTags() { + return tags; + } + public boolean removeCatchedFile() { boolean result = false; if (catchedFile != null) { @@ -114,4 +176,9 @@ public class ArchiveItem implements Serializable { public void setMediaType(String mediaType) { this.mediaType = mediaType; } + + public void setTags(BasicDBList tags) { + this.tags = tags; + } + } diff --git a/server/user.jobengine.osgi.services/src/user/jobengine/osgi/ws/mediacube/MediaCubeAPIWSSocket.java b/server/user.jobengine.osgi.services/src/user/jobengine/osgi/ws/mediacube/MediaCubeAPIWSSocket.java index d7172107..47e62ddc 100644 --- a/server/user.jobengine.osgi.services/src/user/jobengine/osgi/ws/mediacube/MediaCubeAPIWSSocket.java +++ b/server/user.jobengine.osgi.services/src/user/jobengine/osgi/ws/mediacube/MediaCubeAPIWSSocket.java @@ -33,7 +33,9 @@ public class MediaCubeAPIWSSocket extends WebSocketAdapter { public static final String ERROR = "error"; private static final String NO_ACTION_SPECIFIED = "Missing action type."; private static final String TEMPLATE = "template"; + private static final String NAME = "name"; private static final String PARAMETERS = "parameters"; + private static final String NO_NAME_SPECIFIED = "Missing job name."; private static final String NO_TEMPLATE_SPECIFIED = "Missing template name."; private static final String NO_PARAMETERS_SPECIFIED = "Missing job parameters."; private static ObjectMapper mapper; @@ -62,7 +64,8 @@ public class MediaCubeAPIWSSocket extends WebSocketAdapter { IJobRuntime job = event.getJob(); if (job.getId() != runtime.getId()) return; - BasicDBObject response = new BasicDBObject("jobID", job.getId()).append("status", job.getStatus()).append("signal", event.getSignalType()) + BasicDBObject response = new BasicDBObject("jobID", job.getId()).append("status", job.getStatus()) + .append("signal", event.getSignalType()) .append("progress", job.getProgress()); if (JobStatus.SUSPENDED.equals(job.getStatus())) @@ -149,12 +152,17 @@ public class MediaCubeAPIWSSocket extends WebSocketAdapter { if (!data.containsKey(PARAMETERS)) throw new Exception(NO_PARAMETERS_SPECIFIED); BasicDBObject parameters = (BasicDBObject) data.get(PARAMETERS); + + if (!data.containsKey(NAME)) + throw new Exception(NO_NAME_SPECIFIED); + String name = data.getString(NAME); + IJobEngine jobengine = ComponentBinder.getJobengine(); if (parameters != null) deserializeParameters(parameters); - IJobRuntime runtime = jobengine.submit(template, null, parameters); + IJobRuntime runtime = jobengine.submit(template, name, parameters); IJobChangedListener jobChangedListener = createListener(runtime); jobengine.addJobChangedEventListener(jobChangedListener); jobChangedListeners.put(runtime.getId(), jobChangedListener); diff --git a/server/user.tsm.client/src/user/tsm/client/TSMBufferedClient.java b/server/user.tsm.client/src/user/tsm/client/TSMBufferedClient.java index bebe3628..5b95291e 100644 --- a/server/user.tsm.client/src/user/tsm/client/TSMBufferedClient.java +++ b/server/user.tsm.client/src/user/tsm/client/TSMBufferedClient.java @@ -27,11 +27,31 @@ public class TSMBufferedClient extends TSMClient { super(nodeName); } + public void abort() throws TSMException, JNIException { + + } + + public void commit() throws TSMException, JNIException { + if (dataBlk != null) { + try { + rc = TSMAPI.dsmEndSendObj(dsmHandle); + if (rc != TSMAPIConstants.DSM_RC_OK) + throw new TSMException(dsmHandle, rc); + + rc = vote(TSMAPIConstants.DSM_VOTE_COMMIT); + } catch (Exception e) { + // rc = vote(TSMAPIConstants.DSM_VOTE_ABORT); + throw e; + } + } + } + public void finishReceive() throws TSMException { // dsmEndGetObj rc = TSMAPI.dsmEndGetObj(dsmHandle); if (rc != TSMAPIConstants.DSM_RC_OK) { TSMAPI.dsmEndGetData(dsmHandle); + throw new TSMException(dsmHandle, rc); } // dsmEndGetData @@ -42,15 +62,6 @@ public class TSMBufferedClient extends TSMClient { dataBlk = null; } - public void finishSend() throws TSMException, JNIException { - if (dataBlk != null) { - rc = TSMAPI.dsmEndSendObj(dsmHandle); - if (rc != TSMAPIConstants.DSM_RC_OK) - throw new TSMException(dsmHandle, rc); - rc = vote(TSMAPIConstants.DSM_VOTE_COMMIT); - } - } - public int recievePartial(byte[] buffer, TSMObject object, PartialObjData partialObjData) throws NullPointerException, FileNotFoundException, IOException, TSMException { int result = 0; @@ -159,12 +170,15 @@ public class TSMBufferedClient extends TSMClient { logger.catching(e); } - if (result == TSMAPIConstants.DSM_RC_CHECK_REASON_CODE) { - logger.error("dsmEndTxnEx error, reason code {}", endTxnExOut.getReason()); - } else { - if (result != TSMAPIConstants.DSM_RC_OK) - throw new TSMException(dsmHandle, rc, endTxnExOut.getReason()); - } + // if (result == TSMAPIConstants.DSM_RC_CHECK_REASON_CODE) { + // logger.error("dsmEndTxnEx error, reason code {}", endTxnExOut.getReason()); + // } else { + // if (result != TSMAPIConstants.DSM_RC_OK) + // throw new TSMException(dsmHandle, rc, endTxnExOut.getReason()); + // } + + if (result != TSMAPIConstants.DSM_RC_OK) + throw new TSMException(dsmHandle, rc, endTxnExOut.getReason()); return result; } -- 2.54.0