From eda03ca83470f96bd4073e5405939f3311e6964a Mon Sep 17 00:00:00 2001 From: =?utf8?q?V=C3=A1s=C3=A1ry=20D=C3=A1niel?= Date: Fri, 29 May 2020 09:05:16 +0000 Subject: [PATCH] git-tfs-id: [http://tfs.userrendszerhaz.hu:8080/tfs/DefaultCollection]$/MediaCube;C31856 --- server/-configuration/mediacube.json | 2 +- .../run-mediacube-server-mv.launch | 2 +- .../mediacube/executors/tests/Support.java | 7 ++ .../config/config-worker.xml | 3 +- .../config/config.xml | 1 + .../jobtemplates/cancelable.xml | 3 +- .../jobtemplates/copy-from-nexio.xml | 42 ------- .../jobtemplates/fake-concurrent.xml | 2 +- .../jobtemplates/nexio-checker.xml | 19 +++ .../mediacube/steps/DownloadableArchive.java | 7 ++ .../steps/DownloadableListBuilderStep.java | 81 +++++++++++++ .../mediacube/steps/NEXIOCheckerStep.java | 112 ++++++++++++++++++ .../server/steps/CancelableStep.java | 22 +++- .../server/steps/CreateMissingLowresStep.java | 3 - .../jobengine/server/steps/EscortFiles.java | 13 +- .../server/steps/FakeNoParamsStep.java | 7 +- .../user/jobengine/server/steps/FakeStep.java | 1 + .../server/steps/HSMMigrateStep.java | 22 ++-- .../server/steps/NEXIOCheckerStep.java | 100 ---------------- .../jobengine/server/steps/TSMBackupStep.java | 3 +- .../user/jobengine/db/BreakDAO_SJProfile0.ser | Bin 3253 -> 3253 bytes .../db/DomainCategoryDAO_SJProfile0.ser | Bin 2552 -> 2552 bytes .../jobengine/db/DomainDAO_SJProfile0.ser | Bin 4043 -> 4043 bytes .../db/DomainIndexDAO_SJProfile0.ser | Bin 3143 -> 3143 bytes .../jobengine/db/EntityBaseDAO_SJProfile0.ser | Bin 2286 -> 2286 bytes .../jobengine/db/FileTypeDAO_SJProfile0.ser | Bin 3872 -> 3872 bytes .../jobengine/db/FolderDAO_SJProfile0.ser | Bin 2479 -> 2479 bytes .../jobengine/db/HelperDAO_SJProfile0.ser | Bin 1674 -> 1674 bytes .../user/jobengine/db/ItemDAO_SJProfile0.ser | Bin 4337 -> 4337 bytes .../jobengine/db/ItemTypeDAO_SJProfile0.ser | Bin 2933 -> 2933 bytes .../user/jobengine/db/JobDAO_SJProfile0.ser | Bin 4625 -> 4625 bytes .../db/JobParametersDAO_SJProfile0.ser | Bin 2355 -> 2355 bytes .../jobengine/db/MasterIdDAO_SJProfile0.ser | Bin 1655 -> 1655 bytes .../generated/user/jobengine/db/MediaDAO.java | 101 +++++++++++----- .../user/jobengine/db/MediaDAO_SJProfile0.ser | Bin 4106 -> 4847 bytes .../jobengine/db/MediaFileDAO_SJProfile0.ser | Bin 4175 -> 4175 bytes .../jobengine/db/MetadataDAO_SJProfile0.ser | Bin 6747 -> 6747 bytes .../db/MetadataElementDAO_SJProfile0.ser | Bin 2990 -> 2990 bytes .../db/MetadataTypeDAO_SJProfile0.ser | Bin 3013 -> 3013 bytes .../db/RemoteStoreDAO_SJProfile0.ser | Bin 3178 -> 3178 bytes .../db/SceneContentDAO_SJProfile0.ser | Bin 3376 -> 3376 bytes .../user/jobengine/db/SceneDAO_SJProfile0.ser | Bin 2973 -> 2973 bytes .../db/SearchDefinitionDAO_SJProfile0.ser | Bin 3528 -> 3528 bytes .../user/jobengine/db/ShotDAO_SJProfile0.ser | Bin 3487 -> 3487 bytes .../user/jobengine/db/StoreDAO_SJProfile0.ser | Bin 3164 -> 3164 bytes .../jobengine/db/StoreUriDAO_SJProfile0.ser | Bin 4756 -> 4756 bytes .../jobengine/db/UserInfoDAO_SJProfile0.ser | Bin 3637 -> 3637 bytes .../db/WorkflowActionDAO_SJProfile0.ser | Bin 4737 -> 4737 bytes .../scripts/008_missing_proxy_video_only.sql | 49 ++++++++ .../sql/5-CreateFTIndex.db2 | 2 +- .../src/user/jobengine/db/ItemManager.java | 2 + .../pages/login.zul | 2 +- .../src/user/jobengine/server/IJobEngine.java | 2 +- .../src/user/jobengine/server/JobEngine.java | 11 +- .../src/user/jobengine/server/JobRuntime.java | 33 +++--- .../server/JobStatusChangedEvent.java | 4 + .../jobengine/server/JobStepExecutor.java | 9 +- .../user/jobengine/zk/model/LoginModel.java | 6 + .../user/jobengine/server/IT/AMCTests.java | 84 +++++++++++++ 59 files changed, 535 insertions(+), 222 deletions(-) delete mode 100644 server/user.jobengine.executors/jobtemplates/copy-from-nexio.xml create mode 100644 server/user.jobengine.executors/jobtemplates/nexio-checker.xml create mode 100644 server/user.jobengine.executors/src/hu/user/mediacube/steps/DownloadableArchive.java create mode 100644 server/user.jobengine.executors/src/hu/user/mediacube/steps/DownloadableListBuilderStep.java create mode 100644 server/user.jobengine.executors/src/hu/user/mediacube/steps/NEXIOCheckerStep.java delete mode 100644 server/user.jobengine.executors/src/user/jobengine/server/steps/NEXIOCheckerStep.java create mode 100644 server/user.jobengine.osgi.db/migrations/scripts/008_missing_proxy_video_only.sql create mode 100644 server/user.jobengine.osgi.server/test/user/jobengine/server/IT/AMCTests.java diff --git a/server/-configuration/mediacube.json b/server/-configuration/mediacube.json index eecb410e..bd927894 100644 --- a/server/-configuration/mediacube.json +++ b/server/-configuration/mediacube.json @@ -75,7 +75,7 @@ ], "authentication": { "defaultUser": "root", - "defaultPassword": "password", + "defaultPassword": "password", "authEnabled": true, "adHost": "intra.mediavivantis.hu", "adNonSecurePort": 3268, diff --git a/server/-configuration/run-mediacube-server-mv.launch b/server/-configuration/run-mediacube-server-mv.launch index 5de9b7fd..4b613aea 100644 --- a/server/-configuration/run-mediacube-server-mv.launch +++ b/server/-configuration/run-mediacube-server-mv.launch @@ -22,7 +22,7 @@ - + diff --git a/server/hu.user.mediacube.executors.tests/src/hu/user/mediacube/executors/tests/Support.java b/server/hu.user.mediacube.executors.tests/src/hu/user/mediacube/executors/tests/Support.java index 76df49e9..31743295 100644 --- a/server/hu.user.mediacube.executors.tests/src/hu/user/mediacube/executors/tests/Support.java +++ b/server/hu.user.mediacube.executors.tests/src/hu/user/mediacube/executors/tests/Support.java @@ -138,6 +138,13 @@ public class Support { return null; } + @Test + public void getJSONFile() throws Exception { + Path filePath = Paths.get("//10.11.1.90/root/mediacube/data/ARCHIVE/14-03379-0092-1_.mov"); + Path json = Paths.get(Paths.get(filePath.getParent().toString(), ".STATUS", filePath.getFileName().toString()) + ".json"); + System.out.println(json); + } + @Test public void mediaByName() throws Exception { Media media = manager.getMedia("DK_10103300_01"); diff --git a/server/user.jobengine.executors/config/config-worker.xml b/server/user.jobengine.executors/config/config-worker.xml index 21490264..d4dee362 100644 --- a/server/user.jobengine.executors/config/config-worker.xml +++ b/server/user.jobengine.executors/config/config-worker.xml @@ -1,5 +1,6 @@ - + + \ No newline at end of file diff --git a/server/user.jobengine.executors/config/config.xml b/server/user.jobengine.executors/config/config.xml index 4eec324f..e0183cde 100644 --- a/server/user.jobengine.executors/config/config.xml +++ b/server/user.jobengine.executors/config/config.xml @@ -1,5 +1,6 @@ + diff --git a/server/user.jobengine.executors/jobtemplates/cancelable.xml b/server/user.jobengine.executors/jobtemplates/cancelable.xml index 689be151..37185de7 100644 --- a/server/user.jobengine.executors/jobtemplates/cancelable.xml +++ b/server/user.jobengine.executors/jobtemplates/cancelable.xml @@ -1,7 +1,6 @@ - - + \ No newline at end of file diff --git a/server/user.jobengine.executors/jobtemplates/copy-from-nexio.xml b/server/user.jobengine.executors/jobtemplates/copy-from-nexio.xml deleted file mode 100644 index b7d6194d..00000000 --- a/server/user.jobengine.executors/jobtemplates/copy-from-nexio.xml +++ /dev/null @@ -1,42 +0,0 @@ - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - - \ No newline at end of file diff --git a/server/user.jobengine.executors/jobtemplates/fake-concurrent.xml b/server/user.jobengine.executors/jobtemplates/fake-concurrent.xml index 6edbf839..db218503 100644 --- a/server/user.jobengine.executors/jobtemplates/fake-concurrent.xml +++ b/server/user.jobengine.executors/jobtemplates/fake-concurrent.xml @@ -13,7 +13,7 @@ - + diff --git a/server/user.jobengine.executors/jobtemplates/nexio-checker.xml b/server/user.jobengine.executors/jobtemplates/nexio-checker.xml new file mode 100644 index 00000000..332068af --- /dev/null +++ b/server/user.jobengine.executors/jobtemplates/nexio-checker.xml @@ -0,0 +1,19 @@ + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/server/user.jobengine.executors/src/hu/user/mediacube/steps/DownloadableArchive.java b/server/user.jobengine.executors/src/hu/user/mediacube/steps/DownloadableArchive.java new file mode 100644 index 00000000..d4a62b12 --- /dev/null +++ b/server/user.jobengine.executors/src/hu/user/mediacube/steps/DownloadableArchive.java @@ -0,0 +1,7 @@ +package hu.user.mediacube.steps; + +import com.ibm.nosql.json.api.BasicDBObject; + +public class DownloadableArchive extends BasicDBObject { + +} diff --git a/server/user.jobengine.executors/src/hu/user/mediacube/steps/DownloadableListBuilderStep.java b/server/user.jobengine.executors/src/hu/user/mediacube/steps/DownloadableListBuilderStep.java new file mode 100644 index 00000000..a7d507f8 --- /dev/null +++ b/server/user.jobengine.executors/src/hu/user/mediacube/steps/DownloadableListBuilderStep.java @@ -0,0 +1,81 @@ +package hu.user.mediacube.steps; + +import java.io.File; +import java.io.IOException; +import java.nio.file.DirectoryStream; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + +import com.ibm.nosql.json.api.BasicDBObject; + +import user.jobengine.server.IJobEngine; +import user.jobengine.server.IJobRuntime; +import user.jobengine.server.steps.JobStep; +import user.jobengine.server.steps.StepEntry; + +public class DownloadableListBuilderStep extends JobStep { + private static final Logger logger = LogManager.getLogger(); + // private static final String UTF8 = "utf-8"; + private static final String STATUSFOLDER = ".STATUS"; + private static final String JSONEXT = ".json"; + private static final String CATCHEDEXT = ".catched"; + + public static final String ITEM_TITLE = "itemTitle"; + public static final String ITEM_HOUSEID = "itemHouseId"; + public static final String ITEM_DESCRIPTION = "itemDescription"; + public static final String MEDIA_HOUSEID = "mediaHouseId"; + public static final String MEDIA_TITLE = "mediaTitle"; + public static final String MEDIA_DESCRIPTION = "mediaDescription"; + public static final String MEDIA_TYPE = "mediaType"; + private static final String DURATION = "duration"; + private static final String EXISTING_MEDIAID = "existingMediaId"; + + @StepEntry + public Object[] execute(String sourcePath, int limit, IJobEngine jobEngine, IJobRuntime jobRuntime) { + DirectoryStream directoryStream = null; + try { + directoryStream = Files.newDirectoryStream(Paths.get(sourcePath)); + for (Path p : directoryStream) { + processPathItem(p); + } + } catch (Exception e) { + logger.error(getSessionMarker(), e.getMessage()); + } finally { + if (directoryStream != null) { + try { + directoryStream.close(); + } catch (IOException e) { + } + } + } + + return null; + } + + private String getMetadata(BasicDBObject dbObject, String fieldName) { + String result = null; + if (dbObject.containsKey(fieldName)) + result = dbObject.getString(fieldName); + return result; + } + + private boolean processPathItem(Path mediaFilePath) { + File mediaFile = mediaFilePath.toFile(); + + // if (mediaFile.length() > 0) + // return false; + + if (mediaFile.isDirectory()) { + return false; + } + + Path dotStorePath = Paths.get(mediaFilePath.getParent().toString(), STATUSFOLDER); + Path catchedFilePath = Paths.get(dotStorePath.toString(), mediaFile.getName() + CATCHEDEXT); + File catchedFile = catchedFilePath.toFile(); + return true; + } +} diff --git a/server/user.jobengine.executors/src/hu/user/mediacube/steps/NEXIOCheckerStep.java b/server/user.jobengine.executors/src/hu/user/mediacube/steps/NEXIOCheckerStep.java new file mode 100644 index 00000000..ec8177d1 --- /dev/null +++ b/server/user.jobengine.executors/src/hu/user/mediacube/steps/NEXIOCheckerStep.java @@ -0,0 +1,112 @@ +package hu.user.mediacube.steps; + +import java.io.IOException; +import java.sql.Timestamp; +import java.util.Iterator; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + +import user.commons.StoreUri; +import user.commons.nexio.api.Clip; +import user.commons.nexio.api.ClipNotFoundException; +import user.commons.nexio.api.Controller; +import user.commons.nexio.api.Mediabase; +import user.commons.nexio.server.protocol.ProtocolException; +import user.commons.remotestore.RemoteStoreProtocol; +import user.jobengine.db.Media; +import user.jobengine.db.Store; +import user.jobengine.server.steps.EscortFiles; +import user.jobengine.server.steps.JobStep; +import user.jobengine.server.steps.StepEntry; + +public class NEXIOCheckerStep extends JobStep { + private static final String DOT_MXF = ".MXF"; + private static final Logger logger = LogManager.getLogger(); + + private DownloadableArchive createArchive(String title, Timestamp modified, Timestamp created, int duration) { + DownloadableArchive archive = new DownloadableArchive(); + archive.append("title", title); + archive.append("fileName", title + DOT_MXF); + archive.append("modified", modified); + archive.append("created", created); + archive.append("duration", duration); + return archive; + } + + @StepEntry + public Object[] execute(String storeName, String outputPath, int limit) throws Exception { + Store store = getManager().getStore(storeName); + if (store == null) + throw new NullPointerException("Store not available: " + storeName); + + StoreUri storeUri = store.getSourceStoreUri(RemoteStoreProtocol.NEXIO); + if (storeUri == null) + throw new NullPointerException("NEXIO StoreUri not available on: " + storeName); + + Controller controller = null; + + try { + if (storeUri.getPortNumber() == 0) + controller = new Controller(storeUri.getRootPath(), storeUri.getPortNumber()); + else + controller = new Controller(storeUri.getRootPath()); + controller.connect(); + Mediabase mediabase = controller.getMediabase(); + Iterator clips = mediabase.getClips(); + processClips(outputPath, limit, clips); + } catch (Exception e) { + logger.error(getSessionMarker(), e.getMessage()); + throw e; + } finally { + if (controller != null) + controller.disconnect(); + } + + return null; + } + + private void processClips(String outputPath, int limit, Iterator clips) throws ClipNotFoundException, IOException, ProtocolException { + int count = limit; + while (clips.hasNext()) { + + //megszakithatosag + if (getJobRuntime().isWaitingCancel()) + break; + + Clip clip = clips.next(); + String title = clip.getXid().get(); + Timestamp modified = Timestamp.from(clip.getModifiedTimestamp().toInstant()); + Timestamp created = Timestamp.from(clip.getRecordDateTimestamp().toInstant()); + int duration = clip.getDuration(); + + Media media = getManager().getMedia(title); + boolean create = media == null || media.getLength() != duration || !media.getModified().equals(modified) || !media.getCreated().equals(created); + if (create) { + DownloadableArchive archive = createArchive(title, modified, created, duration); + //letezik : mar hozzadva + if (tryAdd(outputPath, archive)) + count--; + } + + //vegtelen : limit=-1 + if (count == 0) { + logger.info("Limit reached: {}", limit); + break; + } + + } + } + + private boolean tryAdd(String outputPath, DownloadableArchive archive) throws IOException { + boolean result = false; + String fileName = archive.getString("fileName"); + if (!EscortFiles.isMetadataExists(outputPath, fileName)) { + String metadata = archive.toPrettyString(""); + EscortFiles.createMetadata(outputPath, fileName, metadata); + result = true; + } + return result; + } + +} diff --git a/server/user.jobengine.executors/src/user/jobengine/server/steps/CancelableStep.java b/server/user.jobengine.executors/src/user/jobengine/server/steps/CancelableStep.java index 79e0a57b..67abe142 100644 --- a/server/user.jobengine.executors/src/user/jobengine/server/steps/CancelableStep.java +++ b/server/user.jobengine.executors/src/user/jobengine/server/steps/CancelableStep.java @@ -1,15 +1,35 @@ package user.jobengine.server.steps; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + +import com.fasterxml.jackson.databind.ObjectMapper; + +import user.commons.rest.ServiceObjectMapper; + public class CancelableStep extends JobStep { + private static final Logger logger = LogManager.getLogger(); int count = 10; + private ObjectMapper mapper = ServiceObjectMapper.getMapper(); @StepEntry public Object[] execute() throws Exception { + + // DownloadableArchive a = new DownloadableArchive(); + // Media media = getManager().createMedia("Generic", "Name", "description", "houseId"); + // a.setMedia(media); + // + // String m1 = mapper.writeValueAsString(media); + // Media m2 = mapper.readValue(m1, Media.class); + // + // String prettyString = mapper.writeValueAsString(a); + // DownloadableArchive b = mapper.readValue(prettyString, DownloadableArchive.class); + // Media media2 = b.getMedia(); try { for (int i = 0; i < count; i++) { if (getJobRuntime().isWaitingCancel()) break; - Thread.sleep(1000); + Thread.sleep(100); setProgress((i + 1) * count); } } catch (Exception e) { diff --git a/server/user.jobengine.executors/src/user/jobengine/server/steps/CreateMissingLowresStep.java b/server/user.jobengine.executors/src/user/jobengine/server/steps/CreateMissingLowresStep.java index 9194123d..339d0417 100644 --- a/server/user.jobengine.executors/src/user/jobengine/server/steps/CreateMissingLowresStep.java +++ b/server/user.jobengine.executors/src/user/jobengine/server/steps/CreateMissingLowresStep.java @@ -14,14 +14,12 @@ import user.commons.MediaCubeUndoMarker; import user.commons.nosql.NoSQLUtils; import user.jobengine.db.IItemManager; import user.jobengine.db.Media; -import user.jobengine.db.Store; import user.jobengine.server.IJobEngine; import user.jobengine.server.IJobRuntime; public class CreateMissingLowresStep extends JobStep { private static final Logger logger = LogManager.getLogger(); private MediaCubeMarker marker; - private Store highResStore; @StepEntry public Object[] execute(String localHiresPath, IJobEngine jobEngine, IJobRuntime jobRuntime) throws Exception { @@ -30,7 +28,6 @@ public class CreateMissingLowresStep extends JobStep { DB db = NoSQLUtils.getNoSQLDB(); DBCollection collection = db.getCollection("missing_lowres"); IItemManager manager = jobEngine.getItemManager(); - highResStore = manager.getSystemStore(false); Media media = getFirstUntranscodedMedia(manager, collection); try { diff --git a/server/user.jobengine.executors/src/user/jobengine/server/steps/EscortFiles.java b/server/user.jobengine.executors/src/user/jobengine/server/steps/EscortFiles.java index 298b456d..0662e64f 100644 --- a/server/user.jobengine.executors/src/user/jobengine/server/steps/EscortFiles.java +++ b/server/user.jobengine.executors/src/user/jobengine/server/steps/EscortFiles.java @@ -37,6 +37,7 @@ import user.commons.StoreUri; import user.commons.remotestore.FtpDirectoryLister; public class EscortFiles { + private static final String DOT_JSON = ".json"; private static final Logger logger = LogManager.getLogger(); private static final String EXTENDEDAGENCY = "ExtendedAgency"; private static final String EXTENDEDDESCRIPTION = "ExtendedDescription"; @@ -65,9 +66,9 @@ public class EscortFiles { return String.format(KILLDATE_FILENAME, fileName, composeKillDate(days)); } - public static void createMetadata(String filePath, String fileName, String metadata, Marker marker) throws IOException { + public static void createMetadata(String filePath, String fileName, String metadata) throws IOException { ensureUNCFolder(filePath, STATUSFOLDER); - String metadataFileName = fileName + ".json"; + String metadataFileName = fileName + DOT_JSON; Path metadataPath = Paths.get(filePath, STATUSFOLDER, metadataFileName); Files.write(metadataPath, metadata.getBytes()); } @@ -149,6 +150,14 @@ public class EscortFiles { ensureUNCFolder(statusPath); } + public static boolean isMetadataExists(String filePath, String fileName) throws IOException { + boolean result = false; + String metadataFileName = fileName + DOT_JSON; + Path metadataPath = Paths.get(filePath, STATUSFOLDER, metadataFileName); + result = metadataPath.toFile().exists(); + return result; + } + public static void setNEXIOKillDate(int killDateDays, String targetFileName, String nexioAgency, StoreUri targetUri) throws Exception { OutputStream outStream = null; try { diff --git a/server/user.jobengine.executors/src/user/jobengine/server/steps/FakeNoParamsStep.java b/server/user.jobengine.executors/src/user/jobengine/server/steps/FakeNoParamsStep.java index cdf70a2f..1c17c973 100644 --- a/server/user.jobengine.executors/src/user/jobengine/server/steps/FakeNoParamsStep.java +++ b/server/user.jobengine.executors/src/user/jobengine/server/steps/FakeNoParamsStep.java @@ -3,13 +3,13 @@ package user.jobengine.server.steps; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; -import user.commons.ListUtils; import user.jobengine.server.IJobEngine; import user.jobengine.server.IJobRuntime; public class FakeNoParamsStep extends JobStep { private static final String CHILD_TITLE = "Párhuzamosított alfolyamat"; - private static final String CHILD_TEMPLATE = "fake-concurrent.xml"; + // private static final String CHILD_TEMPLATE = "fake-concurrent.xml"; + private static final String CHILD_TEMPLATE = "cancelable.xml"; private static final Logger logger = LogManager.getLogger(); int count = 2; @@ -17,7 +17,8 @@ public class FakeNoParamsStep extends JobStep { public Object[] execute(IJobEngine jobEngine, IJobRuntime jobRuntime) throws Exception { if (jobRuntime.forkPrepare()) { for (int i = 0; i < count; i++) { - jobEngine.submit(jobRuntime, null, CHILD_TEMPLATE, CHILD_TITLE, ListUtils.asMap("itemID", i)); + //jobEngine.submit(jobRuntime, null, CHILD_TEMPLATE, CHILD_TITLE, ListUtils.asMap("itemID", i)); + jobEngine.submit(jobRuntime, null, CHILD_TEMPLATE, CHILD_TITLE, null); } } diff --git a/server/user.jobengine.executors/src/user/jobengine/server/steps/FakeStep.java b/server/user.jobengine.executors/src/user/jobengine/server/steps/FakeStep.java index aa9d1540..cd9d1da6 100644 --- a/server/user.jobengine.executors/src/user/jobengine/server/steps/FakeStep.java +++ b/server/user.jobengine.executors/src/user/jobengine/server/steps/FakeStep.java @@ -15,6 +15,7 @@ public class FakeStep extends JobStep { logger.info(getMarker(), "Starting params: {}"); try { + getJobRuntime().setDescription("Teszt"); int step = 100 / count; for (int i = 0; i < count; i++) { if (!canContinue()) diff --git a/server/user.jobengine.executors/src/user/jobengine/server/steps/HSMMigrateStep.java b/server/user.jobengine.executors/src/user/jobengine/server/steps/HSMMigrateStep.java index b5df0ee4..975c87a0 100644 --- a/server/user.jobengine.executors/src/user/jobengine/server/steps/HSMMigrateStep.java +++ b/server/user.jobengine.executors/src/user/jobengine/server/steps/HSMMigrateStep.java @@ -216,14 +216,6 @@ public class HSMMigrateStep extends JobStep { return false; } - IItemManager manager = getManager(); - MediaFileDAO mfDAO = (MediaFileDAO) manager.getBaseDAO(MediaFile.class); - List mediaFiles = mfDAO.getByHouseId(source.getFileName().toString()); - if (mediaFiles != null && mediaFiles.size() > 0) { - logger.warn(marker, "Már archivált: {}", hsmFileName); - return false; - } - File sourceFile = source.toFile(); File targetFile = target.toFile(); @@ -267,9 +259,17 @@ public class HSMMigrateStep extends JobStep { private boolean processHSMFile(String volumeName, String hsmFileName, Path sourceFilePath, Path targetFilePath) throws Exception { int repeat = 4; - boolean successCopy = true; + boolean successCopy = false; + + IItemManager manager = getManager(); + MediaFileDAO mfDAO = (MediaFileDAO) manager.getBaseDAO(MediaFile.class); + List mediaFiles = mfDAO.getByHouseId(sourceFilePath.getFileName().toString()); + if (mediaFiles != null && mediaFiles.size() > 0) { + logger.warn(marker, "Már archivált: {}", hsmFileName); + return false; + } + if (prepareCopy(hsmFileName, sourceFilePath, targetFilePath)) { - successCopy = false; while (repeat > 0) { try { resumeableCopy(sourceFilePath, targetFilePath); @@ -287,7 +287,7 @@ public class HSMMigrateStep extends JobStep { String metadata = null; try { metadata = createMetadata(volumeName, hsmFileName).toPrettyString(""); - EscortFiles.createMetadata(targetFilePath.getParent().toString(), targetFilePath.getFileName().toString(), metadata, marker); + EscortFiles.createMetadata(targetFilePath.getParent().toString(), targetFilePath.getFileName().toString(), metadata); //saveFileHistory(contentFileName); } catch (Exception e) { diff --git a/server/user.jobengine.executors/src/user/jobengine/server/steps/NEXIOCheckerStep.java b/server/user.jobengine.executors/src/user/jobengine/server/steps/NEXIOCheckerStep.java deleted file mode 100644 index fc1a8e07..00000000 --- a/server/user.jobengine.executors/src/user/jobengine/server/steps/NEXIOCheckerStep.java +++ /dev/null @@ -1,100 +0,0 @@ -package user.jobengine.server.steps; - -import java.sql.Timestamp; -import java.time.Instant; -import java.util.ArrayList; -import java.util.Arrays; -import java.util.Iterator; -import java.util.List; - -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; - -import user.commons.StoreUri; -import user.commons.nexio.api.Clip; -import user.commons.nexio.api.Controller; -import user.commons.nexio.api.Mediabase; -import user.commons.remotestore.RemoteStoreProtocol; -import user.jobengine.db.IItemManager; -import user.jobengine.db.Media; -import user.jobengine.db.Store; - -public class NEXIOCheckerStep extends JobStep { - private static final Logger logger = LogManager.getLogger(); - - @StepEntry - public Object[] execute(String storeName, int limit) throws Exception { - List newItems = new ArrayList<>(); - List updatedItems = new ArrayList<>(); - Object[] result = Arrays.asList(newItems, updatedItems).toArray(); - IItemManager manager = getManager(); - - Store store = manager.getStore(storeName); - if (store == null) - throw new NullPointerException("Store not available: " + storeName); - - StoreUri storeUri = store.getSourceStoreUri(RemoteStoreProtocol.NEXIO); - if (storeUri == null) - throw new NullPointerException("NEXIO StoreUri not available on: " + storeName); - - Controller controller = null; - - try { - controller = new Controller(storeUri.getRootPath(), storeUri.getPortNumber()); - controller.connect(); - Mediabase mediabase = controller.getMediabase(); - Iterator clips = mediabase.getClips(); - int count = limit; - while (clips.hasNext()) { - - if (getJobRuntime().isWaitingCancel()) - break; - - Clip clip = clips.next(); - String title = clip.getXid().get(); - Timestamp modified = Timestamp.from(clip.getModifiedTimestamp().toInstant()); - Timestamp created = Timestamp.from(clip.getRecordDateTimestamp().toInstant()); - int duration = clip.getDuration(); - - Media media = manager.getMedia(title); - if (media == null) { - media = new Media(); - media.setModified(modified); - media.setCreated(created); - media.setArchived(Timestamp.from(Instant.now())); - media.setTitle(title); - media.setDescription(title); - media.setLength(duration); - newItems.add(media); - count--; - } else { - if (media.getLength() != duration || !media.getModified().equals(modified)) { - media.setArchived(Timestamp.from(Instant.now())); - media.setModified(modified); - media.setLength(duration); - updatedItems.add(media); - count--; - } - } - - //ha volt limit, akkor egyszer csak 0 lesz, ha nem volt akkor alapbol -1 - if (count == 0) { - logger.info("Limit reached: {}", limit); - break; - } - - } - } catch (Exception e) { - logger.error(e); - } finally { - if (controller != null) - controller.disconnect(); - } - - if (newItems.size() == 0 && updatedItems.size() == 0) - cancel(); - - return result; - } - -} diff --git a/server/user.jobengine.executors/src/user/jobengine/server/steps/TSMBackupStep.java b/server/user.jobengine.executors/src/user/jobengine/server/steps/TSMBackupStep.java index b7e82a36..b16fc1a3 100644 --- a/server/user.jobengine.executors/src/user/jobengine/server/steps/TSMBackupStep.java +++ b/server/user.jobengine.executors/src/user/jobengine/server/steps/TSMBackupStep.java @@ -52,8 +52,9 @@ public class TSMBackupStep extends JobStep { setAndCheck(archiveItem, mediaCubeMedia, getEngine()); - getJobRuntime().setDescription(String.format("%s: %s", getJobRuntime().getDescription(), details)); + getJobRuntime().setDescription(details); + //TODO mxf helyett az osszes kiterjesztest!!!!! //A dupla ellenorzes a napon beluli ismetlesek miatt kell long existingMediaId = archiveItem.getExistingMediaId(); if (sourceMediaFile.length() == 0 && existingMediaId == 0) { diff --git a/server/user.jobengine.osgi.db/generated/user/jobengine/db/BreakDAO_SJProfile0.ser b/server/user.jobengine.osgi.db/generated/user/jobengine/db/BreakDAO_SJProfile0.ser index c5c009bb6025fb7db5e9994889f133b7a4061211..c372d0268fb1a9c11bb6ba826e4cdba0a1898b36 100644 GIT binary patch delta 18 Zcmdlgxm9ul4*vHsf1&4P?exB);p27Lek diff --git a/server/user.jobengine.osgi.db/generated/user/jobengine/db/DomainCategoryDAO_SJProfile0.ser b/server/user.jobengine.osgi.db/generated/user/jobengine/db/DomainCategoryDAO_SJProfile0.ser index 7be7747e98400f7acf20f62bd1e3535cbc2922f0..de4d57b90d7aa78dca4168dbc35051117fb96771 100644 GIT binary patch delta 18 Zcmew%{6lyH4*vHsebn*|x4a{vHIq6b$1 diff --git a/server/user.jobengine.osgi.db/generated/user/jobengine/db/DomainDAO_SJProfile0.ser b/server/user.jobengine.osgi.db/generated/user/jobengine/db/DomainDAO_SJProfile0.ser index 2d2deeabd878312234a52108adca01023802a9fc..0f3df58daf4c9d1c27f1dfe058cebadf926524d1 100644 GIT binary patch delta 18 ZcmX>te_DP64te_DP64*vHsc>n*|wn@c{ruE(ZMo diff --git a/server/user.jobengine.osgi.db/generated/user/jobengine/db/DomainIndexDAO_SJProfile0.ser b/server/user.jobengine.osgi.db/generated/user/jobengine/db/DomainIndexDAO_SJProfile0.ser index 5b5d4de8b1ff0d6b80788fbd5ac76ecd058cb7fb..f44597f0bda0c01c865f56737e8071c9686401cd 100644 GIT binary patch delta 18 ZcmX>uaa>{p4uaa>{p4*vHsf4n*|xoxB)`_25SHS diff --git a/server/user.jobengine.osgi.db/generated/user/jobengine/db/EntityBaseDAO_SJProfile0.ser b/server/user.jobengine.osgi.db/generated/user/jobengine/db/EntityBaseDAO_SJProfile0.ser index 97e9257caec9efafd54b1ef9bb6e8dd0d287d7f9..676f061df7fe9cd310f8d24f930a2e6eb1ab263e 100644 GIT binary patch delta 92 zcmaDS_)c&G4?87(HU+b}WMO+LV8$Hedl%=th00GmCd f_9S*&MguU@7sLz#GqXU*vHsefn*|v|nHh~Yr?bpqWHg_|Zo|Z2JNW>c9TUSnFz4aq18nw; gypz~%8HK=1eGt;M1& diff --git a/server/user.jobengine.osgi.db/generated/user/jobengine/db/FileTypeDAO_SJProfile0.ser b/server/user.jobengine.osgi.db/generated/user/jobengine/db/FileTypeDAO_SJProfile0.ser index e096a8d7ee5f74b17d52d8455be59f8403c5aa10..bdcf6fa6748f84111440eed851fdc79e4dd4c847 100644 GIT binary patch delta 18 ZcmZ1=w?J+K4W*vHsd4n*|y9`2aqu1{MGS diff --git a/server/user.jobengine.osgi.db/generated/user/jobengine/db/FolderDAO_SJProfile0.ser b/server/user.jobengine.osgi.db/generated/user/jobengine/db/FolderDAO_SJProfile0.ser index b0eca6807b6fc1567738d0d04e8c8f67f4016b0b..3e7ddf78e78c4b92d4e627a93efc687019ac9e45 100644 GIT binary patch delta 18 ZcmZ24yk2+%4*vHsdKn*|x?aR2~7kOr>+ diff --git a/server/user.jobengine.osgi.db/generated/user/jobengine/db/HelperDAO_SJProfile0.ser b/server/user.jobengine.osgi.db/generated/user/jobengine/db/HelperDAO_SJProfile0.ser index 258f6ef9ee24c4ae40d93652493f2fc976bd4ddd..5cc27e39283526a1587dae724336fbe3ca97f243 100644 GIT binary patch delta 18 ZcmeC;?c&|Q!^m1>%hguAS&*@k6#y`Y1m*w$ delta 18 ZcmeC;?c&|Q!^j$ctiSfcW%_yIye25SHS delta 18 acmeyU_)&2K4*vHse7n*|vk@B;u$3kRwI diff --git a/server/user.jobengine.osgi.db/generated/user/jobengine/db/ItemTypeDAO_SJProfile0.ser b/server/user.jobengine.osgi.db/generated/user/jobengine/db/ItemTypeDAO_SJProfile0.ser index 288ed7ae6335c3690d19121f7ac037930abaa457..c0d5873b00dbb65dd2ed9a0d0e69dddb2e4c965c 100644 GIT binary patch delta 18 Zcmew=_El^H4*vHsdun*|wDxByCi2PXgk diff --git a/server/user.jobengine.osgi.db/generated/user/jobengine/db/JobDAO_SJProfile0.ser b/server/user.jobengine.osgi.db/generated/user/jobengine/db/JobDAO_SJProfile0.ser index b71a6ec619669d6414bb1e9f3a48f1284b5bc6f3..75ce30f041e83c33922bc4e8efbb4e4da182a638 100644 GIT binary patch delta 17 YcmbQJGErp%4*vHsexiv<}40XQE982|tP diff --git a/server/user.jobengine.osgi.db/generated/user/jobengine/db/JobParametersDAO_SJProfile0.ser b/server/user.jobengine.osgi.db/generated/user/jobengine/db/JobParametersDAO_SJProfile0.ser index 2a5aeb0247cde724a23d3f191b6b0796cdfe683f..33857e114a97d3621f503f353587e255da3d0fb3 100644 GIT binary patch delta 18 Zcmdliv{`5a4*vHsdWn*|w_H~>K)25A5Q diff --git a/server/user.jobengine.osgi.db/generated/user/jobengine/db/MasterIdDAO_SJProfile0.ser b/server/user.jobengine.osgi.db/generated/user/jobengine/db/MasterIdDAO_SJProfile0.ser index 38eb45bdf0f74d575e9e123fa8d0fa8b507a18fb..8b3f2052daaf8139131face98a1482ae4c1d75a8 100644 GIT binary patch delta 18 Zcmey)^POh{4*vHseBn*|xuSOH1-2Pgml diff --git a/server/user.jobengine.osgi.db/generated/user/jobengine/db/MediaDAO.java b/server/user.jobengine.osgi.db/generated/user/jobengine/db/MediaDAO.java index 8c38cf34..5a539709 100644 --- a/server/user.jobengine.osgi.db/generated/user/jobengine/db/MediaDAO.java +++ b/server/user.jobengine.osgi.db/generated/user/jobengine/db/MediaDAO.java @@ -18,6 +18,7 @@ extends sqlj.runtime.ref.ResultSetIterImpl implements sqlj.runtime.NamedIterator { private int posterNdx; + private int modifiedNdx; private int archivedNdx; private int createdNdx; private int lengthNdx; @@ -40,6 +41,7 @@ implements sqlj.runtime.NamedIterator lengthNdx = findColumn("length"); createdNdx = findColumn("created"); archivedNdx = findColumn("archived"); + modifiedNdx = findColumn("modified"); posterNdx = findColumn("poster"); } public MediaIter(sqlj.runtime.profile.RTResultSet resultSet, int fetchSize, int maxRows) @@ -55,6 +57,7 @@ implements sqlj.runtime.NamedIterator lengthNdx = findColumn("length"); createdNdx = findColumn("created"); archivedNdx = findColumn("archived"); + modifiedNdx = findColumn("modified"); posterNdx = findColumn("poster"); } public long id() @@ -102,6 +105,11 @@ implements sqlj.runtime.NamedIterator { return resultSet.getTimestamp(archivedNdx); } + public Timestamp modified() + throws java.sql.SQLException + { + return resultSet.getTimestamp(modifiedNdx); + } public byte[] poster() throws java.sql.SQLException { @@ -112,7 +120,7 @@ implements sqlj.runtime.NamedIterator // ************************************************************ -/*@lineinfo:user-code*//*@lineinfo:10^178*/ +/*@lineinfo:user-code*//*@lineinfo:10^198*/ @SuppressWarnings("unused") class MediaDAO extends EntityBaseDAO { @@ -167,6 +175,7 @@ class MediaDAO extends EntityBaseDAO { entity.setLength(iterator.length()); entity.setCreated(iterator.created()); entity.setArchived(iterator.archived()); + entity.setModified(iterator.modified()); entity.setPoster(iterator.poster()); if (result == null) @@ -180,10 +189,10 @@ class MediaDAO extends EntityBaseDAO { @Override protected ResultSetIterImpl selectByKey(DefaultContext context, long id) throws SQLException{ MediaIter iter = null; - /*@lineinfo:generated-code*//*@lineinfo:78^2*/ + /*@lineinfo:generated-code*//*@lineinfo:79^2*/ // ************************************************************ -// #sql [context] iter = { SELECT ID, TITLE, DESCRIPTION, ITEMID, ITEMTYPEID, HOUSEID, LENGTH, CREATED, ARCHIVED, POSTER FROM MEDIA WHERE ID = :id }; +// #sql [context] iter = { SELECT ID, TITLE, DESCRIPTION, ITEMID, ITEMTYPEID, HOUSEID, LENGTH, CREATED, ARCHIVED, MODIFIED, POSTER FROM MEDIA WHERE ID = :id }; // ************************************************************ { @@ -208,17 +217,17 @@ class MediaDAO extends EntityBaseDAO { // ************************************************************ -/*@lineinfo:user-code*//*@lineinfo:78^146*/ +/*@lineinfo:user-code*//*@lineinfo:79^156*/ return iter; } @Override protected ResultSetIterImpl selectByForeignKey(DefaultContext context, long id) throws SQLException{ MediaIter iter = null; - /*@lineinfo:generated-code*//*@lineinfo:85^2*/ + /*@lineinfo:generated-code*//*@lineinfo:86^2*/ // ************************************************************ -// #sql [context] iter = { SELECT ID, TITLE, DESCRIPTION, ITEMID, ITEMTYPEID, HOUSEID, LENGTH, CREATED, ARCHIVED, POSTER FROM MEDIA WHERE ITEMID = :id }; +// #sql [context] iter = { SELECT ID, TITLE, DESCRIPTION, ITEMID, ITEMTYPEID, HOUSEID, LENGTH, CREATED, ARCHIVED, MODIFIED, POSTER FROM MEDIA WHERE ITEMID = :id }; // ************************************************************ { @@ -243,17 +252,16 @@ class MediaDAO extends EntityBaseDAO { // ************************************************************ -/*@lineinfo:user-code*//*@lineinfo:85^150*/ +/*@lineinfo:user-code*//*@lineinfo:86^160*/ return iter; } - @Override - protected ResultSetIterImpl selectAll(DefaultContext context) throws SQLException{ + public ResultSetIterImpl selectByTitle(DefaultContext context, String title) throws SQLException{ MediaIter iter = null; /*@lineinfo:generated-code*//*@lineinfo:92^2*/ // ************************************************************ -// #sql [context] iter = { SELECT ID, TITLE, DESCRIPTION, ITEMID, ITEMTYPEID, HOUSEID, LENGTH, CREATED, ARCHIVED, POSTER FROM MEDIA }; +// #sql [context] iter = { SELECT ID, TITLE, DESCRIPTION, ITEMID, ITEMTYPEID, HOUSEID, LENGTH, CREATED, ARCHIVED, MODIFIED, POSTER FROM MEDIA WHERE TITLE = :title }; // ************************************************************ { @@ -264,6 +272,41 @@ class MediaDAO extends EntityBaseDAO { synchronized (__sJT_execCtx) { sqlj.runtime.profile.RTStatement __sJT_stmt = __sJT_execCtx.registerStatement(__sJT_connCtx, MediaDAO_SJProfileKeys.getKey(0), 2); try + { + __sJT_stmt.setString(1, title); + iter = new MediaIter(__sJT_execCtx.executeQuery(), __sJT_execCtx.getFetchSize(), __sJT_execCtx.getMaxRows()); + } + finally + { + __sJT_execCtx.releaseStatement(); + } + } +} + + +// ************************************************************ + +/*@lineinfo:user-code*//*@lineinfo:92^162*/ + return iter; + } + + @Override + protected ResultSetIterImpl selectAll(DefaultContext context) throws SQLException{ + MediaIter iter = null; + /*@lineinfo:generated-code*//*@lineinfo:99^2*/ + +// ************************************************************ +// #sql [context] iter = { SELECT ID, TITLE, DESCRIPTION, ITEMID, ITEMTYPEID, HOUSEID, LENGTH, CREATED, ARCHIVED, MODIFIED, POSTER FROM MEDIA }; +// ************************************************************ + +{ + sqlj.runtime.ConnectionContext __sJT_connCtx = context; + if (__sJT_connCtx == null) sqlj.runtime.error.RuntimeRefErrors.raise_NULL_DEFAULT_CONN_CTX(); + sqlj.runtime.ExecutionContext __sJT_execCtx = __sJT_connCtx.getExecutionContext(); + if (__sJT_execCtx == null) sqlj.runtime.error.RuntimeRefErrors.raise_NULL_EXEC_CTX(); + synchronized (__sJT_execCtx) { + sqlj.runtime.profile.RTStatement __sJT_stmt = __sJT_execCtx.registerStatement(__sJT_connCtx, MediaDAO_SJProfileKeys.getKey(0), 3); + try { iter = new MediaIter(__sJT_execCtx.executeQuery(), __sJT_execCtx.getFetchSize(), __sJT_execCtx.getMaxRows()); } @@ -277,13 +320,13 @@ class MediaDAO extends EntityBaseDAO { // ************************************************************ -/*@lineinfo:user-code*//*@lineinfo:92^131*/ +/*@lineinfo:user-code*//*@lineinfo:99^141*/ return iter; } @Override protected void delete(DefaultContext context, long id) throws SQLException{ - /*@lineinfo:generated-code*//*@lineinfo:98^2*/ + /*@lineinfo:generated-code*//*@lineinfo:105^2*/ // ************************************************************ // #sql [context] { DELETE FROM MEDIA WHERE ID = :id }; @@ -295,7 +338,7 @@ class MediaDAO extends EntityBaseDAO { sqlj.runtime.ExecutionContext __sJT_execCtx = __sJT_connCtx.getExecutionContext(); if (__sJT_execCtx == null) sqlj.runtime.error.RuntimeRefErrors.raise_NULL_EXEC_CTX(); synchronized (__sJT_execCtx) { - sqlj.runtime.profile.RTStatement __sJT_stmt = __sJT_execCtx.registerStatement(__sJT_connCtx, MediaDAO_SJProfileKeys.getKey(0), 3); + sqlj.runtime.profile.RTStatement __sJT_stmt = __sJT_execCtx.registerStatement(__sJT_connCtx, MediaDAO_SJProfileKeys.getKey(0), 4); try { __sJT_stmt.setLong(1, id); @@ -311,12 +354,12 @@ class MediaDAO extends EntityBaseDAO { // ************************************************************ -/*@lineinfo:user-code*//*@lineinfo:98^52*/ +/*@lineinfo:user-code*//*@lineinfo:105^52*/ } @Override protected void truncateTable(DefaultContext context) throws SQLException{ - /*@lineinfo:generated-code*//*@lineinfo:103^2*/ + /*@lineinfo:generated-code*//*@lineinfo:110^2*/ // ************************************************************ // #sql [context] { TRUNCATE TABLE MEDIA DROP STORAGE IGNORE DELETE TRIGGERS IMMEDIATE }; @@ -328,7 +371,7 @@ class MediaDAO extends EntityBaseDAO { sqlj.runtime.ExecutionContext __sJT_execCtx = __sJT_connCtx.getExecutionContext(); if (__sJT_execCtx == null) sqlj.runtime.error.RuntimeRefErrors.raise_NULL_EXEC_CTX(); synchronized (__sJT_execCtx) { - sqlj.runtime.profile.RTStatement __sJT_stmt = __sJT_execCtx.registerStatement(__sJT_connCtx, MediaDAO_SJProfileKeys.getKey(0), 4); + sqlj.runtime.profile.RTStatement __sJT_stmt = __sJT_execCtx.registerStatement(__sJT_connCtx, MediaDAO_SJProfileKeys.getKey(0), 5); try { __sJT_execCtx.executeUpdate(); @@ -343,7 +386,7 @@ class MediaDAO extends EntityBaseDAO { // ************************************************************ -/*@lineinfo:user-code*//*@lineinfo:103^86*/ +/*@lineinfo:user-code*//*@lineinfo:110^86*/ } @Override @@ -358,11 +401,12 @@ class MediaDAO extends EntityBaseDAO { long length = obj.getLength(); byte[] poster = obj.getPoster(); Timestamp archived = obj.getArchived(); + Timestamp modified = obj.getModified(); - /*@lineinfo:generated-code*//*@lineinfo:119^2*/ + /*@lineinfo:generated-code*//*@lineinfo:127^2*/ // ************************************************************ -// #sql [context] { UPDATE MEDIA SET TITLE = :title, DESCRIPTION = :description, ITEMID = :itemId, ITEMTYPEID = :itemTypeId, HOUSEID = :houseId, LENGTH = :length, POSTER = :poster, ARCHIVED = :archived WHERE ID = :id }; +// #sql [context] { UPDATE MEDIA SET TITLE = :title, DESCRIPTION = :description, ITEMID = :itemId, ITEMTYPEID = :itemTypeId, HOUSEID = :houseId, LENGTH = :length, POSTER = :poster, ARCHIVED = :archived, MODIFIED = :modified WHERE ID = :id }; // ************************************************************ { @@ -371,7 +415,7 @@ class MediaDAO extends EntityBaseDAO { sqlj.runtime.ExecutionContext __sJT_execCtx = __sJT_connCtx.getExecutionContext(); if (__sJT_execCtx == null) sqlj.runtime.error.RuntimeRefErrors.raise_NULL_EXEC_CTX(); synchronized (__sJT_execCtx) { - sqlj.runtime.profile.RTStatement __sJT_stmt = __sJT_execCtx.registerStatement(__sJT_connCtx, MediaDAO_SJProfileKeys.getKey(0), 5); + sqlj.runtime.profile.RTStatement __sJT_stmt = __sJT_execCtx.registerStatement(__sJT_connCtx, MediaDAO_SJProfileKeys.getKey(0), 6); try { __sJT_stmt.setString(1, title); @@ -382,7 +426,8 @@ class MediaDAO extends EntityBaseDAO { __sJT_stmt.setLong(6, length); __sJT_stmt.setBytes(7, poster); __sJT_stmt.setTimestamp(8, archived); - __sJT_stmt.setLong(9, id); + __sJT_stmt.setTimestamp(9, modified); + __sJT_stmt.setLong(10, id); __sJT_execCtx.executeUpdate(); } finally @@ -395,7 +440,7 @@ class MediaDAO extends EntityBaseDAO { // ************************************************************ -/*@lineinfo:user-code*//*@lineinfo:119^216*/ +/*@lineinfo:user-code*//*@lineinfo:127^238*/ } @Override @@ -410,11 +455,12 @@ class MediaDAO extends EntityBaseDAO { long length = obj.getLength(); byte[] poster = obj.getPoster(); Timestamp archived = obj.getArchived(); + Timestamp modified = obj.getModified(); - /*@lineinfo:generated-code*//*@lineinfo:135^2*/ + /*@lineinfo:generated-code*//*@lineinfo:144^2*/ // ************************************************************ -// #sql [context] { INSERT INTO MEDIA (ID, TITLE, DESCRIPTION, ITEMID, ITEMTYPEID, HOUSEID, LENGTH, POSTER, ARCHIVED) VALUES (:id, :title, :description, :itemId, :itemTypeId, :houseId, :length, :poster, :archived) }; +// #sql [context] { INSERT INTO MEDIA (ID, TITLE, DESCRIPTION, ITEMID, ITEMTYPEID, HOUSEID, LENGTH, POSTER, ARCHIVED, MODIFIED) VALUES (:id, :title, :description, :itemId, :itemTypeId, :houseId, :length, :poster, :archived, :modified) }; // ************************************************************ { @@ -423,7 +469,7 @@ class MediaDAO extends EntityBaseDAO { sqlj.runtime.ExecutionContext __sJT_execCtx = __sJT_connCtx.getExecutionContext(); if (__sJT_execCtx == null) sqlj.runtime.error.RuntimeRefErrors.raise_NULL_EXEC_CTX(); synchronized (__sJT_execCtx) { - sqlj.runtime.profile.RTStatement __sJT_stmt = __sJT_execCtx.registerStatement(__sJT_connCtx, MediaDAO_SJProfileKeys.getKey(0), 6); + sqlj.runtime.profile.RTStatement __sJT_stmt = __sJT_execCtx.registerStatement(__sJT_connCtx, MediaDAO_SJProfileKeys.getKey(0), 7); try { __sJT_stmt.setLong(1, id); @@ -435,6 +481,7 @@ class MediaDAO extends EntityBaseDAO { __sJT_stmt.setLong(7, length); __sJT_stmt.setBytes(8, poster); __sJT_stmt.setTimestamp(9, archived); + __sJT_stmt.setTimestamp(10, modified); __sJT_execCtx.executeUpdate(); } finally @@ -447,11 +494,11 @@ class MediaDAO extends EntityBaseDAO { // ************************************************************ -/*@lineinfo:user-code*//*@lineinfo:135^213*/ +/*@lineinfo:user-code*//*@lineinfo:144^234*/ } /** - * Besz\ufffdrja az Item objektumhoz kapcsol\ufffdd\ufffd statikus/dinamikus Media adatokat. + * Besz\ufffd\ufffd\ufffdrja az Item objektumhoz kapcsol\ufffd\ufffd\ufffdd\ufffd\ufffd\ufffd statikus/dinamikus Media adatokat. */ void addAll(DefaultContext context, IEntityBase entity) throws SQLException { manager.traceIn(); diff --git a/server/user.jobengine.osgi.db/generated/user/jobengine/db/MediaDAO_SJProfile0.ser b/server/user.jobengine.osgi.db/generated/user/jobengine/db/MediaDAO_SJProfile0.ser index d893d8f48b2442c248b6400e5f620352e7c77552..14bcb65a8fba47883997a7b28e7ee2f5252c64cd 100644 GIT binary patch delta 799 zcmb7?xk>|36o$`{OlFcvh=LkXqY-xyL2yMZ1Qi0VxK)C<54dJV5D`b>h87|m3k$Wd zP_zvSqqhGIkAowTHeagyS`ivRU(I@NM1 zqGzH(wYrImRI7*SUY|glzaswM1OR}a^h`WOFAWABu$i)h^`)mNI>w+*20lF{&tFr& zUW~V71P`^EXe$2KU=<#-k)1YLWho)V2TMh`6iJbFcN3D?F19Awj!Sc0vIAze!*D~w zBbVr=WaJXvlI$lB$9+jicAnDw&58%yF{}B-5nsCC-m<(iV8InNhQ`wYo`jm{G1T!5 D*oWFO delta 499 zcmZvYJ4*vm5QWc3*6e1p$+{w&_`ogHK44;zLIi6&wNaFyd1#PmHolOg5yTVc5 zzS>V$nrMboo(2_oPKlH6tUFx0RpP{|*QQ;b(&za~5AnG-tjz*&0Ce%A5vDzU^E$$B zv!Q_ diff --git a/server/user.jobengine.osgi.db/generated/user/jobengine/db/MediaFileDAO_SJProfile0.ser b/server/user.jobengine.osgi.db/generated/user/jobengine/db/MediaFileDAO_SJProfile0.ser index 6220aa0deeaf1e320e41830073363b4b2b8b57a5..6704899322e26ae4105b229cca32e422a535ff33 100644 GIT binary patch delta 18 ZcmX@Fa9&{p4*vHm*I&4P?}`~X381^NI0 diff --git a/server/user.jobengine.osgi.db/generated/user/jobengine/db/MetadataDAO_SJProfile0.ser b/server/user.jobengine.osgi.db/generated/user/jobengine/db/MetadataDAO_SJProfile0.ser index 9af1cb3625cd524d7f3a0cfd7927e36e037fb724..3d961db881496b1ac41fe10db7dd0f482ba979a8 100644 GIT binary patch delta 18 Zcmca@a@%AB4*vHm*c&4P?xk^n{m23`OF diff --git a/server/user.jobengine.osgi.db/generated/user/jobengine/db/MetadataElementDAO_SJProfile0.ser b/server/user.jobengine.osgi.db/generated/user/jobengine/db/MetadataElementDAO_SJProfile0.ser index d8709a2d6dcd1ca1b55739f638d137daf782e6bf..e068336819f541044a091fe7bfc8350fea75056a 100644 GIT binary patch delta 18 ZcmZ1{zD|4t4*vHm)p&4P?`xd1&i1{VMT diff --git a/server/user.jobengine.osgi.db/generated/user/jobengine/db/MetadataTypeDAO_SJProfile0.ser b/server/user.jobengine.osgi.db/generated/user/jobengine/db/MetadataTypeDAO_SJProfile0.ser index 088abbbadfd8384a23be60deda2294acf09f63b7..e52849734cc345ce3e0c1315fc2f780be9a1ce96 100644 GIT binary patch delta 18 ZcmX>qepGw|4qepGw|4*vHm)f&4P?uxBx(*24nyL diff --git a/server/user.jobengine.osgi.db/generated/user/jobengine/db/RemoteStoreDAO_SJProfile0.ser b/server/user.jobengine.osgi.db/generated/user/jobengine/db/RemoteStoreDAO_SJProfile0.ser index 387089de30e7b46817353142b1e2fe31b63a7d9b..8b93baf420701061e8b96601767944c2b68a810b 100644 GIT binary patch delta 18 ZcmaDQ@k(L?4=1-JkJ delta 18 ZcmaDQ@k(L?4*vHm)n&4P@P+yF%;23`OF diff --git a/server/user.jobengine.osgi.db/generated/user/jobengine/db/SceneContentDAO_SJProfile0.ser b/server/user.jobengine.osgi.db/generated/user/jobengine/db/SceneContentDAO_SJProfile0.ser index f3acac86a34d2cab9bc4b7988437d4e2fe66a153..6d49a6a5fa9f2f382984f4ab6ca0ee6b8f8daa19 100644 GIT binary patch delta 18 ZcmdlWwLxkF4*vHm)j&4P^bJODj!1-$?O diff --git a/server/user.jobengine.osgi.db/generated/user/jobengine/db/SceneDAO_SJProfile0.ser b/server/user.jobengine.osgi.db/generated/user/jobengine/db/SceneDAO_SJProfile0.ser index c4b374e03992c58adda9c28cc635c7859ad1fc6c..b3491513eaeec5c07abd61802d28bf57c217eb3d 100644 GIT binary patch delta 18 ZcmbO$K39AL4*vHm*W&4P?QTmUheL{Kz4q0S1}Fdk delta 18 ZcmX>heL{Kz4*vHrTy&4P^Ec>qFH2ABW< diff --git a/server/user.jobengine.osgi.db/generated/user/jobengine/db/ShotDAO_SJProfile0.ser b/server/user.jobengine.osgi.db/generated/user/jobengine/db/ShotDAO_SJProfile0.ser index 2d80dce49e498d9753cb4e916ce6da8075d3dfbe..720879023ef0489b45e0c559145864663352a3b8 100644 GIT binary patch delta 18 ZcmbO)JzshQ4*vHrT)&4P@5JODhw1|*vHrTW&4P^H+yF!>23`OF diff --git a/server/user.jobengine.osgi.db/generated/user/jobengine/db/StoreUriDAO_SJProfile0.ser b/server/user.jobengine.osgi.db/generated/user/jobengine/db/StoreUriDAO_SJProfile0.ser index 55c219d6d755df56649ffb57084feb7c3247d578..e503e68b6d8ff5b975302f2e35912139f0061c2d 100644 GIT binary patch delta 18 ZcmbQDIz@E@4#wJ1ls@r delta 18 ZcmbQDIz@E@4*vHrUJ&4P^0f&e_`1}^{r diff --git a/server/user.jobengine.osgi.db/generated/user/jobengine/db/UserInfoDAO_SJProfile0.ser b/server/user.jobengine.osgi.db/generated/user/jobengine/db/UserInfoDAO_SJProfile0.ser index 91a0983ac53e401335eb6d9630c143d6189ea9ca..6111df5c45d03d4efa20fe18d989bba3b804f347 100644 GIT binary patch delta 18 ZcmdlgvsGpT4*vHrU9&4P?7yZ}Fz1_S^A diff --git a/server/user.jobengine.osgi.db/generated/user/jobengine/db/WorkflowActionDAO_SJProfile0.ser b/server/user.jobengine.osgi.db/generated/user/jobengine/db/WorkflowActionDAO_SJProfile0.ser index 50b109846b77d50841b68748cde5b68d78cb66b4..a0cd23edd2e4965a1feeaabf0bf7df499c9ea4ff 100644 GIT binary patch delta 18 ZcmZovZB*UB!^m1>%hjQ=S&*?n5CAZn1ik Tartomány - Felhasználó diff --git a/server/user.jobengine.osgi.server/src/user/jobengine/server/IJobEngine.java b/server/user.jobengine.osgi.server/src/user/jobengine/server/IJobEngine.java index 8c9a722a..96f29b7b 100644 --- a/server/user.jobengine.osgi.server/src/user/jobengine/server/IJobEngine.java +++ b/server/user.jobengine.osgi.server/src/user/jobengine/server/IJobEngine.java @@ -11,7 +11,7 @@ import user.jobengine.server.scheduler.ScheduledJob; import user.jobengine.server.scheduler.SchedulerService; public interface IJobEngine { - static final int QUEUE_POLL_INTERVAL_MS = 1; + static final int QUEUE_POLL_INTERVAL_MS = 50; void addJobChangedEventListener(IJobChangedListener listener); diff --git a/server/user.jobengine.osgi.server/src/user/jobengine/server/JobEngine.java b/server/user.jobengine.osgi.server/src/user/jobengine/server/JobEngine.java index 6519b314..d6430e7e 100644 --- a/server/user.jobengine.osgi.server/src/user/jobengine/server/JobEngine.java +++ b/server/user.jobengine.osgi.server/src/user/jobengine/server/JobEngine.java @@ -731,11 +731,12 @@ public class JobEngine implements IJobEngine { @Override public void processJobStepCompletedMessage(IJobMessage message) { - // TODO cancel nem megy, valszeg itt van gubasz IJobRuntime jobRuntime = getJobById(message.getJobId()); - if (jobRuntime.getParentJobId() > 0) - removeSpanwChild(jobRuntime); + //TODO ez hibat okoz az archivalasnal, mert hamarabb eltavolitja a childUd-ket + + // if (jobRuntime.getParentJobId() > 0) + // removeSpanwChild(jobRuntime); JobStepCompletedMessage m = (JobStepCompletedMessage) message; //kesz vagyunk, jelezni @@ -1080,7 +1081,7 @@ public class JobEngine implements IJobEngine { @Override public void suspendExecutingJob(Throwable t, IJobRuntime jobRuntime) { - String description = t.getClass().getSimpleName() + ": " + t.getMessage(); + String description = t.getMessage(); jobRuntime.setDescription(description); logger.error(description); //TODO itt miert FINISH a kovetkezo allapot, miert nem SUSPEND @@ -1090,7 +1091,7 @@ public class JobEngine implements IJobEngine { @Override public void suspendWaitExecutorJob(Throwable t, IJobRuntime jobRuntime) { - String description = t.getClass().getSimpleName() + ": " + t.getMessage(); + String description = t.getMessage(); jobRuntime.setDescription(description); logger.error(description); statusMachine.processAction(JobAction.SUSPEND, jobRuntime); diff --git a/server/user.jobengine.osgi.server/src/user/jobengine/server/JobRuntime.java b/server/user.jobengine.osgi.server/src/user/jobengine/server/JobRuntime.java index 7af28c3a..93806715 100644 --- a/server/user.jobengine.osgi.server/src/user/jobengine/server/JobRuntime.java +++ b/server/user.jobengine.osgi.server/src/user/jobengine/server/JobRuntime.java @@ -273,6 +273,7 @@ public class JobRuntime extends Job implements IJobRuntime { if (event.getSignalType().equals(SignalType.CREATE)) { if (child.getParentJobId() == getId()) { childrenIDs.add(child.getId()); + logger.info("Added child {} to {}", child.getId(), getId()); } if (child.getParentJobId() != getId() && childrenIDs.contains(child.getParentJobId())) { @@ -280,20 +281,21 @@ public class JobRuntime extends Job implements IJobRuntime { } } - boolean isChildDeleted = event.getSignalType().equals(SignalType.DELETE) && childrenIDs.contains(child.getId()); - boolean isChildHasError = event.getSignalType().equals(SignalType.UPDATE) && childrenIDs.contains(child.getId()) - && JobStatus.SUSPENDED.equals(child.getStatus()); - if (isChildDeleted || isChildHasError) { - childrenIDs.remove(child.getId()); - if (childrenIDs.size() == 0) + if (childrenIDs.contains(child.getId())) { + logger.info("Child {} notification", child.getId()); + switch (event.getSignalType()) { + case UPDATE: + if (JobStatus.SUSPENDED.equals(child.getStatus())) + forkSempahore.release(); + break; + case DELETE: + removeSpanwChild(child.getId()); + break; + } + + if (!getJobEngine().isRunning()) forkSempahore.release(); } - if (!getJobEngine().isRunning()) - forkSempahore.release(); - - //A gyerek(ek) el sem indultak, pl. nem letezik a template - if (event.getSignalType().equals(SignalType.UPDATE) && JobStatus.SUSPENDED.equals(getStatus())) - forkSempahore.release(); }; logger.info("Adding job changed listener"); @@ -314,7 +316,6 @@ public class JobRuntime extends Job implements IJobRuntime { return; logger.info("Waiting for semaphore {}", forkSempahore); forkSempahore.acquire(); - logger.info("Removing job changed listener"); if (getJobEngine().isRunning()) { logger.info("Removing job changed listener"); getJobEngine().removeJobChangedEventListener(spawnJobListener); @@ -638,7 +639,11 @@ public class JobRuntime extends Job implements IJobRuntime { @Override public void removeSpanwChild(long id) { - childrenIDs.remove(id); + if (childrenIDs != null && childrenIDs.contains(id)) { + childrenIDs.remove(id); + if (childrenIDs.size() == 0 && forkSempahore != null) + forkSempahore.release(); + } } @Override diff --git a/server/user.jobengine.osgi.server/src/user/jobengine/server/JobStatusChangedEvent.java b/server/user.jobengine.osgi.server/src/user/jobengine/server/JobStatusChangedEvent.java index 6cfffb72..c483d02c 100644 --- a/server/user.jobengine.osgi.server/src/user/jobengine/server/JobStatusChangedEvent.java +++ b/server/user.jobengine.osgi.server/src/user/jobengine/server/JobStatusChangedEvent.java @@ -21,4 +21,8 @@ public class JobStatusChangedEvent extends EventObject { return status; } + public boolean isRuntimeTerminated() { + return JobStatus.FINISHED.equals(status) || JobStatus.SUSPENDED.equals(status); + } + } diff --git a/server/user.jobengine.osgi.server/src/user/jobengine/server/JobStepExecutor.java b/server/user.jobengine.osgi.server/src/user/jobengine/server/JobStepExecutor.java index 56275397..ddfb04d3 100644 --- a/server/user.jobengine.osgi.server/src/user/jobengine/server/JobStepExecutor.java +++ b/server/user.jobengine.osgi.server/src/user/jobengine/server/JobStepExecutor.java @@ -151,6 +151,7 @@ public class JobStepExecutor implements IJobStepExecutor { private int maxConcurrent; private String stepUnitName; private boolean isRemote; + private URLClassLoader parentLoader; public JobStepExecutor() { } @@ -194,11 +195,11 @@ public class JobStepExecutor implements IJobStepExecutor { if (StringUtils.isEmpty(stepUnitName)) throw new JobEngineException("Step unit name can't be null."); + parentLoader = URLClassLoader.newInstance(DynamicClassLocator.makeURLs(), getClass().getClassLoader()); if (!isGroovyStep()) { //a groovy-nak nem kell try { - URLClassLoader loader = URLClassLoader.newInstance(DynamicClassLocator.makeURLs(), getClass().getClassLoader()); - stepClass = (Class) loader.loadClass(stepUnitName); + stepClass = (Class) parentLoader.loadClass(stepUnitName); } catch (ClassNotFoundException e1) { logger.catching(e1); throw new JobEngineException("System can't load JobStep implementation: " + stepUnitName); @@ -219,7 +220,7 @@ public class JobStepExecutor implements IJobStepExecutor { IJobStep result = null; if (isGroovyStep()) { - GroovyClassLoader gcl = new GroovyClassLoader(); + GroovyClassLoader gcl = new GroovyClassLoader(parentLoader); Class myClass = gcl.parseClass(Paths.get(System.getProperty("jobengine.jobsteps.groovy.root", ""), stepUnitName).toFile()); result = (IJobStep) myClass.newInstance(); } else @@ -248,7 +249,7 @@ public class JobStepExecutor implements IJobStepExecutor { } private boolean isGroovyStep() { - return stepUnitName.toLowerCase().endsWith(".java"); + return stepUnitName.toLowerCase().endsWith(".java") || stepUnitName.toLowerCase().endsWith(".groovy"); } @Override diff --git a/server/user.jobengine.osgi.server/src/user/jobengine/zk/model/LoginModel.java b/server/user.jobengine.osgi.server/src/user/jobengine/zk/model/LoginModel.java index ad8139bd..52f8c6b2 100644 --- a/server/user.jobengine.osgi.server/src/user/jobengine/zk/model/LoginModel.java +++ b/server/user.jobengine.osgi.server/src/user/jobengine/zk/model/LoginModel.java @@ -17,6 +17,7 @@ public class LoginModel extends BaseModel { private String password; private String message; private String query; + private String domain; @AfterCompose public void afterCompose(@ContextParam(ContextType.VIEW) Component view) { @@ -24,6 +25,7 @@ public class LoginModel extends BaseModel { MediaCubeConfig mediaCubeConfig = SessionUtil.getMediaCubeConfig(); MediaCubeAuthentication authentication = mediaCubeConfig.getAuthentication(); + domain = authentication.getAdHost(); if (StringUtils.isNotEmpty(authentication.getDefaultUser()) && StringUtils.isNotBlank(authentication.getDefaultPassword())) authenticate(authentication.getDefaultUser(), authentication.getDefaultPassword()); } @@ -46,6 +48,10 @@ public class LoginModel extends BaseModel { return account; } + public String getDomain() { + return domain; + } + public String getMessage() { return message; } diff --git a/server/user.jobengine.osgi.server/test/user/jobengine/server/IT/AMCTests.java b/server/user.jobengine.osgi.server/test/user/jobengine/server/IT/AMCTests.java new file mode 100644 index 00000000..73d157a9 --- /dev/null +++ b/server/user.jobengine.osgi.server/test/user/jobengine/server/IT/AMCTests.java @@ -0,0 +1,84 @@ +package user.jobengine.server.IT; + +import static org.junit.Assert.assertEquals; + +import java.io.FileInputStream; +import java.net.URL; +import java.util.HashMap; +import java.util.Map; +import java.util.Properties; +import java.util.concurrent.CountDownLatch; + +import org.junit.AfterClass; +import org.junit.BeforeClass; +import org.junit.Test; + +import user.commons.JobStatus; +import user.commons.StoreUri; +import user.commons.remotestore.RemoteStoreProtocol; +import user.jobengine.db.IItemManager; +import user.jobengine.db.ItemManager; +import user.jobengine.db.Store; +import user.jobengine.server.IJobEngine; +import user.jobengine.server.IJobRuntime; +import user.jobengine.server.JobEngine; + +public class AMCTests { + private static IItemManager manager; + private static IJobEngine jobEngine; + + @BeforeClass + public static void initialize() throws Exception { + //Kornyezeti valtozok betoltese + Properties properties = new Properties(); + URL srcLocation = AMCTests.class.getProtectionDomain().getCodeSource().getLocation(); + URL location = new URL(srcLocation, "../../-configuration/mediacube-dev-user.properties"); + properties.load(new FileInputStream(location.toURI().getPath().toString())); + System.getProperties().putAll(properties); + + manager = new ItemManager(); + manager.connect(); + + jobEngine = new JobEngine(); + jobEngine.startup(); + jobEngine.bindItemManagerService(manager); + + } + + @AfterClass + public static void terminate() throws Exception { + jobEngine.shutdown(); + manager.disconnect(); + } + + @Test + public void testNexioCheckerStep() throws Exception { + Store nexioStore = manager.createStore("NEXIO", false, false); + nexioStore.add(); + + //Tesztelni a HirTV-n lehet + StoreUri storeUri = manager.createStoreUri(RemoteStoreProtocol.NEXIO, "10.10.1.55"); + storeUri.setStoreId(nexioStore.getId()); + storeUri.add(); + + CountDownLatch finishLatch = new CountDownLatch(1); + + Map jobParams = new HashMap<>(); + jobParams.put("storeName", "NEXIO"); + jobParams.put("outputPath", "/opt"); + jobParams.put("limit", 1); + + IJobRuntime jobRuntime = jobEngine.submit(null, e -> { + if (e.isRuntimeTerminated()) + finishLatch.countDown(); + }, "nexio-check.xml", "NEXIO check", jobParams); + + finishLatch.await(); + + storeUri.remove(); + nexioStore.remove(); + + assertEquals(JobStatus.FINISHED, jobRuntime.getStatus()); + } + +} -- 2.54.0