From: Vásáry Dániel Date: Fri, 4 Sep 2020 19:16:21 +0000 (+0000) Subject: git-tfs-id: [http://tfs.userrendszerhaz.hu:8080/tfs/DefaultCollection]$/MediaCube... X-Git-Url: http://git.useribm.hu/?a=commitdiff_plain;h=9f50002d874a25487015eda18f84365df0d1b468;p=mediacube.git git-tfs-id: [tfs.userrendszerhaz.hu:8080/tfs/DefaultCollection]$/MediaCube;C31950 --- diff --git a/server/-configuration/log4j2.xml b/server/-configuration/log4j2.xml index dbc3d3bc..f988a5e1 100644 --- a/server/-configuration/log4j2.xml +++ b/server/-configuration/log4j2.xml @@ -19,8 +19,8 @@ - + diff --git a/server/-configuration/mediacube.json b/server/-configuration/mediacube.json index 733e5b30..44fe9f45 100644 --- a/server/-configuration/mediacube.json +++ b/server/-configuration/mediacube.json @@ -1,4 +1,7 @@ { + "targetRestoreFilters": [ + "FILEZILLA_AVID", "FILEZILLA_PASARESTORE" + ], "topTypeFilters": [ { "name": "Hír bejátszó", diff --git a/server/-configuration/scheduledjobs.json b/server/-configuration/scheduledjobs.json index 948c8778..3f606aa9 100644 --- a/server/-configuration/scheduledjobs.json +++ b/server/-configuration/scheduledjobs.json @@ -22,7 +22,8 @@ ] }, { - "template": "cancelable.xml" + "template": "cancelable.xml", + "parameters": [ {"name": "param", "value": 1, "type": "java.lang.Integer"} ] }, { "template": "submit-child.xml" diff --git a/server/hu.user.mediacube.executors.tests/src/hu/user/mediacube/executors/tests/HSMMigrateStepTest.java b/server/hu.user.mediacube.executors.tests/src/hu/user/mediacube/executors/tests/HSMMigrateStepTest.java index 6c083e0c..5fa41f5f 100644 --- a/server/hu.user.mediacube.executors.tests/src/hu/user/mediacube/executors/tests/HSMMigrateStepTest.java +++ b/server/hu.user.mediacube.executors.tests/src/hu/user/mediacube/executors/tests/HSMMigrateStepTest.java @@ -3,7 +3,10 @@ package hu.user.mediacube.executors.tests; import static org.junit.Assert.assertNotNull; import static org.junit.Assert.assertNull; +import java.nio.file.Files; import java.nio.file.Paths; +import java.util.LinkedHashSet; +import java.util.List; import org.junit.BeforeClass; import org.junit.Test; @@ -25,6 +28,46 @@ public class HSMMigrateStepTest { System.setProperty("jobengine.nosql.db.password", "password"); } + @Test + public void createMigrateReport() throws Exception { + LinkedHashSet success = new LinkedHashSet<>(); + List migrated = Files.readAllLines(Paths.get("/opt/MV-migrate-status/migrated.txt")); + System.out.println("Starting"); + for (String m : migrated) { + String name = m.trim(); + int indexOf = name.indexOf("-"); + if (indexOf > -1) { + name = name.substring(9); + if (success.contains(name)) + System.out.println("Error " + m.trim()); + else + success.add(name); + } + // System.out.println(name); + } + + List tomigrate = Files.readAllLines(Paths.get("/opt/MV-migrate-status/to-migrate.txt")); + int nosuccesscount = 0; + for (String t : tomigrate) { + String name = t.trim(); + name = name.substring(2); + int indexOf = name.indexOf("."); + if (indexOf > -1) { + indexOf = name.lastIndexOf("/"); + name = name.substring(indexOf + 1); + if (!success.contains(name)) { + System.out.println(name); + nosuccesscount++; + } + } + } + + System.out.println("Success " + migrated.size()); + System.out.println("No success " + nosuccesscount); + System.out.println(100 - (nosuccesscount * 100 / migrated.size())); + + } + private HSMMigrateStep createSUT() { HSMMigrateStep sut = new HSMMigrateStep() { @Override diff --git a/server/hu.user.mediacube.executors.tests/src/hu/user/mediacube/executors/tests/MediaBaseTest.java b/server/hu.user.mediacube.executors.tests/src/hu/user/mediacube/executors/tests/MediaBaseTest.java index 8bf6b149..e5d1917b 100644 --- a/server/hu.user.mediacube.executors.tests/src/hu/user/mediacube/executors/tests/MediaBaseTest.java +++ b/server/hu.user.mediacube.executors.tests/src/hu/user/mediacube/executors/tests/MediaBaseTest.java @@ -1,8 +1,10 @@ package hu.user.mediacube.executors.tests; +import java.sql.Timestamp; import java.text.SimpleDateFormat; import java.time.Duration; import java.time.Instant; +import java.util.Arrays; import java.util.Iterator; import java.util.List; @@ -10,18 +12,93 @@ import org.apache.commons.io.FilenameUtils; import org.apache.commons.io.output.ByteArrayOutputStream; import org.apache.commons.net.ftp.FTPClient; import org.junit.Test; +import org.w3c.dom.Document; +import org.w3c.dom.Node; +import org.w3c.dom.NodeList; import user.commons.RemoteFile; import user.commons.StoreUri; +import user.commons.nexio.INexioAPI; import user.commons.nexio.api.Clip; import user.commons.nexio.api.Controller; import user.commons.nexio.api.Mediabase; +import user.commons.nexio.server.protocol.Id; +import user.commons.nexio.server.protocol.NexioServerProtocol; +import user.commons.nexio.server.protocol.Xid; import user.commons.remotestore.FtpDirectoryLister; import user.commons.remotestore.IDirectoryLister; import user.commons.remotestore.RemoteStoreProtocol; +import user.jobengine.server.steps.EscortFiles; public class MediaBaseTest { + private static final char[] HEX_ARRAY = "0123456789ABCDEF".toCharArray(); + + public static String bytesToHex(byte[] bytes) { + char[] hexChars = new char[bytes.length * 3]; + for (int j = 0; j < bytes.length; j++) { + int v = bytes[j] & 0xFF; + hexChars[j * 3] = HEX_ARRAY[v >>> 4]; + hexChars[j * 3 + 1] = HEX_ARRAY[v & 0x0F]; + hexChars[j * 3 + 2] = ' '; + } + return new String(hexChars).trim(); + } + + public static byte[] hexStringToByteArray(String input) { + String s = input.replace(" ", ""); + int len = s.length(); + byte[] data = new byte[len / 2]; + for (int i = 0; i < len; i += 2) { + data[i / 2] = (byte) ((Character.digit(s.charAt(i), 16) << 4) + Character.digit(s.charAt(i + 1), 16)); + } + return data; + } + + @Test + public void checkFileContentFTP() throws Exception { + StoreUri nexioUri = new StoreUri(); + nexioUri.setProtocol(RemoteStoreProtocol.FTP); + nexioUri.setUri("10.10.1.55"); + nexioUri.setPortNumber(2098); + nexioUri.setUserName("ftp"); + nexioUri.setPassword("ftp"); + byte[] content = nexioUri.getFileWithContent("test-dani.xml").getContent(); + System.out.println(new String(content)); + + Document document = EscortFiles.createNEXIOMeta(content); + Node firstChild = document.getFirstChild(); + NodeList list = firstChild.getChildNodes(); + for (int i = 0; i < list.getLength(); i++) { + Node item = list.item(i); + //System.out.println(item.getNodeName()); + if (item.getNodeName().equals("ModifiedTimeStamp")) { + System.out.println(item.getTextContent()); + SimpleDateFormat df = new SimpleDateFormat("MM-dd-yyyy (HH:mm:ss)"); + System.out.println(Timestamp.from(df.parse(item.getTextContent()).toInstant())); + break; + } + } + + // if (elementsByTagName.getLength() == 1) { + // + // } + nexioUri.cleanUp(); + } + + @Test + public void checkFileExistsFTP() throws Exception { + StoreUri nexioUri = new StoreUri(); + nexioUri.setProtocol(RemoteStoreProtocol.FTP); + nexioUri.setUri("10.10.1.55"); + nexioUri.setPortNumber(2098); + nexioUri.setUserName("ftp"); + nexioUri.setPassword("ftp"); + System.out.println(nexioUri.exists("test-dani.xml")); + + nexioUri.cleanUp(); + } + @Test public void listMediaBaseFTP() throws Exception { Instant start = Instant.now(); @@ -73,13 +150,15 @@ public class MediaBaseTest { controller.connect(); Mediabase mediabase = controller.getMediabase(); - int i = 100; + int i = 200; + try { - SimpleDateFormat df = new SimpleDateFormat("yyy-MM-dd HH:mm:ss"); + //SimpleDateFormat df = new SimpleDateFormat("yyy-MM-dd HH:mm:ss"); Iterator clips = mediabase.getClips(); while (clips.hasNext() && i > 0) { Clip clip = clips.next(); - System.out.println(clip.getId() + " " + clip.getXid().get() + " " + df.format(clip.getModifiedTimestamp().getTime())); + //System.out.println(clip.getId() + " " + clip.getXid().get() + " " + df.format(clip.getModifiedTimestamp().getTime())); + System.out.println(clip.getId() + " " + clip.getXid().get() + " "); i--; } } catch (Exception e) { @@ -88,7 +167,93 @@ public class MediaBaseTest { controller.disconnect(); } Instant end = Instant.now(); - System.out.println(Duration.between(start, end)); + long d = Duration.between(start, end).toMillis() / 1000; + String elapsed = String.format("%d:%02d:%02d", d / 3600, (d % 3600) / 60, (d % 60)); + System.out.println("Done in " + elapsed); + + } + + /* + * 19. oldal + C8 B1 00 00 00 02 00 00 00 02 + If not already established, sets up the interchange of data to be in Unicode, where appropriate + + C1 4C 21 + Retrieves the first ID in the system. Mode 21 retrieves both the 8-byte ID handle and the extended ID at the same time. More traditional methods of ID retrieval use A0 14 to get the first 8-byte ID handle followed by C8 C3 to get its matching extended ID. + C8 4A 25 30 30 30 30 31 32 33 + + Retrieves the normal set of metadata associated with the specified 8-byte ID handle retrieved in the previous command + C9 C3 25 30 30 30 30 31 32 33 00 + + Retrieves the metadata for the specified clip in the first extended field + C9 C3 25 30 30 30 30 31 32 33 nn + + Retrieves the metadata for the specified clip in each of the remaining extended fields + C8 84 25 30 30 30 30 31 32 33 + + Retrieves the special attributes associated with the specified clip + C0 4D + + Retrieves the next ID in the system including its 8-byte ID handle and extended ID + C8 4A 25 30 30 30 30 31 32 34 + + Retrieves the normal set of metadata associated with the next ID in the list + … + Continue this cycle of retrieving the next ID in the list and its associated metadata until the server returns no more IDs * */ + + @Test + public void listMediaBaseNEXIO2() throws Exception { + Instant start = Instant.now(); + Controller controller = new Controller("10.10.1.55"); + try { + controller.connect(); + Mediabase mediabase = controller.getMediabase(); + NexioServerProtocol protocol = mediabase.getProtocol(); + + byte[] command; + byte[] bytes; + + //byte[] command = hexStringToByteArray("A0 14"); + //byte[] command = { (byte) 0xC8, (byte) 0xB1, (byte) 0x00, (byte) 0x00, (byte) 0x00, (byte) 0x02, (byte) 0x00, (byte) 0x00, (byte) 0x00, (byte) 0x02 }; + + // command = hexStringToByteArray("C8 B1 00 00 00 02 00 00 00 02"); + // bytes = protocol.executeCommand(command); + // System.out.println(bytesToHex(bytes)); + + command = hexStringToByteArray("C1 4C 21"); + bytes = protocol.executeCommand(command); + + int i = 200; + Xid xid; + Id id; + + xid = new Xid(new String(Arrays.copyOfRange(bytes, 11, bytes.length), INexioAPI.ENCODING)); + id = new Id(new String(Arrays.copyOf(bytes, 8))); + System.out.println(id + " " + xid.get()); + i--; + //D0 4D a lista vege + while (!bytesToHex(bytes).equals("D0 4D")) { + command = hexStringToByteArray("C0 4D"); + bytes = protocol.executeCommand(command); + xid = new Xid(new String(Arrays.copyOfRange(bytes, 11, bytes.length), INexioAPI.ENCODING)); + xid = new Xid(new String(Arrays.copyOfRange(bytes, 11, bytes.length), INexioAPI.ENCODING)); + id = new Id(new String(Arrays.copyOf(bytes, 8))); + System.out.println(id + " " + xid.get()); + i--; + if (i == 0) + break; + } + + } catch (Exception e) { + System.err.println(e.getMessage()); + } finally { + controller.disconnect(); + } + Instant end = Instant.now(); + + long d = Duration.between(start, end).toMillis() / 1000; + String elapsed = String.format("%d:%02d:%02d", d / 3600, (d % 3600) / 60, (d % 60)); + System.out.println("Done in " + elapsed); } } diff --git a/server/hu.user.mediacube.executors.tests/src/hu/user/mediacube/executors/tests/SmallTests.java b/server/hu.user.mediacube.executors.tests/src/hu/user/mediacube/executors/tests/SmallTests.java index 93f0294c..a8d2a1e9 100644 --- a/server/hu.user.mediacube.executors.tests/src/hu/user/mediacube/executors/tests/SmallTests.java +++ b/server/hu.user.mediacube.executors.tests/src/hu/user/mediacube/executors/tests/SmallTests.java @@ -7,13 +7,19 @@ import java.nio.file.Path; import java.nio.file.Paths; import java.util.ArrayList; import java.util.Arrays; +import java.util.Collection; +import java.util.Collections; +import java.util.Date; import java.util.List; +import java.util.Set; import java.util.TreeMap; +import java.util.TreeSet; import org.apache.commons.io.FilenameUtils; import org.junit.Test; import user.jobengine.db.Media; +import user.jobengine.server.steps.EscortFiles; import user.jobengine.server.steps.TSMExtendedRetrieveStep; public class SmallTests { @@ -63,6 +69,79 @@ public class SmallTests { System.out.println(file); } + @Test + public void test11() throws Exception { + String x = "20200805T07300000"; + String y = "20200805T08000000"; + + System.out.println(x.compareTo(y)); + + //A TreeSet novekvo sorrendben adja vissza, a korabbi datum lesz elol + Set sut = new TreeSet<>(); + + sut.add("2020.03.01_12:00:00"); + sut.add("2020.01.01_10:00:00"); + sut.add("2020.04.01_13:00:00"); + sut.add("2020.02.01_11:00:00"); + + sut.forEach(z -> System.out.println(z)); + + List sut1 = Arrays.asList(1, 3, 2); + Collections.sort(sut1, (p1, p2) -> { + return p1.compareTo(p2); + }); + + sut1.forEach(z -> System.out.println(z)); + + } + + @Test + public void test12() throws Exception { + DirectoryStream p = Files.newDirectoryStream(Paths.get("/_video/"), "*KESZ.mxf"); + System.out.println(p.iterator().hasNext()); + + System.out.println(Paths.get("/mnt", ".STATUS")); + } + + @Test + public void test13() throws Exception { + DirectoryStream p = Files.newDirectoryStream(Paths.get("/_video/"), "*KESZ.mxf"); + System.out.println(p.iterator().hasNext()); + + System.out.println(Paths.get("/mnt", ".STATUS")); + } + + @Test + public void test14() throws Exception { + int tryconnect = 3; + + while (tryconnect > 0) { + tryconnect--; + System.out.println(tryconnect); + } + } + + @Test + public void test15() throws Exception { + + List started = Files.readAllLines(Paths.get("/_workspace/USER/MediaCube/started_f.txt")); + List completed = Files.readAllLines(Paths.get("/_workspace/USER/MediaCube/completed_f.txt")); + + for (String s : started) { + if (completed.contains(s)) + continue; + + System.out.println(s); + } + } + + @Test + public void test16() throws Exception { + Date now = new Date(); + String x = new String(EscortFiles.createNEXIODatesMeta("XXX", now, now)); + System.out.println(x); + } + @Test public void test2() throws Exception { Integer i = 5; @@ -109,15 +188,19 @@ public class SmallTests { public void test6() throws Exception { TreeMap sut = new TreeMap<>(); - sut.put("20200301T10:00:00", ""); - sut.put("20200101T12:00:00", ""); - sut.put("20200101T10:00:00", ""); - sut.put("20200201T00:00:00", ""); + sut.put("20200301T12:00:00", "20200301T12:00:00"); + sut.put("20200101T10:00:00", "20200101T10:00:00"); + sut.put("20200401T13:00:00", "20200401T13:00:00"); + sut.put("20200201T11:00:00", "20200201T11:00:00"); for (String key : sut.keySet()) { - System.out.println(key); + System.out.println(key + " - " + sut.get(key)); } + Collection values = sut.values(); + for (String value : values) { + System.out.println(value); + } String name = "valammi.mxf"; System.out.println(name.substring(0, name.lastIndexOf("."))); @@ -160,4 +243,35 @@ public class SmallTests { System.out.println(file); } + @Test + public void test91() throws Exception { + + try { + throw new Exception("E1"); + } catch (Exception e) { + System.out.println(e.getMessage()); + try { + throw new Exception("E2"); + } catch (Exception e1) { + System.out.println(e1.getMessage()); + throw e1; + } + } finally { + System.out.println("Finally"); + } + + } + + @Test + public void test92() throws Exception { + + Path p = Paths.get(Paths.get("/Temp/xxx/").toString(), "test1.json"); + System.out.println(p.toFile().exists()); + + System.out.println(Files.isSymbolicLink(p)); + + Path p1 = Paths.get("/Temp/xxx/test2.json"); + System.out.println(p1.toFile().exists()); + System.out.println(Files.isSymbolicLink(p1)); + } } diff --git a/server/user.commons.log4j2/build-log4j2-module.launch b/server/user.commons.log4j2/build-log4j2-module.launch new file mode 100644 index 00000000..ee351d6f --- /dev/null +++ b/server/user.commons.log4j2/build-log4j2-module.launch @@ -0,0 +1,19 @@ + + + + + + + + + + + + + + + + + + + diff --git a/server/user.commons.log4j2/src/user/commons/log4j2/appender/SmtpManager.java b/server/user.commons.log4j2/src/user/commons/log4j2/appender/SmtpManager.java index 7b7e9e24..ac471ac3 100644 --- a/server/user.commons.log4j2/src/user/commons/log4j2/appender/SmtpManager.java +++ b/server/user.commons.log4j2/src/user/commons/log4j2/appender/SmtpManager.java @@ -27,7 +27,9 @@ import java.util.Map; import java.util.Properties; import java.util.concurrent.ConcurrentHashMap; +import javax.activation.CommandMap; import javax.activation.DataSource; +import javax.activation.MailcapCommandMap; import javax.mail.Authenticator; import javax.mail.Message; import javax.mail.Message.RecipientType; @@ -275,6 +277,17 @@ public class SmtpManager extends AbstractManager { return headers; } + private Level getLowestLevel(final Level level, final List events) { + Level mailLevel = level; + if (events != null) { + for (LogEvent event : events) { + if (event.getLevel().intLevel() < mailLevel.intLevel()) + mailLevel = event.getLevel(); + } + } + return mailLevel; + } + protected MimeMultipart getMimeMultipart(final byte[] encodedBytes, final InternetHeaders headers) throws MessagingException { final MimeMultipart mp = new MimeMultipart(); final MimeBodyPart part = new MimeBodyPart(headers, encodedBytes); @@ -284,6 +297,12 @@ public class SmtpManager extends AbstractManager { private void sendEmail(final String to, final String subject, byte[] content, String contentType) { try { + MailcapCommandMap mc = (MailcapCommandMap) CommandMap.getDefaultCommandMap(); + mc.addMailcap("text/html;; x-java-content-handler=com.sun.mail.handlers.text_html"); + mc.addMailcap("text/xml;; x-java-content-handler=com.sun.mail.handlers.text_xml"); + mc.addMailcap("text/plain;; x-java-content-handler=com.sun.mail.handlers.text_plain"); + mc.addMailcap("multipart/*;; x-java-content-handler=com.sun.mail.handlers.multipart_mixed"); + mc.addMailcap("message/rfc822;; x-java-content- handler=com.sun.mail.handlers.message_rfc822"); message.setSubject(subject); message.setRecipients(RecipientType.TO, to); final String encoding = getEncoding(content, contentType); @@ -315,8 +334,17 @@ public class SmtpManager extends AbstractManager { if (appendEvent.getMarker() instanceof MediaCubeFinishMarker) { MediaCubeFinishMarker mcm = (MediaCubeFinishMarker) appendEvent.getMarker(); storeSessionEvent(mcm, appendEvent); - sendSessionEvents(mcm, layout, appendEvent.getLevel()); - return; + String sessionID = mcm.getSessionID(); + if (sessionID == null) + return; + + List events = sessionEvents.get(sessionID); + + //ha error, akkor mindenkepp elmegy + if (mcm.isUseSessionLog() || getLowestLevel(Level.INFO, events).intLevel() < Level.WARN.intLevel()) { + sendSessionEvents(events, mcm, layout, appendEvent.getLevel()); + } + } if (appendEvent.getMarker() instanceof MediaCubeUndoMarker) { @@ -361,20 +389,15 @@ public class SmtpManager extends AbstractManager { } } - private void sendSessionEvents(final MediaCubeMarker mcm, final Layout layout, final Level level) { + private void sendSessionEvents(final List events, final MediaCubeMarker mcm, final Layout layout, final Level level) { String sessionID = mcm.getSessionID(); - List events = sessionEvents.get(sessionID); if (events == null) return; - Level mailLevel = level; - for (LogEvent event : events) { - if (event.getLevel().intLevel() < mailLevel.intLevel()) - mailLevel = event.getLevel(); - } + Level mailLevel = getLowestLevel(level, events); String to = mcm.getTo() == null ? defaultRecipient : defaultRecipient + "," + mcm.getTo(); - String subject = mcm.getSessionName() == null ? defaultSubject : String.format("%s: %s #%s", mailLevel, mcm.getSessionName(), mcm.getSessionID()); + String subject = mcm.getSessionName() == null ? defaultSubject : String.format("%s: %s #%s", mailLevel, mcm.getSessionName(), sessionID); sendEvents(to, subject, layout, events); sessionEvents.remove(sessionID); } diff --git a/server/user.jobengine.executors/jobtemplates/cancelable.xml b/server/user.jobengine.executors/jobtemplates/cancelable.xml index f7b501c5..fa29cc3f 100644 --- a/server/user.jobengine.executors/jobtemplates/cancelable.xml +++ b/server/user.jobengine.executors/jobtemplates/cancelable.xml @@ -1,5 +1,5 @@ - + diff --git a/server/user.jobengine.executors/jobtemplates/fork-validate-and-archive.xml b/server/user.jobengine.executors/jobtemplates/fork-validate-and-archive.xml index 0195aa57..5136c36d 100644 --- a/server/user.jobengine.executors/jobtemplates/fork-validate-and-archive.xml +++ b/server/user.jobengine.executors/jobtemplates/fork-validate-and-archive.xml @@ -1,5 +1,5 @@ - + diff --git a/server/user.jobengine.executors/jobtemplates/fork-validate-and-restore.xml b/server/user.jobengine.executors/jobtemplates/fork-validate-and-restore.xml index cc52684f..bddbab68 100644 --- a/server/user.jobengine.executors/jobtemplates/fork-validate-and-restore.xml +++ b/server/user.jobengine.executors/jobtemplates/fork-validate-and-restore.xml @@ -1,5 +1,5 @@ - + diff --git a/server/user.jobengine.executors/jobtemplates/harris-missingmaterial-checker.xml b/server/user.jobengine.executors/jobtemplates/harris-missingmaterial-checker.xml index 060ecc31..570b57e9 100644 --- a/server/user.jobengine.executors/jobtemplates/harris-missingmaterial-checker.xml +++ b/server/user.jobengine.executors/jobtemplates/harris-missingmaterial-checker.xml @@ -1,6 +1,6 @@ - + diff --git a/server/user.jobengine.executors/jobtemplates/nexio-archive-checker.xml b/server/user.jobengine.executors/jobtemplates/nexio-archive-checker.xml index 063c4be1..a785ebdc 100644 --- a/server/user.jobengine.executors/jobtemplates/nexio-archive-checker.xml +++ b/server/user.jobengine.executors/jobtemplates/nexio-archive-checker.xml @@ -6,7 +6,6 @@ - @@ -21,9 +20,6 @@ - - - diff --git a/server/user.jobengine.executors/jobtemplates/peablebeach-missingmaterial-checker.xml b/server/user.jobengine.executors/jobtemplates/peablebeach-missingmaterial-checker.xml index b7cf9495..b05b2ef8 100644 --- a/server/user.jobengine.executors/jobtemplates/peablebeach-missingmaterial-checker.xml +++ b/server/user.jobengine.executors/jobtemplates/peablebeach-missingmaterial-checker.xml @@ -1,6 +1,6 @@ - + diff --git a/server/user.jobengine.executors/jobtemplates/register-user-restore.xml b/server/user.jobengine.executors/jobtemplates/register-user-restore.xml index 6870f979..eae8c9b5 100644 --- a/server/user.jobengine.executors/jobtemplates/register-user-restore.xml +++ b/server/user.jobengine.executors/jobtemplates/register-user-restore.xml @@ -1,5 +1,5 @@ - + diff --git a/server/user.jobengine.executors/jobtemplates/validate-and-restore.xml b/server/user.jobengine.executors/jobtemplates/validate-and-restore.xml index 99e6e049..43fa0c1e 100644 --- a/server/user.jobengine.executors/jobtemplates/validate-and-restore.xml +++ b/server/user.jobengine.executors/jobtemplates/validate-and-restore.xml @@ -1,8 +1,9 @@ - + + @@ -19,7 +20,7 @@ - + @@ -67,13 +68,16 @@ - - - - - - - + + + + + + + + + + diff --git a/server/user.jobengine.executors/jobtemplates/validate-dir-mxf.xml b/server/user.jobengine.executors/jobtemplates/validate-dir-mxf.xml new file mode 100644 index 00000000..276152ec --- /dev/null +++ b/server/user.jobengine.executors/jobtemplates/validate-dir-mxf.xml @@ -0,0 +1,17 @@ + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/server/user.jobengine.executors/resources/nexio_meta.xml b/server/user.jobengine.executors/resources/nexio_meta.xml new file mode 100644 index 00000000..0a86beeb Binary files /dev/null and b/server/user.jobengine.executors/resources/nexio_meta.xml differ diff --git a/server/user.jobengine.executors/resources/pb_getcustomview_response.xml b/server/user.jobengine.executors/resources/pb_getcustomview_response.xml new file mode 100644 index 00000000..7c9e3486 --- /dev/null +++ b/server/user.jobengine.executors/resources/pb_getcustomview_response.xml @@ -0,0 +1,109 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/server/user.jobengine.executors/resources/pb_getmediausagebyutrange_response.xml b/server/user.jobengine.executors/resources/pb_getmediausagebyutrange_response.xml new file mode 100644 index 00000000..bd5808b1 --- /dev/null +++ b/server/user.jobengine.executors/resources/pb_getmediausagebyutrange_response.xml @@ -0,0 +1,3431 @@ + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/server/user.jobengine.executors/src/user/jobengine/server/steps/ArchiveListBuilderStep.java b/server/user.jobengine.executors/src/user/jobengine/server/steps/ArchiveListBuilderStep.java index 7774de97..cf973249 100644 --- a/server/user.jobengine.executors/src/user/jobengine/server/steps/ArchiveListBuilderStep.java +++ b/server/user.jobengine.executors/src/user/jobengine/server/steps/ArchiveListBuilderStep.java @@ -85,7 +85,7 @@ public class ArchiveListBuilderStep extends JobStep { @StepEntry public Object[] execute(String sourcePath, int limit, IJobEngine jobEngine, IJobRuntime jobRuntime) { - marker = jobRuntime.getMarker(); + marker = jobRuntime.getSessionMarker(); List archiveList = new LinkedList(); DirectoryStream directoryStream = null; try { diff --git a/server/user.jobengine.executors/src/user/jobengine/server/steps/ArchiveListBuilderStep2.java b/server/user.jobengine.executors/src/user/jobengine/server/steps/ArchiveListBuilderStep2.java index 0f93d62d..c921a7d7 100644 --- a/server/user.jobengine.executors/src/user/jobengine/server/steps/ArchiveListBuilderStep2.java +++ b/server/user.jobengine.executors/src/user/jobengine/server/steps/ArchiveListBuilderStep2.java @@ -88,7 +88,7 @@ public class ArchiveListBuilderStep2 extends JobStep { @StepEntry public Object[] execute(String sourcePath, int limit, IJobEngine jobEngine, IJobRuntime jobRuntime) { - marker = jobRuntime.getMarker(); + marker = jobRuntime.getSessionMarker(); List archiveList = new LinkedList(); DirectoryStream directoryStream = null; try { diff --git a/server/user.jobengine.executors/src/user/jobengine/server/steps/ArchiveMaterialSubmitStep.java b/server/user.jobengine.executors/src/user/jobengine/server/steps/ArchiveMaterialSubmitStep.java index 4a8d30eb..166a200e 100644 --- a/server/user.jobengine.executors/src/user/jobengine/server/steps/ArchiveMaterialSubmitStep.java +++ b/server/user.jobengine.executors/src/user/jobengine/server/steps/ArchiveMaterialSubmitStep.java @@ -21,7 +21,7 @@ public class ArchiveMaterialSubmitStep extends JobStep { @StepEntry public Object[] execute(List archiveList, int killDateDays, IJobEngine jobEngine, IJobRuntime jobRuntime) throws Exception { - marker = jobRuntime.getMarker(); + marker = jobRuntime.getSessionMarker(); if (archiveList == null || archiveList.size() == 0) return null; diff --git a/server/user.jobengine.executors/src/user/jobengine/server/steps/BatchRetrieveForkStep.java b/server/user.jobengine.executors/src/user/jobengine/server/steps/BatchRetrieveForkStep.java index 839d6495..15cf36c1 100644 --- a/server/user.jobengine.executors/src/user/jobengine/server/steps/BatchRetrieveForkStep.java +++ b/server/user.jobengine.executors/src/user/jobengine/server/steps/BatchRetrieveForkStep.java @@ -25,7 +25,7 @@ public class BatchRetrieveForkStep extends JobStep { @StepEntry public Object[] execute(List basket, String houseId, String recipient, String targetPathType, IJobEngine jobEngine, IJobRuntime jobRuntime) throws Exception { - marker = (MediaCubeMarker) jobRuntime.getMarker(); + marker = (MediaCubeMarker) jobRuntime.getSessionMarker(); //session szinten csak a finishMarker cimzettje az erdekes, es ezt a cimet pluszban hasznalja a konfigban megadott cimmel //a finishMarker orokli a cim bellitast a sessionMarkertol @@ -65,7 +65,7 @@ public class BatchRetrieveForkStep extends JobStep { parameters.put(RECIPIENT, recipient); parameters.put(TARGET_PATH_TYPE, targetPathType); IJobRuntime child = jobEngine.submit(jobRuntime, null, CHILD_TEMPLATE, String.format("Visszatöltés %s részére", recipient), parameters); - ((MediaCubeMarker) child.getMarker()).setTo(recipient); + ((MediaCubeMarker) child.getSessionMarker()).setTo(recipient); } catch (Exception e) { logger.catching(e); logger.error(marker, "Hiba a kötegelt visszatöltésben. A rendszer üzenete: {}", e.getMessage()); diff --git a/server/user.jobengine.executors/src/user/jobengine/server/steps/CancelableStep.java b/server/user.jobengine.executors/src/user/jobengine/server/steps/CancelableStep.java index db37b4ca..dae68860 100644 --- a/server/user.jobengine.executors/src/user/jobengine/server/steps/CancelableStep.java +++ b/server/user.jobengine.executors/src/user/jobengine/server/steps/CancelableStep.java @@ -29,6 +29,8 @@ public class CancelableStep extends JobStep { // getJobRuntime().setDescription(String.valueOf(param)); try { + //logger.info(new MediaCubeMarker("vasary@elgekko.net", "TESZT"), "Heloka"); + logger.error(getSessionMarker(), "Heloka"); for (int i = 0; i < count; i++) { if (getJobRuntime().isWaitingCancel()) break; diff --git a/server/user.jobengine.executors/src/user/jobengine/server/steps/CheckLOWRESIntegrity.java b/server/user.jobengine.executors/src/user/jobengine/server/steps/CheckLOWRESIntegrity.java index 5cc81690..373545eb 100644 --- a/server/user.jobengine.executors/src/user/jobengine/server/steps/CheckLOWRESIntegrity.java +++ b/server/user.jobengine.executors/src/user/jobengine/server/steps/CheckLOWRESIntegrity.java @@ -32,7 +32,7 @@ public class CheckLOWRESIntegrity extends JobStep { @StepEntry public Object[] execute(String webPath, IJobEngine jobEngine, IJobRuntime jobRuntime) throws Exception { this.webPath = webPath; - marker = jobRuntime.getMarker(); + marker = jobRuntime.getSessionMarker(); manager = jobEngine.getItemManager(); List mediaIdentities = getTranscodedMediaIdentities(); int allCount = mediaIdentities.size(); diff --git a/server/user.jobengine.executors/src/user/jobengine/server/steps/CleanupMountedLocationStep.java b/server/user.jobengine.executors/src/user/jobengine/server/steps/CleanupMountedLocationStep.java index d6559574..5bcad928 100644 --- a/server/user.jobengine.executors/src/user/jobengine/server/steps/CleanupMountedLocationStep.java +++ b/server/user.jobengine.executors/src/user/jobengine/server/steps/CleanupMountedLocationStep.java @@ -79,7 +79,7 @@ public class CleanupMountedLocationStep extends JobStep implements FileVisitor

directoryStream = null; if (StringUtils.isBlank(sourcePath.toString())) { diff --git a/server/user.jobengine.executors/src/user/jobengine/server/steps/CopyForArchiveNEXIOMaterialsStep.java b/server/user.jobengine.executors/src/user/jobengine/server/steps/CopyForArchiveNEXIOMaterialsStep.java index 75fedded..5fdb9918 100644 --- a/server/user.jobengine.executors/src/user/jobengine/server/steps/CopyForArchiveNEXIOMaterialsStep.java +++ b/server/user.jobengine.executors/src/user/jobengine/server/steps/CopyForArchiveNEXIOMaterialsStep.java @@ -172,7 +172,7 @@ public class CopyForArchiveNEXIOMaterialsStep extends JobStep { @StepEntry public Object[] execute(int nexioPort, String nexioUserName, String nexioPassword, String archiveFtp, String archiveUserName, String archivePassword, int daysBeforeNow, int nexioKillDateDays, String nexioAgency, IJobEngine jobEngine, IJobRuntime jobRuntime) throws Exception { - systemMarker = jobRuntime.getMarker(); + systemMarker = jobRuntime.getSessionMarker(); setAndCheck(nexioPort, nexioUserName, nexioPassword, archiveFtp, archiveUserName, archivePassword, nexioKillDateDays, nexioAgency, jobEngine); octopusAPI = new OctopusAPI(); Calendar scheduledDate = Calendar.getInstance(); diff --git a/server/user.jobengine.executors/src/user/jobengine/server/steps/CopyForArchiveNEXIORecordingsStep.java b/server/user.jobengine.executors/src/user/jobengine/server/steps/CopyForArchiveNEXIORecordingsStep.java index 0b98ad33..5424fd36 100644 --- a/server/user.jobengine.executors/src/user/jobengine/server/steps/CopyForArchiveNEXIORecordingsStep.java +++ b/server/user.jobengine.executors/src/user/jobengine/server/steps/CopyForArchiveNEXIORecordingsStep.java @@ -151,7 +151,7 @@ public class CopyForArchiveNEXIORecordingsStep extends JobStep { @StepEntry public Object[] execute(int nexioPort, String nexioUserName, String nexioPassword, String archiveFtp, String archiveUserName, String archivePassword, String filterAgencies, int limit, int nexioKillDateDays, String targetAgency, IJobEngine jobEngine, IJobRuntime jobRuntime) throws Exception { - systemMarker = jobRuntime.getMarker(); + systemMarker = jobRuntime.getSessionMarker(); setAndCheck(nexioPort, nexioUserName, nexioPassword, archiveFtp, archiveUserName, archivePassword, filterAgencies, limit, nexioKillDateDays, targetAgency, jobEngine); octopusAPI = new OctopusAPI(); diff --git a/server/user.jobengine.executors/src/user/jobengine/server/steps/CreateMissingLowresStep.java b/server/user.jobengine.executors/src/user/jobengine/server/steps/CreateMissingLowresStep.java index 339d0417..dadb7f72 100644 --- a/server/user.jobengine.executors/src/user/jobengine/server/steps/CreateMissingLowresStep.java +++ b/server/user.jobengine.executors/src/user/jobengine/server/steps/CreateMissingLowresStep.java @@ -24,7 +24,7 @@ public class CreateMissingLowresStep extends JobStep { @StepEntry public Object[] execute(String localHiresPath, IJobEngine jobEngine, IJobRuntime jobRuntime) throws Exception { Object[] result = { null, null, "%s", null, 0, true }; - marker = (MediaCubeMarker) jobRuntime.getMarker(); + marker = (MediaCubeMarker) jobRuntime.getSessionMarker(); DB db = NoSQLUtils.getNoSQLDB(); DBCollection collection = db.getCollection("missing_lowres"); IItemManager manager = jobEngine.getItemManager(); diff --git a/server/user.jobengine.executors/src/user/jobengine/server/steps/DeleteFileStep.java b/server/user.jobengine.executors/src/user/jobengine/server/steps/DeleteFileStep.java index 68ae1d64..7682440f 100644 --- a/server/user.jobengine.executors/src/user/jobengine/server/steps/DeleteFileStep.java +++ b/server/user.jobengine.executors/src/user/jobengine/server/steps/DeleteFileStep.java @@ -17,7 +17,7 @@ public class DeleteFileStep extends JobStep { @StepEntry public Object[] execute(ArchiveItem archiveItem, boolean isDelete, IJobEngine jobEngine, IJobRuntime jobRuntime) throws Exception { - marker = jobRuntime.getMarker(); + marker = jobRuntime.getSessionMarker(); if (isDelete) { Path filePath = Paths.get(archiveItem.getMediaFile()); File file = filePath.toFile(); diff --git a/server/user.jobengine.executors/src/user/jobengine/server/steps/DeleteNEXIOMaterialsStep.java b/server/user.jobengine.executors/src/user/jobengine/server/steps/DeleteNEXIOMaterialsStep.java index cd14224d..db1693a7 100644 --- a/server/user.jobengine.executors/src/user/jobengine/server/steps/DeleteNEXIOMaterialsStep.java +++ b/server/user.jobengine.executors/src/user/jobengine/server/steps/DeleteNEXIOMaterialsStep.java @@ -39,13 +39,13 @@ public class DeleteNEXIOMaterialsStep extends JobStep { try { remoteFile = sourceUri.getRemoteFile(name); if (remoteFile == null) { - logger.warn(jobRuntime.getMarker(), "A '{}' fájl már nem található meg a NEXIO szerveren", name); + logger.warn(jobRuntime.getSessionMarker(), "A '{}' fájl már nem található meg a NEXIO szerveren", name); return; } sourceUri.delete(remoteFile); - logger.info(jobRuntime.getMarker(), "A '{}' fájl törlése sikeres volt.", remoteFile.getName()); + logger.info(jobRuntime.getSessionMarker(), "A '{}' fájl törlése sikeres volt.", remoteFile.getName()); } catch (Exception e) { - logger.error(jobRuntime.getMarker(), "A '{}' fájl nem törölhető. A rendszer hibaüzenete: {}", remoteFile.getName(), e.getMessage()); + logger.error(jobRuntime.getSessionMarker(), "A '{}' fájl nem törölhető. A rendszer hibaüzenete: {}", remoteFile.getName(), e.getMessage()); } } @@ -55,12 +55,12 @@ public class DeleteNEXIOMaterialsStep extends JobStep { this.jobRuntime = jobRuntime; String nexioHost = System.getProperty("nexio.host"); if (StringUtils.isBlank(nexioHost)) { - logger.error(jobRuntime.getMarker(), "A 'nexio.host' rendszer paraméter nem található."); + logger.error(jobRuntime.getSessionMarker(), "A 'nexio.host' rendszer paraméter nem található."); throw new NullPointerException("System is not configured properly, 'nexio.host' startup parameter missing."); } if (StringUtils.isBlank(filterAgencies)) { - logger.error(jobRuntime.getMarker(), "A 'nexioAgencies' folyamat paraméter nem található."); + logger.error(jobRuntime.getSessionMarker(), "A 'nexioAgencies' folyamat paraméter nem található."); throw new NullPointerException("System is not configured properly, 'nexioAgencies' job parameter missing."); } @@ -95,7 +95,7 @@ public class DeleteNEXIOMaterialsStep extends JobStep { String name = String.valueOf(clip.get(LONGNAMEID)); Date killdate = clip.getDate(KILLDATE); if (notificationOnly) - logger.info(jobRuntime.getMarker(), "Az {} fájl törölhető. Lejárt: {} ", name, df.format(killdate)); + logger.info(jobRuntime.getSessionMarker(), "Az {} fájl törölhető. Lejárt: {} ", name, df.format(killdate)); else delete(name); i++; diff --git a/server/user.jobengine.executors/src/user/jobengine/server/steps/DetectMissingLengthStep.java b/server/user.jobengine.executors/src/user/jobengine/server/steps/DetectMissingLengthStep.java index 81d6d89a..d3180f51 100644 --- a/server/user.jobengine.executors/src/user/jobengine/server/steps/DetectMissingLengthStep.java +++ b/server/user.jobengine.executors/src/user/jobengine/server/steps/DetectMissingLengthStep.java @@ -25,7 +25,7 @@ public class DetectMissingLengthStep extends JobStep { @StepEntry public Object[] execute(String localHiresPath, IJobEngine jobEngine, IJobRuntime jobRuntime) throws Exception { Object[] result = { null, null, "%s", null, 0, true }; - marker = jobRuntime.getMarker(); + marker = jobRuntime.getSessionMarker(); DB db = NoSQLUtils.getNoSQLDB(); DBCollection collection = db.getCollection("missing_length"); diff --git a/server/user.jobengine.executors/src/user/jobengine/server/steps/DirMXFValidatorStep.java b/server/user.jobengine.executors/src/user/jobengine/server/steps/DirMXFValidatorStep.java new file mode 100644 index 00000000..104806e4 --- /dev/null +++ b/server/user.jobengine.executors/src/user/jobengine/server/steps/DirMXFValidatorStep.java @@ -0,0 +1,69 @@ +package user.jobengine.server.steps; + +import java.nio.file.DirectoryStream; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + +import user.commons.mediatool.MediaInfo; +import user.jobengine.db.Media; + +public class DirMXFValidatorStep extends JobStep { + private static final Logger logger = LogManager.getLogger(); + + @StepEntry + public Object[] execute(String source) throws Exception { + Path path = Paths.get(source); + DirectoryStream directoryStream = Files.newDirectoryStream(path); + for (Path file : directoryStream) { + if (getJobRuntime().isWaitingCancel()) { + cancel(); + return null; + } + + if (file.toFile().isDirectory()) + continue; + + String fileName = file.getFileName().toString(); + if (fileName.contains(".")) + fileName = fileName.substring(0, fileName.lastIndexOf(".")); + + if (file.toFile().length() == 0) { + logger.error("Zero: {}", fileName); + continue; + } + + Media media = getManager().getMedia(fileName); + if (media == null) { + logger.error("Unchecked: {}", fileName); + continue; + } + + long frames = media.getLength(); + + MediaInfo mi = null; + + try { + mi = new MediaInfo(file); + mi.process(); + } catch (Exception e) { + logger.error(getSessionMarker(), e.getMessage()); + logger.error("Unreadable: {}", fileName); + } + + if (frames == 0) { + logger.info("Ma valid: {}", fileName); + continue; + } + + if (Math.abs(frames - mi.getFrames()) < 4) { + logger.info("Valid: {} {} {}", fileName, frames, mi.getFrames()); + } + } + + return null; + } +} diff --git a/server/user.jobengine.executors/src/user/jobengine/server/steps/DownloadRecordingFromNexioStep.java b/server/user.jobengine.executors/src/user/jobengine/server/steps/DownloadRecordingFromNexioStep.java index c334f89c..c764a8a6 100644 --- a/server/user.jobengine.executors/src/user/jobengine/server/steps/DownloadRecordingFromNexioStep.java +++ b/server/user.jobengine.executors/src/user/jobengine/server/steps/DownloadRecordingFromNexioStep.java @@ -59,7 +59,7 @@ public class DownloadRecordingFromNexioStep extends JobStep { @StepEntry public Object[] execute(ArchiveItem archiveItem, String targetPath, String targetFileName, int nexioPort, String nexioUserName, String nexioPassword, IJobEngine jobEngine, IJobRuntime jobRuntime) throws Exception { - marker = jobRuntime.getMarker(); + marker = jobRuntime.getSessionMarker(); manager = jobEngine.getItemManager(); setAndCheck(archiveItem, targetPath, targetFileName, nexioPort, nexioUserName, nexioPassword); String sourceFileName = targetFileName; diff --git a/server/user.jobengine.executors/src/user/jobengine/server/steps/DuplicateRemoverStep.java b/server/user.jobengine.executors/src/user/jobengine/server/steps/DuplicateRemoverStep.java index f8c24c58..132fa3cc 100644 --- a/server/user.jobengine.executors/src/user/jobengine/server/steps/DuplicateRemoverStep.java +++ b/server/user.jobengine.executors/src/user/jobengine/server/steps/DuplicateRemoverStep.java @@ -25,7 +25,7 @@ public class DuplicateRemoverStep extends JobStep { @StepEntry public Object[] execute(int limit, IJobEngine jobEngine, IJobRuntime jobRuntime) throws Exception { - marker = jobRuntime.getMarker(); + marker = jobRuntime.getSessionMarker(); manager = jobEngine.getItemManager(); processLowresDuplicates(limit); return null; diff --git a/server/user.jobengine.executors/src/user/jobengine/server/steps/EscortFiles.java b/server/user.jobengine.executors/src/user/jobengine/server/steps/EscortFiles.java index 7d7a76b1..e885a7a0 100644 --- a/server/user.jobengine.executors/src/user/jobengine/server/steps/EscortFiles.java +++ b/server/user.jobengine.executors/src/user/jobengine/server/steps/EscortFiles.java @@ -1,9 +1,12 @@ package user.jobengine.server.steps; +import java.io.ByteArrayInputStream; import java.io.File; import java.io.IOException; +import java.io.InputStream; import java.io.OutputStream; import java.io.StringWriter; +import java.io.UnsupportedEncodingException; import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.Paths; @@ -19,7 +22,10 @@ import javax.xml.parsers.DocumentBuilder; import javax.xml.parsers.DocumentBuilderFactory; import javax.xml.transform.OutputKeys; import javax.xml.transform.Transformer; +import javax.xml.transform.TransformerConfigurationException; +import javax.xml.transform.TransformerException; import javax.xml.transform.TransformerFactory; +import javax.xml.transform.TransformerFactoryConfigurationError; import javax.xml.transform.dom.DOMSource; import javax.xml.transform.stream.StreamResult; @@ -28,17 +34,22 @@ import org.apache.commons.net.ftp.FTPClient; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.apache.logging.log4j.Marker; +import org.apache.logging.log4j.message.Message; import org.w3c.dom.DOMImplementation; import org.w3c.dom.Document; import org.w3c.dom.Element; +import com.ibm.nosql.json.api.BasicDBObject; import com.ibm.nosql.json.util.JSON; import user.commons.CalendarUtils; +import user.commons.MediaCubeMarker; import user.commons.StoreUri; import user.commons.remotestore.FtpDirectoryLister; public class EscortFiles { + private static final String RECORDTIMESTAMP = "RecordTimeStamp"; + private static final String MODIFIEDTIMESTAMP = "ModifiedTimeStamp"; public static final String DOT_CATCHED = ".catched"; public static final String DOT_JSON = ".json"; private static final Logger logger = LogManager.getLogger(); @@ -102,6 +113,24 @@ public class EscortFiles { Files.write(xmlPath, content.getBytes()); } + public static byte[] createNEXIODatesMeta(String fileName, Date recorded, Date modified) throws Exception { + DocumentBuilderFactory dbf = DocumentBuilderFactory.newInstance(); + DocumentBuilder db = dbf.newDocumentBuilder(); + DOMImplementation impl = db.getDOMImplementation(); + Document xmlDocument = impl.createDocument(null, null, null); + + Element root = xmlDocument.createElement(ID); + root.setAttribute(EXTENDEDID, fileName); + // 07-13-2020 (19:36:52) + // 05-18-2013 (18:52:24) + SimpleDateFormat df = new SimpleDateFormat("MM-dd-yyyy (HH:mm:ss)"); + root.appendChild(xmlDocument.createElement(MODIFIEDTIMESTAMP)).appendChild(xmlDocument.createTextNode(df.format(modified))); + root.appendChild(xmlDocument.createElement(RECORDTIMESTAMP)).appendChild(xmlDocument.createTextNode(df.format(recorded))); + xmlDocument.appendChild(root); + + return xmDocumentToString(xmlDocument); + } + public static byte[] createNEXIOKillDateFile(String fileName, Date killDate, String description, String agency) throws Exception { DocumentBuilderFactory dbf = DocumentBuilderFactory.newInstance(); DocumentBuilder db = dbf.newDocumentBuilder(); @@ -121,20 +150,22 @@ public class EscortFiles { root.appendChild(xmlDocument.createElement(EXTENDEDAGENCY)).appendChild(xmlDocument.createTextNode(agency)); xmlDocument.appendChild(root); - DOMSource domSource = new DOMSource(xmlDocument); - TransformerFactory tf = TransformerFactory.newInstance(); - Transformer transformer = tf.newTransformer(); - // transformer.setOutputProperty(OutputKeys.OMIT_XML_DECLARATION, - // "yes"); - transformer.setOutputProperty(OutputKeys.METHOD, "xml"); - transformer.setOutputProperty(OutputKeys.ENCODING, "UTF-16"); - transformer.setOutputProperty(OutputKeys.INDENT, "yes"); - StringWriter sw = new StringWriter(); - StreamResult sr = new StreamResult(sw); - transformer.transform(domSource, sr); - String result = sw.toString(); - sw.close(); - return result.getBytes("UTF-16"); + return xmDocumentToString(xmlDocument); + } + + public static Document createNEXIOMeta(byte[] content) throws Exception { + DocumentBuilderFactory dbf = DocumentBuilderFactory.newInstance(); + DocumentBuilder db = dbf.newDocumentBuilder(); + DOMImplementation impl = db.getDOMImplementation(); + Document xmlDocument = null; + + try (InputStream is = new ByteArrayInputStream(content)) { + xmlDocument = db.parse(is); + } catch (Exception e) { + logger.catching(e); + } + + return xmlDocument; } public static void createUNCKillDate(String filePath, String fileName, int days, Marker marker) throws IOException { @@ -155,7 +186,7 @@ public class EscortFiles { String content = new String(bytes); result = (T) JSON.parse(content); } catch (Exception e) { - logger.error(e.getMessage()); + logger.error("Decode error. System message is: ", e.getMessage()); } return result; } @@ -197,6 +228,22 @@ public class EscortFiles { return result; } + public static void notifyRecipient(Path escortFile, Logger logger, Message msg) { + if (escortFile.toFile().exists()) { + try { + BasicDBObject downloadable = EscortFiles.decode(escortFile); + String recipientKey = "recipient"; + if (downloadable.containsKey(recipientKey)) { + String recipient = downloadable.getString(recipientKey); + logger.info(new MediaCubeMarker(recipient, "MediaCube rendszerüzenet"), msg); + } + } catch (Exception e) { + logger.catching(e); + } + + } + } + public static void remove(Path file) { try { file.toFile().delete(); @@ -234,4 +281,20 @@ public class EscortFiles { targetUri.cleanUp(); } } + + private static byte[] xmDocumentToString(Document xmlDocument) + throws TransformerFactoryConfigurationError, TransformerConfigurationException, TransformerException, IOException, UnsupportedEncodingException { + DOMSource domSource = new DOMSource(xmlDocument); + TransformerFactory tf = TransformerFactory.newInstance(); + Transformer transformer = tf.newTransformer(); + transformer.setOutputProperty(OutputKeys.METHOD, "xml"); + transformer.setOutputProperty(OutputKeys.ENCODING, "UTF-16"); + transformer.setOutputProperty(OutputKeys.INDENT, "yes"); + StringWriter sw = new StringWriter(); + StreamResult sr = new StreamResult(sw); + transformer.transform(domSource, sr); + String result = sw.toString(); + sw.close(); + return result.getBytes("UTF-16"); + } } diff --git a/server/user.jobengine.executors/src/user/jobengine/server/steps/FileCleanupStep.java b/server/user.jobengine.executors/src/user/jobengine/server/steps/FileCleanupStep.java index c5387649..505526cf 100644 --- a/server/user.jobengine.executors/src/user/jobengine/server/steps/FileCleanupStep.java +++ b/server/user.jobengine.executors/src/user/jobengine/server/steps/FileCleanupStep.java @@ -5,27 +5,25 @@ import java.nio.file.Paths; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; +import org.apache.logging.log4j.message.Message; +import org.apache.logging.log4j.message.ParameterizedMessage; import user.commons.StoreUri; public class FileCleanupStep extends JobStep { private static final Logger logger = LogManager.getLogger(); - private void deleteSilently(Path file) { - try { - file.toFile().delete(); - } catch (Exception e) { - logger.error("Unable to delete {}", file.toAbsolutePath().toString()); - } - } - @StepEntry public Object[] execute(StoreUri sourceStoreUri, String fileName, String escortFile) throws Exception { try { + + Message msg = new ParameterizedMessage("A {} fájl visszatöltése befejeződött.", fileName); + EscortFiles.notifyRecipient(Paths.get(escortFile), logger, msg); + Path filePath = Paths.get(sourceStoreUri.toString(true), fileName); - deleteSilently(filePath); - deleteSilently(Paths.get(escortFile)); - deleteSilently(Paths.get(escortFile + EscortFiles.DOT_CATCHED)); + EscortFiles.remove(filePath); + EscortFiles.remove(Paths.get(escortFile)); + EscortFiles.remove(Paths.get(escortFile + EscortFiles.DOT_CATCHED)); } catch (Exception e) { logger.error(e.getMessage()); throw e; diff --git a/server/user.jobengine.executors/src/user/jobengine/server/steps/FileCopyStep.java b/server/user.jobengine.executors/src/user/jobengine/server/steps/FileCopyStep.java index 4d5633a6..c4172550 100644 --- a/server/user.jobengine.executors/src/user/jobengine/server/steps/FileCopyStep.java +++ b/server/user.jobengine.executors/src/user/jobengine/server/steps/FileCopyStep.java @@ -57,7 +57,7 @@ public class FileCopyStep extends JobStep { @StepEntry public Object[] execute(String sourceProtocol, String sourcePath, String sourceFileName, String targetProtocol, String targetPath, String targetFileName, int killDateDays, IJobEngine jobEngine, IJobRuntime jobRuntime) throws Exception { - marker = jobRuntime.getMarker(); + marker = jobRuntime.getSessionMarker(); check(sourceProtocol, sourcePath, sourceFileName, targetProtocol, targetPath, targetFileName, jobEngine, jobRuntime); diff --git a/server/user.jobengine.executors/src/user/jobengine/server/steps/FileValidatorStep.java b/server/user.jobengine.executors/src/user/jobengine/server/steps/FileValidatorStep.java index fee9abe2..101c66bd 100644 --- a/server/user.jobengine.executors/src/user/jobengine/server/steps/FileValidatorStep.java +++ b/server/user.jobengine.executors/src/user/jobengine/server/steps/FileValidatorStep.java @@ -7,24 +7,20 @@ import java.nio.file.Paths; import org.apache.commons.lang.StringUtils; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; +import org.apache.logging.log4j.message.Message; +import org.apache.logging.log4j.message.ParameterizedMessage; +import user.commons.DownloadableMedia; import user.commons.StoreUri; import user.commons.mediatool.MediaInfo; import user.commons.remotestore.RemoteStoreProtocol; +import user.jobengine.db.Media; public class FileValidatorStep extends JobStep { private static final String COLORSPACE = "COLORSPACE"; private static final Logger logger = LogManager.getLogger(); private static final String testSimulate = System.getProperty("test.simulate"); - private void deleteSilently(Path file) { - try { - file.toFile().delete(); - } catch (Exception e) { - logger.error("Unable to delete {}", file.toAbsolutePath().toString()); - } - } - @StepEntry public Object[] execute(StoreUri sourceStoreUri, String fileName, long expectedFrameNumber, long expectedSize, String exceptedColorSpace, String escortFile) throws Exception { @@ -34,14 +30,33 @@ public class FileValidatorStep extends JobStep { if (!filePath.toFile().exists()) throw new FileNotFoundException("File not exists: " + filePath.toString()); + Path escortFilePath = Paths.get(escortFile); + if (getJobRuntime().isWaitingCancel()) { - deleteSilently(filePath); - deleteSilently(Paths.get(escortFile)); - deleteSilently(Paths.get(escortFile + EscortFiles.DOT_CATCHED)); + EscortFiles.remove(filePath); + EscortFiles.remove(escortFilePath); + EscortFiles.remove(Paths.get(escortFile + EscortFiles.DOT_CATCHED)); + + Message msg = new ParameterizedMessage("A {} fájl visszatöltése kezelői beavatkozás miatt megszakadt.", fileName); + EscortFiles.notifyRecipient(escortFilePath, logger, msg); + cancel(); return null; } + DownloadableMedia downloadable = null; + try { + downloadable = EscortFiles.decode(escortFilePath); + } catch (Exception e) { + logger.error(getSessionMarker(), e.getMessage()); + } + + if (downloadable != null && expectedFrameNumber == 0) + storeCurrentFrames(filePath, downloadable); + + if (downloadable != null && downloadable.containsKey("skipValidation") && downloadable.getBoolean("skipValidation")) + return null; + if (expectedFrameNumber > 0 || StringUtils.isNotBlank(exceptedColorSpace)) validateMedia(filePath, expectedFrameNumber, exceptedColorSpace, escortFile); @@ -49,16 +64,38 @@ public class FileValidatorStep extends JobStep { validateSize(filePath, expectedSize, escortFile); //A metadata persister miatt torlunk mindet teszt modban + /* if (StringUtils.isNotBlank(testSimulate)) { - deleteSilently(Paths.get(escortFile)); - deleteSilently(Paths.get(escortFile + EscortFiles.DOT_CATCHED)); - logger.info(getSessionMarker(), "{} deleted to prevent real processing", escortFile); + EscortFiles.remove(escortFilePath); + EscortFiles.remove(Paths.get(escortFile + EscortFiles.DOT_CATCHED)); + logger.info(getMarker(), "{} deleted to prevent real processing", escortFile); } - - logger.info(getSessionMarker(), "{} passed validation", fileName); + */ + logger.info(getMarker(), "{} passed validation", fileName); return null; } + private void storeCurrentFrames(Path filePath, DownloadableMedia downloadable) { + MediaInfo mi = null; + try { + mi = new MediaInfo(filePath); + mi.process(); + long frames = mi.getFrames(); + + if (downloadable.containsKey("mediaId")) { + Media media = getManager().getMedia(downloadable.getLong("mediaId")); + if (media != null) { + media.setLength(frames); + getManager().modify(media); + } + + } + + } catch (Exception e) { + logger.error(getSessionMarker(), e.getMessage()); + } + } + private void validateMedia(Path filePath, long expectedFrameNumber, String exceptedColorSpace, String escortFile) throws Exception { MediaInfo mi = null; try { @@ -67,12 +104,18 @@ public class FileValidatorStep extends JobStep { if (expectedFrameNumber > 0) { long frames = mi.getFrames(); - if (frames != expectedFrameNumber) { - deleteSilently(filePath); - deleteSilently(Paths.get(escortFile)); - deleteSilently(Paths.get(escortFile + EscortFiles.DOT_CATCHED)); + if (Math.abs(frames - expectedFrameNumber) > 3) { + Message msg = new ParameterizedMessage("Validációs hiba miatt a {} fájl másolása nem lehetséges.", filePath.getFileName()); + EscortFiles.notifyRecipient(Paths.get(escortFile), logger, msg); + + EscortFiles.remove(filePath); + EscortFiles.remove(Paths.get(escortFile)); + EscortFiles.remove(Paths.get(escortFile + EscortFiles.DOT_CATCHED)); throw new Exception("Length mismatch in " + filePath + ". Expected: " + expectedFrameNumber + ", found: " + frames); } + if (frames != expectedFrameNumber) + logger.warn(getSessionMarker(), "A {} fájl másolása lehetséges, de a képkockaszám nem egyezik. Várt: {}, aktuális: {}", + expectedFrameNumber, frames); } if (StringUtils.isNotBlank(exceptedColorSpace)) { diff --git a/server/user.jobengine.executors/src/user/jobengine/server/steps/ForkDownloadStep.java b/server/user.jobengine.executors/src/user/jobengine/server/steps/ForkDownloadStep.java index 8be4b846..7923ae65 100644 --- a/server/user.jobengine.executors/src/user/jobengine/server/steps/ForkDownloadStep.java +++ b/server/user.jobengine.executors/src/user/jobengine/server/steps/ForkDownloadStep.java @@ -44,11 +44,14 @@ public class ForkDownloadStep extends JobStep { escortFiles.add(escortFile); } allCount = escortFiles.size(); - //a file neve az adasbakerules idopontjaval kezdodik + //Comparison method violates its general contract! miatt try-catch escortFiles.sort((p1, p2) -> { - if (p1.toFile().lastModified() == p2.toFile().lastModified()) - return 0; - return p1.toFile().lastModified() > p2.toFile().lastModified() ? 1 : -1; + int result = 0; + try { + result = p1.toFile().lastModified() > p2.toFile().lastModified() ? 1 : -1; + } catch (Exception e) { + } + return result; }); if (allCount > 0) { @@ -64,7 +67,8 @@ public class ForkDownloadStep extends JobStep { break; } currentCount++; - setProgress(currentCount * 100 / allCount); + int progress = currentCount * 100 / allCount; + setProgress(progress); } } @@ -108,7 +112,12 @@ public class ForkDownloadStep extends JobStep { try { DownloadableMedia downloadable = EscortFiles.decode(escortFile); - logger.info("Starting {}", template); + if (downloadable == null) { + logger.error(getSessionMarker(), "Can't encode {}", escortFile); + return false; + } + + logger.info(getMarker(), "Starting {} for {}", template, fileName); StoreUri sourceStoreUri = getManager().getStoreUri(downloadable.getLong("sourceStoreUriId")); StoreUri targetStoreUri = getManager().getStoreUri(downloadable.getLong("targetStoreUriId")); @@ -124,12 +133,13 @@ public class ForkDownloadStep extends JobStep { //IJobRuntime parent, String template, String name, int priority, Map parameters IJobRuntime child = getEngine().submit(null, e -> { - if (e.getStatus().equals(JobStatus.CANCELED) || e.getStatus().equals(JobStatus.SUSPENDED)) + if (e.getStatus().equals(JobStatus.CANCELED)) EscortFiles.removeCatchedFile(escortFile); }, template, "Archiválás", 0, IJobEngine.DEFAULT_OWNER, parameters); if (child == null) throw new Exception("Submit error."); + child.setDescription(fileName); child.setRelated(downloadable.getString("title")); EscortFiles.createCatchedFile(escortFile); diff --git a/server/user.jobengine.executors/src/user/jobengine/server/steps/ForkUploadStep.java b/server/user.jobengine.executors/src/user/jobengine/server/steps/ForkUploadStep.java index ede48c76..c6544ce6 100644 --- a/server/user.jobengine.executors/src/user/jobengine/server/steps/ForkUploadStep.java +++ b/server/user.jobengine.executors/src/user/jobengine/server/steps/ForkUploadStep.java @@ -17,6 +17,8 @@ import user.commons.DownloadableMedia; import user.commons.JobStatus; import user.commons.StoreUri; import user.commons.remotestore.RemoteStoreProtocol; +import user.jobengine.db.Media; +import user.jobengine.db.MediaFile; import user.jobengine.server.IJobEngine; import user.jobengine.server.IJobRuntime; @@ -62,7 +64,8 @@ public class ForkUploadStep extends JobStep { break; } currentCount++; - setProgress(currentCount * 100 / allCount); + int progress = currentCount * 100 / allCount; + setProgress(progress); } } @@ -82,6 +85,19 @@ public class ForkUploadStep extends JobStep { return null; } + private String getTSMFileName(long mediaId, StoreUri sourceStoreUri) { + String result = null; + Media media = getManager().getMedia(mediaId); + List mediaFiles = media.getMediaFiles(); + for (MediaFile mediaFile : mediaFiles) { + if (mediaFile.getStoreId() == sourceStoreUri.getStoreId()) { + result = mediaFile.getRelativePath(); + break; + } + } + return result; + } + private boolean processPathItem(StoreUri tempStoreUri, String template, Path escortFile) throws IOException { if (escortFile.toFile().isDirectory()) return false; @@ -106,13 +122,21 @@ public class ForkUploadStep extends JobStep { try { DownloadableMedia downloadable = EscortFiles.decode(escortFile); - logger.info(getSessionMarker(), "Starting {} for {}", template, fileName); + + if (downloadable == null) { + logger.error(getSessionMarker(), "Can't encode {}", escortFile); + return false; + } + logger.info(getMarker(), "Starting {} for {}", template, fileName); StoreUri sourceStoreUri = getManager().getStoreUri("TSM", RemoteStoreProtocol.TSM); StoreUri targetStoreUri = getManager().getStoreUri(downloadable.getLong("targetStoreUriId")); + String tsmFileName = getTSMFileName(downloadable.getLong("mediaId"), sourceStoreUri); + Map parameters = new HashMap<>(); parameters.put("sourceStoreUri", sourceStoreUri); + parameters.put("tsmFileName", tsmFileName); parameters.put("fileName", downloadable.getString("fileName")); parameters.put("tempStoreUri", tempStoreUri); parameters.put("targetStoreUri", targetStoreUri); @@ -130,6 +154,8 @@ public class ForkUploadStep extends JobStep { if (child == null) throw new Exception("Submit error."); + child.setDescription(fileName); + child.setRelated(downloadable.getString("title")); EscortFiles.createCatchedFile(escortFile); } catch (Exception e) { diff --git a/server/user.jobengine.executors/src/user/jobengine/server/steps/GenerateMorpheusMetadataStep.java b/server/user.jobengine.executors/src/user/jobengine/server/steps/GenerateMorpheusMetadataStep.java index c1ab0af0..549c6c97 100644 --- a/server/user.jobengine.executors/src/user/jobengine/server/steps/GenerateMorpheusMetadataStep.java +++ b/server/user.jobengine.executors/src/user/jobengine/server/steps/GenerateMorpheusMetadataStep.java @@ -21,7 +21,7 @@ public class GenerateMorpheusMetadataStep extends JobStep { EscortFiles.createMorpheusXML(targetMetadataPath, name + ".xml", content); } catch (Exception e) { logger.catching(e); - logger.error(jobRuntime.getMarker(), "Hiba a Morpheus kísérő XML létrehozásakor. A rendszer üzenete: {}", e.getMessage()); + logger.error(jobRuntime.getSessionMarker(), "Hiba a Morpheus kísérő XML létrehozásakor. A rendszer üzenete: {}", e.getMessage()); throw e; } return null; diff --git a/server/user.jobengine.executors/src/user/jobengine/server/steps/HSMMigrateStep.java b/server/user.jobengine.executors/src/user/jobengine/server/steps/HSMMigrateStep.java index 975c87a0..507a804e 100644 --- a/server/user.jobengine.executors/src/user/jobengine/server/steps/HSMMigrateStep.java +++ b/server/user.jobengine.executors/src/user/jobengine/server/steps/HSMMigrateStep.java @@ -95,7 +95,7 @@ public class HSMMigrateStep extends JobStep { @SuppressWarnings("serial") @StepEntry public Object[] execute(String sourceLocation, String targetLocation) throws Exception { - marker = getJobRuntime().getMarker(); + marker = getJobRuntime().getSessionMarker(); //remove from prod //cleanupHistory(); hsmProvider = getMetadataProvider(MetadataProviderType.HSM); diff --git a/server/user.jobengine.executors/src/user/jobengine/server/steps/HarrisMissingMaterialCheckerStep.java b/server/user.jobengine.executors/src/user/jobengine/server/steps/HarrisMissingMaterialCheckerStep.java index d5851f59..0663d5b3 100644 --- a/server/user.jobengine.executors/src/user/jobengine/server/steps/HarrisMissingMaterialCheckerStep.java +++ b/server/user.jobengine.executors/src/user/jobengine/server/steps/HarrisMissingMaterialCheckerStep.java @@ -1,10 +1,13 @@ package user.jobengine.server.steps; +import java.io.IOException; import java.nio.file.Path; import java.nio.file.Paths; +import java.util.Arrays; import java.util.List; import org.apache.commons.lang.StringUtils; +import org.apache.commons.net.ftp.FTPClient; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; @@ -14,11 +17,12 @@ import user.commons.DownloadableMedia; import user.commons.StoreUri; import user.commons.harris.HarrisRecord; import user.commons.harris.VICFileParser; +import user.commons.remotestore.FtpDirectoryLister; import user.commons.remotestore.RemoteStoreProtocol; import user.jobengine.db.Media; public class HarrisMissingMaterialCheckerStep extends JobStep { - private static final Logger logger = LogManager.getLogger(); + private static final Logger logger = LogManager.getLogger(HarrisMissingMaterialCheckerStep.class); private static final String lineFormat = System.getProperty("harris.vic.lineformat", "A TTTTTTTT LLLLLLLLLLL MMMMMMMMMMMMMMMMMMMMMMM X DDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDD"); private static final String validTypeCodes = System.getProperty("harris.vic.validtypecodes", "SPACE,E"); @@ -27,9 +31,12 @@ public class HarrisMissingMaterialCheckerStep extends JobStep { @StepEntry public Object[] execute(BasicDBList vicFiles, String escortStoreName, String targetStoreName, String targetProtocol) throws Exception { StoreUri escortStoreUri = null; + StoreUri targetStoreUri = null; try { + getJobRuntime().setRelated(targetStoreName + " restore"); + escortStoreUri = getManager().getStoreUri(escortStoreName, RemoteStoreProtocol.LOCAL); - StoreUri targetStoreUri = getManager().getStoreUri(targetStoreName, Enum.valueOf(RemoteStoreProtocol.class, targetProtocol)); + targetStoreUri = getManager().getStoreUri(targetStoreName, Enum.valueOf(RemoteStoreProtocol.class, targetProtocol)); StoreUri sourceStoreUri = getManager().getStoreUri("TSM", RemoteStoreProtocol.TSM); List records = null; for (Object vicFile : vicFiles) { @@ -39,18 +46,21 @@ public class HarrisMissingMaterialCheckerStep extends JobStep { } Path vicFilePath = Paths.get(String.valueOf(vicFile)); - logger.info("Processing {} (exists: {})", vicFilePath, vicFilePath.toFile().exists()); - VICFileParser parser = new VICFileParser(vicFilePath, lineFormat, validTypeCodes); - List currentRecords = parser.getRecords(); - if (records == null) - records = currentRecords; - else { - if (currentRecords != null) - records.addAll(currentRecords); + + if (!vicFilePath.toFile().exists()) { + logger.warn("File not exists: {}", vicFilePath); + continue; } + + logger.info(getMarker(), "Processing {}", vicFilePath); + records = processVICFile(records, vicFilePath); } if (records != null && !getJobRuntime().isWaitingCancel()) { + FTPClient client = ((FtpDirectoryLister) targetStoreUri.getLister()).connect(); + logger.info(getMarker(), "Reading target file list"); + List fileNames = Arrays.asList(client.listNames()); + logger.info(getMarker(), "Target file list reading completed, mediabase size is {}", fileNames.size()); int i = 0; for (HarrisRecord record : records) { @@ -59,51 +69,103 @@ public class HarrisMissingMaterialCheckerStep extends JobStep { break; } - processRecord(record, targetStoreName, sourceStoreUri, targetStoreUri, escortStoreUri); + boolean contains = fileNames.contains(record.getFileName()); + if (contains) + logger.info(getMarker(), "File {} already exists on {}", record.getFileName(), targetStoreName); + else + processRecord(record, targetStoreName, sourceStoreUri, targetStoreUri, escortStoreUri); i++; int progress = i * 100 / records.size(); setProgress(progress); } } + + if (records == null || records.size() == 0) + logger.info(getMarker(), "Nothing to do"); + else + logger.info(getMarker(), "Completed"); + setProgress(100); } catch (Exception e) { - logger.error(getSessionMarker(), e.getMessage()); + logger.error(e.getMessage()); throw e; } finally { if (escortStoreUri != null) escortStoreUri.cleanUp(); + if (targetStoreUri != null) + targetStoreUri.cleanUp(); } return null; } - private void processRecord(HarrisRecord record, String targetStoreName, StoreUri sourceStoreUri, StoreUri targetStoreUri, StoreUri escortStoreUri) { - Media media = getManager().getMedia(record.getFileName()); + private boolean processRecord(HarrisRecord record, String targetStoreName, StoreUri sourceStoreUri, StoreUri targetStoreUri, StoreUri escortStoreUri) + throws Exception { String fileName = record.getFileName(); + String outputPath = Paths.get(escortStoreUri.toString(true)).toString(); + if (StringUtils.isNotBlank(appendExtension)) fileName += appendExtension; + String escortFileName = targetStoreName + "." + fileName; + + //ha mar letezik, nem toltjuk fel ujra (a vic fajlbol kezzel torlik valamikor...) + try { + if (EscortFiles.isMetadataExists(outputPath, escortFileName)) { + logger.debug(getMarker(), "Status file already exists {}", escortFileName); + return true; + } + } catch (IOException e1) { + logger.info(getMarker(), "Status file check error for {}", fileName); + return true; + } + + //nincs bent MC-ben + Media media = getManager().getMedia(record.getFileName()); if (media == null) { - logger.warn(getSessionMarker(), "File is not available in archive {}", fileName); - return; + logger.error(getSessionMarker(), "File is not available in archive {}", record.getFileName()); + return true; } + // if (existsOnNexio(record, targetStoreName, targetStoreUri)) { + // logger.info(getMarker(), "File {} already exists on {}", record.getFileName(), targetStoreName); + // return false; + // } + DownloadableMedia downloadable = DownloadableMedia.create(media.getTitle(), fileName, media.getModified(), media.getCreated(), media.getLength(), 0L, sourceStoreUri.getId(), targetStoreUri.getId(), media.getId()); - downloadable.put("priority", 100); - String escortFileName = targetStoreName + "." + downloadable.getString("fileName"); - String outputPath = null; + + //alapbol magasabb (1-99) + downloadable.put("priority", 50); + downloadable.put("isNexio", true); + try { - outputPath = Paths.get(escortStoreUri.toString(true)).toString(); - if (EscortFiles.createMetadataIfNotExists(outputPath, escortFileName, downloadable.toPrettyString(""))) - logger.info(getSessionMarker(), "Status file created {}", escortFileName); - else - logger.info(getSessionMarker(), "Status file already exists {}", escortFileName); + EscortFiles.createMetadata(outputPath, escortFileName, downloadable.toPrettyString("")); + logger.info(getMarker(), "Status file created {}", escortFileName); } catch (Exception e) { - logger.error("Can't create escort file {}", Paths.get(outputPath.toString(), escortFileName)); + logger.error("Can't create escort file {}. System message is: {}", escortFileName, e.getMessage()); } + return true; + } + + private List processVICFile(List records, Path vicFilePath) { + VICFileParser parser = null; + try { + parser = new VICFileParser(vicFilePath, lineFormat, validTypeCodes); + + List currentRecords = parser.getRecords(); + if (records == null) + records = currentRecords; + else { + if (currentRecords != null) + records.addAll(currentRecords); + } + } catch (Exception e) { + logger.error(e.getMessage()); + } + return records; } } diff --git a/server/user.jobengine.executors/src/user/jobengine/server/steps/ImportMORPHEUSMissingMaterialsStep.java b/server/user.jobengine.executors/src/user/jobengine/server/steps/ImportMORPHEUSMissingMaterialsStep.java index 4ca332cf..e5170194 100644 --- a/server/user.jobengine.executors/src/user/jobengine/server/steps/ImportMORPHEUSMissingMaterialsStep.java +++ b/server/user.jobengine.executors/src/user/jobengine/server/steps/ImportMORPHEUSMissingMaterialsStep.java @@ -76,7 +76,7 @@ public class ImportMORPHEUSMissingMaterialsStep extends JobStep { processPathItems(filePaths); } catch (Exception e) { logger.catching(e); - logger.error(jobRuntime.getMarker(), "Hiba a végrehajtás során. A rendszer üzenete: {}", e.getMessage()); + logger.error(jobRuntime.getSessionMarker(), "Hiba a végrehajtás során. A rendszer üzenete: {}", e.getMessage()); } return null; } @@ -103,7 +103,7 @@ public class ImportMORPHEUSMissingMaterialsStep extends JobStep { DBCollection collection = db.getCollection(MorpheusStrings.COLLECTION_NAME); BasicDBObject existingObject = (BasicDBObject) collection.findOne(query); if (existingObject != null) { - logger.warn(jobRuntime.getMarker(), "Az '{}' anyag már feldolgozásra került az {} időpontban.", materialID, + logger.warn(jobRuntime.getSessionMarker(), "Az '{}' anyag már feldolgozásra került az {} időpontban.", materialID, existingObject.getDate(MorpheusStrings.IMPORTED)); return; } @@ -122,18 +122,18 @@ public class ImportMORPHEUSMissingMaterialsStep extends JobStep { Path targetFilePath = Paths.get(targetPath, fileName); boolean exists = Files.exists(targetFilePath); if (exists && targetFilePath.toFile().length() > 0) { - logger.warn(jobRuntime.getMarker(), "Az '{}' anyag már be van töltve.", materialID); + logger.warn(jobRuntime.getSessionMarker(), "Az '{}' anyag már be van töltve.", materialID); dbObject.put(MorpheusStrings.STATUS, MorpheusStrings.STATUS_SKIPPED); } else { List medias = dao.getByHouseId(fileName); if (medias == null || medias.size() == 0) { - logger.warn(jobRuntime.getMarker(), "Az '{}' anyag nem található az archívumban.", materialID); + logger.warn(jobRuntime.getSessionMarker(), "Az '{}' anyag nem található az archívumban.", materialID); dbObject.put(MorpheusStrings.STATUS, MorpheusStrings.STATUS_UNAVAILABLE); } else if (medias.size() > 1) { - logger.warn(jobRuntime.getMarker(), "Az '{}' anyagból egynél több található az archívumban.", materialID); + logger.warn(jobRuntime.getSessionMarker(), "Az '{}' anyagból egynél több található az archívumban.", materialID); dbObject.put(MorpheusStrings.STATUS, MorpheusStrings.STATUS_MULTIPLE); } else { - logger.info(jobRuntime.getMarker(), "Az '{}' anyag megtalálható az archívumban.", materialID); + logger.info(jobRuntime.getSessionMarker(), "Az '{}' anyag megtalálható az archívumban.", materialID); dbObject.put(MorpheusStrings.STATUS, MorpheusStrings.STATUS_RESTORABLE); MediaFile mf = (MediaFile) medias.get(0); @@ -177,7 +177,7 @@ public class ImportMORPHEUSMissingMaterialsStep extends JobStep { moveProcessedCSV(csvFilePath); } catch (Exception e) { logger.catching(e); - logger.error(jobRuntime.getMarker(), "A {} MORPHEUS állomány mozgatásakor hiba történt. A rendszer hibaüzenete: {}.", csvFile.getName(), + logger.error(jobRuntime.getSessionMarker(), "A {} MORPHEUS állomány mozgatásakor hiba történt. A rendszer hibaüzenete: {}.", csvFile.getName(), e.getMessage()); } } @@ -202,39 +202,39 @@ public class ImportMORPHEUSMissingMaterialsStep extends JobStep { private void setAndCheck(String sourcePath, String processedFolder, String targetPath, IJobEngine jobEngine) { if (jobEngine == null) { - logger.error(jobRuntime.getMarker(), "Az folyamatkezelő réteg nem elérhető."); + logger.error(jobRuntime.getSessionMarker(), "Az folyamatkezelő réteg nem elérhető."); throw new NullPointerException("Internal error, missing JobEngine reference."); } IItemManager manager = jobEngine.getItemManager(); if (manager == null) { - logger.error(jobRuntime.getMarker(), "Az adatbáziskezelő réteg nem elérhető."); + logger.error(jobRuntime.getSessionMarker(), "Az adatbáziskezelő réteg nem elérhető."); throw new NullPointerException("Internal error, missing ItemManager reference."); } dao = (MediaFileDAO) manager.getBaseDAO(MediaFile.class); if (dao == null) { - logger.error(jobRuntime.getMarker(), "Az adatbáziskezelő réteg MediaFile kezelöje nem elérhető."); + logger.error(jobRuntime.getSessionMarker(), "Az adatbáziskezelő réteg MediaFile kezelöje nem elérhető."); throw new NullPointerException("Internal error, missing MediaFile DAO reference."); } if (sourcePath == null) { - logger.error(jobRuntime.getMarker(), "A folyamat 'sourcePath' bemeneti paramétere üres."); + logger.error(jobRuntime.getSessionMarker(), "A folyamat 'sourcePath' bemeneti paramétere üres."); throw new NullPointerException("System is not configured properly, 'sourcePath' input parameter missing."); } if (processedFolder == null) { - logger.error(jobRuntime.getMarker(), "A folyamat 'processedFolder' bemeneti paramétere üres."); + logger.error(jobRuntime.getSessionMarker(), "A folyamat 'processedFolder' bemeneti paramétere üres."); throw new NullPointerException("System is not configured properly, 'processedFolder' input parameter missing."); } this.processedFolder = processedFolder; if (targetPath == null) { - logger.error(jobRuntime.getMarker(), "A folyamat 'targetPath' bemeneti paramétere üres."); + logger.error(jobRuntime.getSessionMarker(), "A folyamat 'targetPath' bemeneti paramétere üres."); throw new NullPointerException("System is not configured properly, 'targetPath' input parameter missing."); } this.targetPath = targetPath; db = NoSQLUtils.getNoSQLDB(); if (db == null) { - logger.error(jobRuntime.getMarker(), "Sikertelen kapcsolódás a NoSQL adatbázishoz."); + logger.error(jobRuntime.getSessionMarker(), "Sikertelen kapcsolódás a NoSQL adatbázishoz."); throw new NullPointerException("Can not connect to NoSQL database."); } diff --git a/server/user.jobengine.executors/src/user/jobengine/server/steps/ImportStatisticsStep.java b/server/user.jobengine.executors/src/user/jobengine/server/steps/ImportStatisticsStep.java index 0ac75423..beb9ded5 100644 --- a/server/user.jobengine.executors/src/user/jobengine/server/steps/ImportStatisticsStep.java +++ b/server/user.jobengine.executors/src/user/jobengine/server/steps/ImportStatisticsStep.java @@ -45,7 +45,7 @@ public class ImportStatisticsStep extends JobStep { octopusAPI = new OctopusAPI(); } catch (Exception e) { logger.catching(e); - logger.error(jobRuntime.getMarker(), "Az Octopus adatbázis nem érhető el. A rendszer üzenete: {}", e.getMessage()); + logger.error(jobRuntime.getSessionMarker(), "Az Octopus adatbázis nem érhető el. A rendszer üzenete: {}", e.getMessage()); throw e; } } @@ -81,7 +81,7 @@ public class ImportStatisticsStep extends JobStep { @StepEntry public Object[] execute(int daysBeforeNow, IJobEngine jobEngine, IJobRuntime jobRuntime) throws Exception { - marker = jobRuntime.getMarker(); + marker = jobRuntime.getSessionMarker(); manager = jobEngine.getItemManager(); db = NoSQLUtils.getNoSQLDB(); clipCollection = db.getCollection(NEXIOCLIPS); @@ -104,10 +104,10 @@ public class ImportStatisticsStep extends JobStep { overall += folders.size(); Map stories = processRundowns(rundowns); - logger.info(jobRuntime.getMarker(), "Adástükörben megtalálható anyagok száma {}", stories.size()); + logger.info(jobRuntime.getSessionMarker(), "Adástükörben megtalálható anyagok száma {}", stories.size()); Map folderStories = processFolders(folders, scheduledDate); stories.putAll(folderStories); - logger.info(jobRuntime.getMarker(), "Gyűjtőkben megtalálható anyagok száma {}", stories.size()); + logger.info(jobRuntime.getSessionMarker(), "Gyűjtőkben megtalálható anyagok száma {}", stories.size()); storeStories(scheduledDate, stories); return null; diff --git a/server/user.jobengine.executors/src/user/jobengine/server/steps/MXFCutterStep.java b/server/user.jobengine.executors/src/user/jobengine/server/steps/MXFCutterStep.java index b435b6a5..58a5944d 100644 --- a/server/user.jobengine.executors/src/user/jobengine/server/steps/MXFCutterStep.java +++ b/server/user.jobengine.executors/src/user/jobengine/server/steps/MXFCutterStep.java @@ -53,7 +53,7 @@ public class MXFCutterStep extends JobStep { this.nexioUserName = nexioUserName; this.nexioPassword = nexioPassword; nexioHost = System.getProperty("nexio.host"); - marker = jobRuntime.getMarker(); + marker = jobRuntime.getSessionMarker(); if (useNexioTarget && archivedMedia.getTcIn() != null && archivedMedia.getTcOut() != null) { setAndCheck(archivedMedia, houseId, targetPath, useNexioTarget, jobEngine); diff --git a/server/user.jobengine.executors/src/user/jobengine/server/steps/MetadataPersisterStep.java b/server/user.jobengine.executors/src/user/jobengine/server/steps/MetadataPersisterStep.java index cd216217..c8cb85f2 100644 --- a/server/user.jobengine.executors/src/user/jobengine/server/steps/MetadataPersisterStep.java +++ b/server/user.jobengine.executors/src/user/jobengine/server/steps/MetadataPersisterStep.java @@ -31,30 +31,47 @@ public class MetadataPersisterStep extends JobStep { DownloadableMedia downloadable = EscortFiles.decode(escortFilePath); String title = downloadable.getString("title"); String fileName = downloadable.getString("fileName"); + long mediaId = downloadable.containsKey("mediaId") ? downloadable.getLong("mediaId") : 0; String fileTypeName = fileName.substring(fileName.lastIndexOf(".") + 1).toUpperCase(); IItemManager manager = getManager(); - String targetStoreName = getManager().getStore(targetStoreUri.getStoreId()).getName(); - //ez elszallhat, mert nincs filetype vagy store - MediaFile mediaFile = manager.createMediaFile(fileName, fileTypeName, targetStoreName); - //ez elszallhat, mert nincs itemtype - Item item = manager.createItem(GENERIC, title, sourceStoreName, null); - //ez elszallhat, mert nincs itemtype - Media media = manager.createMedia(GENERIC, title, sourceStoreName, null); + //TODO ha van mediaId, akkor update!!!!!!!!! + + if (mediaId == 0) { + String targetStoreName = getManager().getStore(targetStoreUri.getStoreId()).getName(); + //ez elszallhat, mert nincs filetype vagy store + MediaFile mediaFile = manager.createMediaFile(fileName, fileTypeName, targetStoreName); + //ez elszallhat, mert nincs itemtype + Item item = manager.createItem(GENERIC, title, sourceStoreName, null); + //ez elszallhat, mert nincs itemtype + Media media = manager.createMedia(GENERIC, title, sourceStoreName, null); + + //TODO megoldani egy tranzakcioban + item.add(); + media.setItemId(item.getId()); + media.setLength(downloadable.getLong("frames")); + media.setArchived(Timestamp.from(Instant.now())); + Timestamp modified = downloadable.getTimestamp("modified"); + Timestamp created = downloadable.getTimestamp("created"); + media.setModified(Timestamp.from(Instant.now())); + media.setCreated(created); + media.add(); + mediaFile.setMediaId(media.getId()); + mediaFile.setLastModified(modified); + mediaFile.add(); + } else { + Media media = manager.getMedia(mediaId); + Timestamp modified = downloadable.getTimestamp("modified"); + Timestamp created = downloadable.getTimestamp("created"); + media.setModified(Timestamp.from(Instant.now())); + media.setCreated(created); + manager.modify(media); + MediaFile mediaFile = manager.getSystemMediaFile(media); + mediaFile.setLastModified(modified); + manager.modify(mediaFile); + } - //TODO megoldani egy tranzakcioban - item.add(); - media.setItemId(item.getId()); - media.setLength(downloadable.getLong("frames")); - media.setArchived(Timestamp.from(Instant.now())); - Timestamp modified = downloadable.getTimestamp("modified"); - Timestamp created = downloadable.getTimestamp("created"); - media.setModified(modified); - media.setCreated(created); - media.add(); - mediaFile.setMediaId(media.getId()); - mediaFile.add(); } else logger.error(getMarker(), "Status file does not exists {}", escortFilePath); } catch (Exception e) { diff --git a/server/user.jobengine.executors/src/user/jobengine/server/steps/MetadataTransformStep.java b/server/user.jobengine.executors/src/user/jobengine/server/steps/MetadataTransformStep.java index 94190c63..4bb7992f 100644 --- a/server/user.jobengine.executors/src/user/jobengine/server/steps/MetadataTransformStep.java +++ b/server/user.jobengine.executors/src/user/jobengine/server/steps/MetadataTransformStep.java @@ -83,7 +83,7 @@ public class MetadataTransformStep extends JobStep { @StepEntry public Object[] execute(ArchiveItem archiveItem, IJobEngine jobEngine, IJobRuntime jobRuntime) throws Exception { - marker = jobRuntime.getMarker(); + marker = jobRuntime.getSessionMarker(); Media mediaCubeMedia = null; itemManager = (ItemManager) jobEngine.getItemManager(); if (itemManager == null) diff --git a/server/user.jobengine.executors/src/user/jobengine/server/steps/NEXIOArchiveCheckerStep.java b/server/user.jobengine.executors/src/user/jobengine/server/steps/NEXIOArchiveCheckerStep.java index f3b88b6a..895be243 100644 --- a/server/user.jobengine.executors/src/user/jobengine/server/steps/NEXIOArchiveCheckerStep.java +++ b/server/user.jobengine.executors/src/user/jobengine/server/steps/NEXIOArchiveCheckerStep.java @@ -3,6 +3,7 @@ package user.jobengine.server.steps; import java.io.IOException; import java.nio.file.Paths; import java.sql.Timestamp; +import java.text.SimpleDateFormat; import java.time.Duration; import java.time.Instant; import java.util.Iterator; @@ -19,16 +20,62 @@ import user.commons.nexio.api.Mediabase; import user.commons.nexio.server.protocol.ProtocolException; import user.commons.remotestore.RemoteStoreProtocol; import user.jobengine.db.Media; +import user.jobengine.db.MediaFile; public class NEXIOArchiveCheckerStep extends JobStep { private static final String DOT_MXF = ".mxf"; private static final Logger logger = LogManager.getLogger(); + private SimpleDateFormat df = new SimpleDateFormat("yyyyMMdd-HHmmss"); + + private boolean check(Timestamp modified, Timestamp created, int duration, Media media, int videoFormat, int lastModifiedHours) { + String title = media.getTitle(); + + if (!(videoFormat == 3 || videoFormat == 19)) { + logger.info(getMarker(), "{} unsupported video format", videoFormat); + return false; + } + + long modifiedHours = Duration.between(modified.toInstant(), Instant.now()).toHours(); + if (modifiedHours < lastModifiedHours) { + logger.info(getMarker(), "{} modification time is too cloose for now", title); + return false; + } + + // long count = executeQuery( + // "select count(*) from job where related = ? and NAME = 'Visszatöltés' and STATUS='FINISHED' and HOURS_BETWEEN(?, finished) = 0", title, + // created); + // + // if (count > 0) { + // logger.info(getMarker(), "{} has record date and TSM restore too close", title); + // return true; + // } + + MediaFile mediaFile = getManager().getSystemMediaFile(media); + boolean isModified = true; + if (mediaFile != null) + isModified = df.format(mediaFile.getLastModified()).compareTo(df.format(modified)) < 0; + + if (isModified) { + logger.info(getMarker(), "{} has different last modification time {} < {}", title, media.getModified(), modified); + return true; + } + + boolean isLengthChanged = media.getLength() > 0 && Math.abs(media.getLength() - duration) > 3; + if (isLengthChanged) { + logger.info(getMarker(), "{} has different length {} != {}", title, media.getLength(), duration); + return true; + } + + return false; + } @StepEntry - public Object[] execute(String sourceStoreName, String escortStoreName, int lastModifiedHours, int limit) throws Exception { + public Object[] execute(String sourceStoreName, String escortStoreName, int lastModifiedHours) throws Exception { Controller controller = null; try { + getJobRuntime().setRelated(sourceStoreName + " archive"); + StoreUri nexioStoreUri = getManager().getStoreUri(sourceStoreName, RemoteStoreProtocol.NEXIO); StoreUri sourceStoreUri = getManager().getStoreUri(sourceStoreName, RemoteStoreProtocol.FTP); StoreUri targetStoreUri = getManager().getStoreUri("TSM", RemoteStoreProtocol.TSM); @@ -45,9 +92,10 @@ public class NEXIOArchiveCheckerStep extends JobStep { Iterator clips = mediabase.getClips(); logger.info(getMarker(), "Processing clips"); - processClips(sourceStoreName, sourceStoreUri, targetStoreUri, outputPath, limit, clips, lastModifiedHours); + + processClips(sourceStoreName, sourceStoreUri, targetStoreUri, outputPath, clips, lastModifiedHours); } catch (Exception e) { - logger.error(getSessionMarker(), e.getMessage()); + logger.error(getSessionMarker(), e.getMessage() == null ? "Unknown error" : e.getMessage()); throw e; } finally { setProgress(100); @@ -58,20 +106,50 @@ public class NEXIOArchiveCheckerStep extends JobStep { return null; } - private boolean check(Timestamp modified, Timestamp created, int duration, Media media) { - boolean create = media == null; - if (!create) { - create |= !media.getModified().equals(modified); - create |= !media.getCreated().equals(created); - //a regi moziknal nincs hossz mentve, es csak arra lovunk aminel 2 framnel nagyobb a kulonbseg - create |= media.getLength() > 0 && Math.abs(media.getLength() - duration) > 2; - } - return create; - } - - private void processClips(String storeName, StoreUri sourceStoreUri, StoreUri targetStoreUri, String outputPath, int limit, Iterator clips, + // public long executeQuery(String query, String title, Timestamp created) { + // long count = 0; + // + // DefaultContext context = getManager().getDbContext(); + // Connection connection = context.getConnection(); + // + // ResultSet rs = null; + // PreparedStatement st = null; + // try { + // st = connection.prepareStatement(query); + // st.setString(1, title); + // st.setTimestamp(2, created); + // rs = st.executeQuery(); + // if (rs.next()) { + // count = rs.getLong(1); + // } + // + // connection.commit(); + // } catch (Exception e) { + // logger.catching(e); + // try { + // connection.rollback(); + // } catch (Exception e1) { + // } + // } finally { + // try { + // if (rs != null) + // rs.close(); + // } catch (Exception e1) { + // } + // try { + // if (st != null) + // st.close(); + // } catch (Exception e1) { + // } + // + // getManager().putDbContext(context); + // } + // return count; + // } + + private void processClips(String storeName, StoreUri sourceStoreUri, StoreUri targetStoreUri, String outputPath, Iterator clips, int lastModifiedHours) throws ClipNotFoundException, IOException, ProtocolException { - int currentCount = 0; + int processed = 0; while (clips.hasNext()) { if (getJobRuntime().isWaitingCancel()) { @@ -80,51 +158,52 @@ public class NEXIOArchiveCheckerStep extends JobStep { } Clip clip = clips.next(); - String title = clip.getXid().get(); - Instant clipModified = clip.getModifiedTimestamp().toInstant(); - Timestamp modified = Timestamp.from(clipModified); - Timestamp created = Timestamp.from(clip.getRecordDateTimestamp().toInstant()); - int frames = clip.getDuration(); - - Media media = getManager().getMedia(title); - boolean createEscort = check(modified, created, frames, media); + try { + String title = null; + Timestamp created = null; + Timestamp modified = null; + int frames = 0; + int videoFormat = 0; + title = clip.getXid().get(); + frames = clip.getDuration(); + created = Timestamp.from(clip.getRecordDateTimestamp().toInstant()); + modified = Timestamp.from(clip.getModifiedTimestamp().toInstant()); + videoFormat = clip.getVideoFormat(); + + Media media = getManager().getMedia(title); + logger.debug(getMarker(), "Checking {}", title); + boolean createEscort = false; - if (createEscort) { long mediaId = 0; - - //lastModifiedHours oran belul modositott anyagokat beken hagyjuk - if (media != null) { - //negativ ha a masodik az elso elott van - long modifiedHours = Duration.between(clipModified, Instant.now()).toHours(); - if (modifiedHours > lastModifiedHours) { - logger.info(getMarker(), "Modification time is too cloose for {}", title); - continue; - } + if (media == null) { + createEscort = true; + logger.info(getMarker(), "{} not archived yet", title); + } else { mediaId = media.getId(); + createEscort = check(modified, created, frames, media, videoFormat, lastModifiedHours); } - String fileName = title + DOT_MXF; - DownloadableMedia downloadable = DownloadableMedia.create(title, fileName, modified, created, frames, 0, sourceStoreUri.getId(), - targetStoreUri.getId(), mediaId); - String escortFileName = storeName + "." + downloadable.getString("fileName"); - if (EscortFiles.createMetadataIfNotExists(outputPath, escortFileName, downloadable.toPrettyString(""))) { - logger.info(getMarker(), "Archive status file created for {}", fileName); - currentCount++; - } - } + if (createEscort && !title.contains("*")) { + String fileName = title + DOT_MXF; + DownloadableMedia downloadable = DownloadableMedia.create(title, fileName, modified, created, frames, 0, sourceStoreUri.getId(), + targetStoreUri.getId(), mediaId); + String escortFileName = storeName + "." + downloadable.getString("fileName"); + if (EscortFiles.createMetadataIfNotExists(outputPath, escortFileName, downloadable.toPrettyString(""))) { + logger.info(getMarker(), "Archive status file created for {}", fileName); + } else { + logger.info(getMarker(), "Archive status file already exists for {}", fileName); + } - if (limit > 0) { - int p = (currentCount / limit) * 100; - setProgress(p); - } + } - if (currentCount == limit) { - logger.info(getMarker(), "Limit reached: {}", limit); - break; + processed++; + if (processed % 100 == 0) + logger.info(getMarker(), "Processed {} records", processed); + } catch (Exception e) { + logger.error(getSessionMarker(), e.getMessage()); + continue; } - - // logger.info(getSessionMarker(), "{} {}", limit, currentCount); } } diff --git a/server/user.jobengine.executors/src/user/jobengine/server/steps/NEXIOMetadataPersisterStep.java b/server/user.jobengine.executors/src/user/jobengine/server/steps/NEXIOMetadataPersisterStep.java new file mode 100644 index 00000000..bf202481 --- /dev/null +++ b/server/user.jobengine.executors/src/user/jobengine/server/steps/NEXIOMetadataPersisterStep.java @@ -0,0 +1,108 @@ +package user.jobengine.server.steps; + +import java.nio.file.Path; +import java.nio.file.Paths; +import java.sql.Timestamp; +import java.text.SimpleDateFormat; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.w3c.dom.Document; +import org.w3c.dom.Node; +import org.w3c.dom.NodeList; + +import user.commons.DownloadableMedia; +import user.commons.RemoteFile; +import user.commons.StoreUri; +import user.jobengine.db.Media; +import user.jobengine.db.MediaFile; + +public class NEXIOMetadataPersisterStep extends JobStep { + private static final Logger logger = LogManager.getLogger(); + private static final String DOT_XML = ".xml"; + + @StepEntry + public Object[] execute(StoreUri targetStoreUri, String escortFile) throws Exception { + try { + Path escortFilePath = Paths.get(escortFile); + if (!escortFilePath.toFile().exists()) { + logger.error(getMarker(), "Status file does not exists {}", escortFilePath); + return null; + } + + DownloadableMedia downloadable = EscortFiles.decode(escortFilePath); + if (!downloadable.containsKey("isNexio") || !downloadable.getBoolean("isNexio")) + return null; + + if (!downloadable.containsKey("mediaId")) + throw new NullPointerException("No media ID found for " + downloadable.getString("title")); + + Media media = getManager().getMedia(downloadable.getLong("mediaId")); + MediaFile mediaFile = getManager().getSystemMediaFile(media); + if (mediaFile == null) + throw new NullPointerException("No TSM media file found for " + downloadable.getString("title")); + + RemoteFile remoteFile = targetStoreUri.getFileWithContent(downloadable.getString("title") + ".xml"); + + //TODO mediafile lastmodified = feltoltott file nexio xml lastmodified + Timestamp lastModified = getLastModified(remoteFile); + mediaFile.setLastModified(lastModified); + getManager().modify(mediaFile); + + // if (!RemoteStoreProtocol.FTP.equals(targetStoreUri.getProtocol())) + // return null; + + // + // String title = downloadable.getString("title"); + // String fileName = downloadable.getString("fileName"); + // Date created = downloadable.getDate("created"); + // Date modified = downloadable.getDate("modified"); + // byte[] metadata = EscortFiles.createNEXIODatesMeta(title, created, modified); + // uploadMeta(targetStoreUri, fileName, metadata); + } catch (Exception e) { + logger.error(getSessionMarker(), e.getMessage()); + throw e; + } finally { + if (targetStoreUri != null) + targetStoreUri.cleanUp(); + } + return null; + } + + private Timestamp getLastModified(RemoteFile remoteFile) throws Exception { + Timestamp result = null; + Document document = EscortFiles.createNEXIOMeta(remoteFile.getContent()); + Node firstChild = document.getFirstChild(); + NodeList list = firstChild.getChildNodes(); + for (int i = 0; i < list.getLength(); i++) { + Node item = list.item(i); + if (item.getNodeName().equals("ModifiedTimeStamp")) { + SimpleDateFormat df = new SimpleDateFormat("MM-dd-yyyy (HH:mm:ss)"); + result = Timestamp.from(df.parse(item.getTextContent()).toInstant()); + break; + } + } + return result; + } + + // private void uploadMeta(StoreUri storeUri, String fileName, byte[] metadata) throws Exception { + // logger.info(getMarker(), "Uploading metadata for {}", fileName); + // OutputStream outStream = null; + // try { + // FTPClient sourceFtp = ((FtpDirectoryLister) storeUri.getLister()).connect(); + // outStream = sourceFtp.storeFileStream(fileName + DOT_XML); + // if (outStream == null) + // throw new NullPointerException("Can not create: " + fileName + DOT_XML + " on FTP. Reply is:" + sourceFtp.getReplyString()); + // outStream.write(metadata); + // outStream.flush(); + // } catch (Exception e) { + // logger.catching(e); + // throw e; + // } finally { + // if (outStream != null) + // outStream.close(); + // storeUri.cleanUp(); + // } + // } + +} diff --git a/server/user.jobengine.executors/src/user/jobengine/server/steps/OutputPathAndNameSelectorStep.java b/server/user.jobengine.executors/src/user/jobengine/server/steps/OutputPathAndNameSelectorStep.java index 746a325a..7ac81628 100644 --- a/server/user.jobengine.executors/src/user/jobengine/server/steps/OutputPathAndNameSelectorStep.java +++ b/server/user.jobengine.executors/src/user/jobengine/server/steps/OutputPathAndNameSelectorStep.java @@ -67,7 +67,7 @@ public class OutputPathAndNameSelectorStep extends JobStep { public Object[] execute(String localRetrievePath, String materialOutputFolder, String promoOutputFolder, String advertisementOutputFolder, String octopusOutputFolder, String genericOutputFolder, String onlineOutputFolder, String houseId, String targetPathType, ArchivedMedia archivedMedia, IJobEngine jobEngine, IJobRuntime jobRuntime) throws IOException { - marker = jobRuntime.getMarker(); + marker = jobRuntime.getSessionMarker(); check(localRetrievePath, materialOutputFolder, promoOutputFolder, advertisementOutputFolder, octopusOutputFolder, genericOutputFolder, houseId, targetPathType); Object[] result = null; @@ -142,7 +142,7 @@ public class OutputPathAndNameSelectorStep extends JobStep { try { EscortFiles.ensureUNCFolder(Paths.get(targetPath)); } catch (Exception e) { - logger.error(jobRuntime.getMarker(), "A cél mappa '{}' nem létezik és nem hozható létre. A rendszer hibaüzenete: {}", targetPath, e.getMessage()); + logger.error(jobRuntime.getSessionMarker(), "A cél mappa '{}' nem létezik és nem hozható létre. A rendszer hibaüzenete: {}", targetPath, e.getMessage()); throw e; } return new Object[] { targetPath, targetNamePattern, false }; diff --git a/server/user.jobengine.executors/src/user/jobengine/server/steps/PBQuery.java b/server/user.jobengine.executors/src/user/jobengine/server/steps/PBQuery.java index 9566b6bb..458b1375 100644 --- a/server/user.jobengine.executors/src/user/jobengine/server/steps/PBQuery.java +++ b/server/user.jobengine.executors/src/user/jobengine/server/steps/PBQuery.java @@ -10,16 +10,17 @@ import java.nio.file.Path; import java.nio.file.Paths; import java.text.SimpleDateFormat; import java.util.Calendar; +import java.util.Collection; import java.util.Date; -import java.util.LinkedHashSet; +import java.util.HashMap; import java.util.List; +import java.util.Map; import java.util.TreeMap; import javax.xml.parsers.DocumentBuilder; import javax.xml.parsers.DocumentBuilderFactory; import org.apache.commons.lang.StringUtils; -import org.apache.http.Header; import org.apache.http.HttpEntity; import org.apache.http.HttpResponse; import org.apache.http.auth.AuthScope; @@ -41,13 +42,27 @@ import org.xml.sax.InputSource; public class PBQuery { public class MMMedia { - String name; - String usage; + private final String name; + private final String usage; + + MMMedia(String name, String usage) { + this.name = name; + this.usage = usage; + + } + + public String getName() { + return name; + } + + public String getUsage() { + return usage; + } } private static final Logger logger = LogManager.getLogger(); - private static final int CONNECTION_TIMEOUT = 1000; - private static final int SOCKET_TIMEOUT = 2000; + private static final int CONNECTION_TIMEOUT = 5000; + private static final int SOCKET_TIMEOUT = 5000; public static final String GETMEDIAUSAGEBYUTRANGE = "/getMediaUsageByUTRange"; public static final String GETCUSTOMVIEW = "/getCustomView"; @@ -90,11 +105,17 @@ public class PBQuery { return result; } - public LinkedHashSet getPossibelMissingMaterialNames(List poolContent) throws Exception { - LinkedHashSet result = querySortableMedias(poolContent); - LinkedHashSet otherMedias = queryOtherMedias(poolContent); - result.addAll(otherMedias); - return result; + public Collection getPossibelMissingMaterialNames(List poolContent) throws Exception { + Map result = querySortableMedias(poolContent); + queryOtherMedias(poolContent, result); + + TreeMap sortedUsages = new TreeMap<>(); + for (MMMedia m : result.values()) { + //a TreeMap lehetove teszi: nincs duplikatum + a korabbi musorok elore kerulnek + sortedUsages.put(m.getUsage() + "." + m.getName(), m); + } + + return sortedUsages.values(); } @@ -109,6 +130,16 @@ public class PBQuery { this.rangeForwardHours = rangeForwardHours; } + private void putEarliest(Map result, String mediaName, String usageTime) { + if (result.containsKey(mediaName)) { + MMMedia m = result.get(mediaName); + //az aktualis a korabbi idopont + if (usageTime.compareTo(m.getUsage()) < 0) + result.put(mediaName, new MMMedia(mediaName, usageTime)); + } else + result.put(mediaName, new MMMedia(mediaName, usageTime)); + } + public String query(String endPoint, String action, String soap, String user, String pwd) throws Exception { HttpParams httpParameters = new BasicHttpParams(); @@ -130,13 +161,13 @@ public class PBQuery { httppost.setHeader("Content-Type", "text/xml; charset=utf-8"); HttpEntity entity = new StringEntity(soap.toString(), HTTP.UTF_8); httppost.setEntity(entity); - logger.info("Executing call"); + logger.info("Executing call {}", action); HttpResponse response = httpclient.execute(httppost);// calling server HttpEntity r_entity = response.getEntity(); //get response - Header[] headers = response.getAllHeaders(); - for (Header h : headers) { - logger.info("Reponse Header", h.getName() + ": " + h.getValue()); - } + // Header[] headers = response.getAllHeaders(); + // for (Header h : headers) { + // logger.info("Reponse Header", h.getName() + ": " + h.getValue()); + // } if (r_entity != null) { byte[] bytes = new byte[(int) r_entity.getContentLength()]; if (r_entity.isStreaming()) { @@ -146,7 +177,7 @@ public class PBQuery { result = new String(bytes); } } catch (Exception e) { - logger.error("Exception While Connecting", "" + e.getMessage()); + logger.error("Exception while connecting to endpoint {}. System message is: {}", e.getMessage()); throw e; } finally { if (is != null) { @@ -163,8 +194,7 @@ public class PBQuery { return result; } - public LinkedHashSet queryOtherMedias(List poolContent) throws Exception { - LinkedHashSet result = new LinkedHashSet<>(); + public void queryOtherMedias(List poolContent, Map result) throws Exception { String xml = null; String soap = null; Path template = Paths.get(SOAP_TEMPLATE_ROOT, GETCUSTOMVIEW); @@ -181,47 +211,39 @@ public class PBQuery { } } - if (StringUtils.isNotBlank(xml)) { - - Document document = toDocument(xml); - NodeList mediaNodes = document.getElementsByTagName("media"); + if (StringUtils.isBlank(xml)) { + logger.info("Query result is empty"); + return; + } - if (mediaNodes == null || mediaNodes.getLength() == 0) - return result; + Document document = toDocument(xml); + NodeList mediaNodes = document.getElementsByTagName("media"); - TreeMap usages = new TreeMap<>(); + if (mediaNodes == null || mediaNodes.getLength() == 0) + return; - for (int i = 0; i < mediaNodes.getLength(); i++) { - Node mediaNode = mediaNodes.item(i); - String mediaName = getAttribute(mediaNode, "mediaName"); - if (StringUtils.isBlank(mediaName)) { - logger.info("Missing mediaName"); - continue; - } + for (int i = 0; i < mediaNodes.getLength(); i++) { + Node mediaNode = mediaNodes.item(i); + String mediaName = getAttribute(mediaNode, "mediaName"); + if (StringUtils.isBlank(mediaName)) { + logger.info("Missing mediaName"); + continue; + } - if (poolContent.contains(mediaName)) - continue; + if (poolContent.contains(mediaName)) + continue; - String usageTime = getAttribute(mediaNode, "earliestUsageTime"); - logger.info("{}, {}", mediaName, usageTime); - //a TreeMap lehetove teszi: nincs duplikatum + a korabbi musorok elore kerulnek - usages.put(usageTime + "_" + mediaName, mediaName); + String usageTime = getAttribute(mediaNode, "earliestUsageTime"); + putEarliest(result, mediaName, usageTime); + } - } + logger.info("Document done, items {}", mediaNodes.getLength()); - for (String key : usages.keySet()) { - MMMedia media = new MMMedia(); - media.name = usages.get(key); - media.usage = key.split("_")[0]; - result.add(media); - } - logger.info("Document done, items {}", mediaNodes.getLength()); - } - return result; } - public LinkedHashSet querySortableMedias(List poolContent) throws Exception { - LinkedHashSet result = new LinkedHashSet<>(); + public Map querySortableMedias(List poolContent) throws Exception { + Map result = new HashMap<>(); + String xml = null; String soap = null; @@ -250,58 +272,54 @@ public class PBQuery { } } - if (StringUtils.isNotBlank(xml)) { - - Document document = toDocument(xml); - NodeList mediaNodes = document.getElementsByTagName("mediaUsage"); - - if (mediaNodes == null || mediaNodes.getLength() == 0) - return result; + if (StringUtils.isBlank(xml)) { + logger.info("Query result is empty"); + return result; + } - TreeMap usages = new TreeMap<>(); + Document document = toDocument(xml); + NodeList mediaNodes = document.getElementsByTagName("mediaUsage"); - for (int i = 0; i < mediaNodes.getLength(); i++) { - Node mediaNode = mediaNodes.item(i); - String mediaName = getAttribute(mediaNode, "mediaName"); - if (StringUtils.isBlank(mediaName)) { - logger.info("Missing mediaName"); - continue; - } + if (mediaNodes == null || mediaNodes.getLength() == 0) + return result; - if (poolContent.contains(mediaName)) - continue; + for (int i = 0; i < mediaNodes.getLength(); i++) { + Node mediaNode = mediaNodes.item(i); + String mediaName = getAttribute(mediaNode, "mediaName"); + String mediaType = getAttribute(mediaNode, "mediaType"); + if (StringUtils.isBlank(mediaName)) { + logger.info("Missing mediaName"); + continue; + } - Node usageChild = mediaNode.getFirstChild(); - if (usageChild == null) { - logger.info("Skipping {}, no media usage defined", mediaName); - continue; - } + if (!StringUtils.equals("Video", mediaType)) { + continue; + } - NodeList usageNodes = usageChild.getChildNodes(); - if (usageNodes == null || usageNodes.getLength() == 0) { - logger.info("Skipping {}, no media usage defined", mediaName); - continue; - } + if (poolContent.contains(mediaName)) + continue; - for (int j = 0; j < usageNodes.getLength(); j++) { - Node mediaUsageNode = usageNodes.item(j); - String usageTime = getAttribute(mediaUsageNode, "earliestUsageTime"); - logger.info("{}, {}", mediaName, usageTime); - //a TreeMap lehetove teszi: nincs duplikatum + a korabbi musorok elore kerulnek - usages.put(usageTime + "_" + mediaName, mediaName); - } + Node usageChild = mediaNode.getFirstChild(); + if (usageChild == null) { + logger.info("Skipping {}, no media usage defined", mediaName); + continue; + } + NodeList usageNodes = usageChild.getChildNodes(); + if (usageNodes == null || usageNodes.getLength() == 0) { + logger.info("Skipping {}, no media usage defined", mediaName); + continue; } - for (String key : usages.keySet()) { - MMMedia media = new MMMedia(); - media.name = usages.get(key); - media.usage = key.split("_")[0]; - result.add(media); + for (int j = 0; j < usageNodes.getLength(); j++) { + Node mediaUsageNode = usageNodes.item(j); + String usageTime = getAttribute(mediaUsageNode, "earliestUsageTime"); + logger.info("{}, {}", mediaName, usageTime); + putEarliest(result, mediaName, usageTime); } - logger.info("Document done, items {}", mediaNodes.getLength()); } + return result; } diff --git a/server/user.jobengine.executors/src/user/jobengine/server/steps/PeableBeachMissingMaterialCheckerStep.java b/server/user.jobengine.executors/src/user/jobengine/server/steps/PeableBeachMissingMaterialCheckerStep.java index f66aa8a2..1c64232f 100644 --- a/server/user.jobengine.executors/src/user/jobengine/server/steps/PeableBeachMissingMaterialCheckerStep.java +++ b/server/user.jobengine.executors/src/user/jobengine/server/steps/PeableBeachMissingMaterialCheckerStep.java @@ -1,7 +1,11 @@ package user.jobengine.server.steps; +import java.nio.file.DirectoryStream; +import java.nio.file.Files; +import java.nio.file.Path; import java.nio.file.Paths; import java.util.ArrayList; +import java.util.Collection; import java.util.LinkedHashSet; import java.util.List; @@ -9,56 +13,71 @@ import org.apache.commons.lang.StringUtils; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; +import com.ibm.nosql.json.api.BasicDBList; + import user.commons.DownloadableMedia; import user.commons.RemoteFile; import user.commons.StoreUri; import user.commons.remotestore.RemoteStoreProtocol; import user.jobengine.db.Media; +import user.jobengine.server.steps.PBQuery.MMMedia; public class PeableBeachMissingMaterialCheckerStep extends JobStep { private static final Logger logger = LogManager.getLogger(); private static final String appendExtension = System.getProperty("missingmaterial.appendextension", ".mxf"); @StepEntry - public Object[] execute(String escortStoreName, String targetStoreName, String targetProtocol, String primaryEndPoint, String primaryUserName, - String primaryPassword, String secondaryEndPoint, String secondaryUserName, String secondaryPassword, int rangeForwardHours) throws Exception { + public Object[] execute(String escortStoreName, BasicDBList lookupStoreNames, String targetStoreName, String targetProtocol, String primaryEndPoint, + String primaryUserName, String primaryPassword, String secondaryEndPoint, String secondaryUserName, String secondaryPassword, int rangeForwardHours) + throws Exception { StoreUri escortStoreUri = null; try { + getJobRuntime().setRelated(targetStoreName + " restore"); + escortStoreUri = getManager().getStoreUri(escortStoreName, RemoteStoreProtocol.LOCAL); StoreUri sourceStoreUri = getManager().getStoreUri("TSM", RemoteStoreProtocol.TSM); StoreUri targetStoreUri = getManager().getStoreUri(targetStoreName, Enum.valueOf(RemoteStoreProtocol.class, targetProtocol)); - List poolContent = new ArrayList<>(); + LinkedHashSet poolContent = new LinkedHashSet<>(); + + int progress = 0; + int count = lookupStoreNames.size(); + int processed = 0; + + for (Object lookup : lookupStoreNames) { + if (getJobRuntime().isWaitingCancel()) { + cancel(); + return null; + } + String lookupStoreName = String.valueOf(lookup); + StoreUri lookupStoreUri = getManager().getStoreUri(lookupStoreName, RemoteStoreProtocol.LOCAL); + List lookupContent = getPoolContent(lookupStoreUri); + poolContent.addAll(lookupContent); + processed++; + progress = processed * 50 / count; + setProgress(progress); + } PBQuery query = new PBQuery(); query.init(primaryEndPoint, primaryUserName, primaryPassword, secondaryEndPoint, secondaryUserName, secondaryPassword, rangeForwardHours); - LinkedHashSet titles = query.getPossibelMissingMaterialNames(poolContent); - logger.info(getMarker(), "API returned {} items", titles == null ? 0 : titles.size()); - - if (titles != null && titles.size() > 0) { - - List remoteFiles = targetStoreUri.getRemoteFiles(); - if (remoteFiles != null) - remoteFiles.forEach(i -> { - String name = i.getName(); - poolContent.add(name.substring(0, name.lastIndexOf("."))); - }); - - logger.info(getMarker(), "Pool contains {} items", poolContent == null ? 0 : poolContent.size()); - - if (!getJobRuntime().isWaitingCancel()) { - int count = titles.size(); - int processed = 0; - for (String title : titles) { - processRecord(title, sourceStoreUri, targetStoreName, targetStoreUri, escortStoreUri); - processed++; - int progress = processed * 100 / count; - setProgress(progress); + Collection medias = query.getPossibelMissingMaterialNames(new ArrayList<>(poolContent)); + logger.info(getMarker(), "API returned {} items", medias == null ? 0 : medias.size()); + + if (medias != null && medias.size() > 0) { + count = medias.size(); + for (MMMedia media : medias) { + if (getJobRuntime().isWaitingCancel()) { + cancel(); + break; } + processRecord(media, sourceStoreUri, targetStoreName, targetStoreUri, escortStoreUri); + processed++; + progress = 50 + (processed * 50 / count); + setProgress(progress); } } - + setProgress(100); } catch (Exception e) { logger.error(getSessionMarker(), e.getMessage()); throw e; @@ -70,39 +89,74 @@ public class PeableBeachMissingMaterialCheckerStep extends JobStep { return null; } - private void processRecord(String title, StoreUri sourceStoreUri, String targetStoreName, StoreUri targetStoreUri, StoreUri escortStoreUri) { + private List getPoolContent(StoreUri targetStoreUri) throws Exception { + List poolContent = new ArrayList<>(); + List remoteFiles = targetStoreUri.getRemoteFiles(); + if (remoteFiles != null) + remoteFiles.forEach(i -> { + String name = i.getName(); + poolContent.add(name.substring(0, name.lastIndexOf("."))); + }); + + logger.info(getMarker(), "Pool contains {} items", poolContent == null ? 0 : poolContent.size()); + return poolContent; + } + + //csak visszaterunk + private String normalizeName(String mediaName) { + String result = mediaName; + if (StringUtils.isNotBlank(appendExtension)) + result += appendExtension; + // if (StringUtils.isNotBlank(mediaName)) { + // result = mediaName.trim().toLowerCase(); + // } + return result; + } + + private void processRecord(MMMedia mmedia, StoreUri sourceStoreUri, String targetStoreName, StoreUri targetStoreUri, StoreUri escortStoreUri) { //TODO kisbetu/nagybetu problema kezelese - Media media = getManager().getMedia(title); + Media media = getManager().getMedia(mmedia.getName()); - String fileName = normalizeName(title); + String fileName = normalizeName(mmedia.getName()); if (media == null) { - logger.error(getMarker(), "File {} not archived yet", fileName); + logger.error(getMarker(), "File {} not archived yet", mmedia.getName()); return; } - DownloadableMedia downloadable = DownloadableMedia.create(title, fileName, media.getModified(), media.getCreated(), media.getLength(), 0L, + DownloadableMedia downloadable = DownloadableMedia.create(mmedia.getName(), fileName, media.getModified(), media.getCreated(), media.getLength(), 0L, sourceStoreUri.getId(), targetStoreUri.getId(), media.getId()); - String escortFileName = targetStoreName + "." + downloadable.getString("fileName"); - String outputPath = null; + downloadable.put("priority", 50); + String usage = mmedia.getUsage(); + usage = usage.replace("-", "").replace(":", ""); + String escortFileName = usage + "." + targetStoreName + "." + fileName; + Path outputPath = null; try { - outputPath = Paths.get(escortStoreUri.toString(true)).toString(); - EscortFiles.createMetadataIfNotExists(outputPath, escortFileName, downloadable.toPrettyString("")); + outputPath = Paths.get(escortStoreUri.toString(true)); + boolean exsists = statusFileExists(outputPath, fileName); + if (exsists) + return; + EscortFiles.createMetadata(outputPath.toString(), escortFileName, downloadable.toPrettyString("")); + logger.info(getMarker(), "Status file created {}", escortFileName); } catch (Exception e) { - logger.error(getSessionMarker(), "Can't create escort file {}", Paths.get(outputPath.toString(), escortFileName)); + logger.error(getSessionMarker(), "Can't create status file {}", Paths.get(outputPath.toString(), escortFileName)); } } - //csak visszaterunk - private String normalizeName(String mediaName) { - String result = mediaName; - if (StringUtils.isNotBlank(appendExtension)) - result += appendExtension; - // if (StringUtils.isNotBlank(mediaName)) { - // result = mediaName.trim().toLowerCase(); - // } - return result; + private boolean statusFileExists(Path outputPath, String fileName) { + boolean exsists = false; + Path statusPath = Paths.get(outputPath.toString(), EscortFiles.STATUSFOLDER); + try (DirectoryStream p = Files.newDirectoryStream(statusPath, "*" + fileName + EscortFiles.DOT_JSON)) { + if (p.iterator().hasNext()) { + logger.info(getMarker(), "Status file for {} already exists", fileName); + exsists = true; + } + + } catch (Exception e1) { + logger.error(getSessionMarker(), e1.getMessage()); + } + return exsists; } } diff --git a/server/user.jobengine.executors/src/user/jobengine/server/steps/ProjectCleanupMountedLocationStep.java b/server/user.jobengine.executors/src/user/jobengine/server/steps/ProjectCleanupMountedLocationStep.java index 4b088798..096ef6b7 100644 --- a/server/user.jobengine.executors/src/user/jobengine/server/steps/ProjectCleanupMountedLocationStep.java +++ b/server/user.jobengine.executors/src/user/jobengine/server/steps/ProjectCleanupMountedLocationStep.java @@ -54,7 +54,7 @@ public class ProjectCleanupMountedLocationStep extends JobStep implements FileVi @StepEntry public Object[] execute(String sourceFolder, IJobEngine jobEngine, IJobRuntime jobRuntime) throws Exception { - marker = jobRuntime.getMarker(); + marker = jobRuntime.getSessionMarker(); sourcePath = Paths.get(sourceFolder); DirectoryStream directoryStream = null; if (StringUtils.isBlank(sourcePath.toString())) { diff --git a/server/user.jobengine.executors/src/user/jobengine/server/steps/RecordingsArchiveItemBuilderStep.java b/server/user.jobengine.executors/src/user/jobengine/server/steps/RecordingsArchiveItemBuilderStep.java index b476a20d..39b714e8 100644 --- a/server/user.jobengine.executors/src/user/jobengine/server/steps/RecordingsArchiveItemBuilderStep.java +++ b/server/user.jobengine.executors/src/user/jobengine/server/steps/RecordingsArchiveItemBuilderStep.java @@ -84,7 +84,7 @@ public class RecordingsArchiveItemBuilderStep extends JobStep { result.setCatchedFile(catchedFilePath.toString()); } catch (Exception e) { logger.catching(e); - logger.error(getJobRuntime().getMarker(), "A metaadat nem elérhető. A rendszer üzenete: {}", e.getMessage()); + logger.error(getJobRuntime().getSessionMarker(), "A metaadat nem elérhető. A rendszer üzenete: {}", e.getMessage()); return null; } @@ -107,7 +107,7 @@ public class RecordingsArchiveItemBuilderStep extends JobStep { final ArchiveItem[] archiveItems = { null }; DB db = NoSQLUtils.getNoSQLDB(); existingRecordings = db.getCollection("tmp_existing_recordings"); - marker = getJobRuntime().getMarker(); + marker = getJobRuntime().getSessionMarker(); try { Files.walkFileTree(Paths.get(sourcePath), new SimpleFileVisitor() { diff --git a/server/user.jobengine.executors/src/user/jobengine/server/steps/RegisterUserRestoreStep.java b/server/user.jobengine.executors/src/user/jobengine/server/steps/RegisterUserRestoreStep.java index c98f2ce4..ba23e3f6 100644 --- a/server/user.jobengine.executors/src/user/jobengine/server/steps/RegisterUserRestoreStep.java +++ b/server/user.jobengine.executors/src/user/jobengine/server/steps/RegisterUserRestoreStep.java @@ -53,6 +53,7 @@ public class RegisterUserRestoreStep extends JobStep { DownloadableMedia downloadable = DownloadableMedia.create(media.getTitle(), fileName, media.getModified(), media.getCreated(), media.getLength(), 0L, sourceStoreUri.getId(), targetStoreUri.getId(), media.getId()); downloadable.put("recipient", recipient); + downloadable.put("skipValidation", true); String escortFileName = targetStoreName + "." + downloadable.getString("fileName"); String outputPath = null; try { diff --git a/server/user.jobengine.executors/src/user/jobengine/server/steps/TSMRestoreStep.java b/server/user.jobengine.executors/src/user/jobengine/server/steps/TSMRestoreStep.java index 9981bb39..07743474 100644 --- a/server/user.jobengine.executors/src/user/jobengine/server/steps/TSMRestoreStep.java +++ b/server/user.jobengine.executors/src/user/jobengine/server/steps/TSMRestoreStep.java @@ -55,7 +55,7 @@ public class TSMRestoreStep extends JobStep { @StepEntry public Object[] execute(Media mediaCubeMedia, String targetPath, String targetNamePattern, String successRecipient, int killDateDays, String localRetrievePath, String globalRetrievePath, IJobEngine jobEngine, IJobRuntime jobRuntime) throws Exception { - marker = jobRuntime.getMarker(); + marker = jobRuntime.getSessionMarker(); setAndCheck(mediaCubeMedia, targetPath, targetNamePattern, localRetrievePath, globalRetrievePath, jobEngine); String targetFileName = String.format(targetNamePattern, sourceFileName); try { diff --git a/server/user.jobengine.executors/src/user/jobengine/server/steps/TSMSystemRestoreStep.java b/server/user.jobengine.executors/src/user/jobengine/server/steps/TSMSystemRestoreStep.java index 314cd8e8..fd564a7c 100644 --- a/server/user.jobengine.executors/src/user/jobengine/server/steps/TSMSystemRestoreStep.java +++ b/server/user.jobengine.executors/src/user/jobengine/server/steps/TSMSystemRestoreStep.java @@ -55,7 +55,7 @@ public class TSMSystemRestoreStep extends JobStep { @StepEntry public Object[] execute(Media mediaCubeMedia, String targetPath, String targetNamePattern, String successRecipient, int killDateDays, IJobEngine jobEngine, IJobRuntime jobRuntime) throws Exception { - marker = jobRuntime.getMarker(); + marker = jobRuntime.getSessionMarker(); setAndCheck(mediaCubeMedia, targetPath, targetNamePattern, jobEngine); String targetFileName = String.format(targetNamePattern, sourceFileName); try { diff --git a/server/user.jobengine.executors/src/user/jobengine/server/steps/TranscodeFFAStranStep.java b/server/user.jobengine.executors/src/user/jobengine/server/steps/TranscodeFFAStranStep.java index 6c172d78..310df0d2 100644 --- a/server/user.jobengine.executors/src/user/jobengine/server/steps/TranscodeFFAStranStep.java +++ b/server/user.jobengine.executors/src/user/jobengine/server/steps/TranscodeFFAStranStep.java @@ -40,7 +40,7 @@ public class TranscodeFFAStranStep extends JobStep { public Object[] execute(ArchiveItem archiveItem, Media mediaCubeMedia, String transcoderAddress, String transcoderTemplateName, String globalHiresSourcePath, String localLowresTargetPath, boolean deleteSource, IJobEngine jobEngine, IJobRuntime jobRuntime) throws Exception { - this.marker = jobRuntime.getMarker(); + this.marker = jobRuntime.getSessionMarker(); this.manager = jobEngine.getItemManager(); this.store = check(manager.getCurrentLowresStore(), "lowres Store"); this.fileType = check(manager.getFileType(LOWRES_FILETYPE), "lowres FileType"); diff --git a/server/user.jobengine.executors/src/user/jobengine/server/steps/TranscodeSELENIOStep.java b/server/user.jobengine.executors/src/user/jobengine/server/steps/TranscodeSELENIOStep.java index 7b5407c3..a4b82c9b 100644 --- a/server/user.jobengine.executors/src/user/jobengine/server/steps/TranscodeSELENIOStep.java +++ b/server/user.jobengine.executors/src/user/jobengine/server/steps/TranscodeSELENIOStep.java @@ -92,7 +92,7 @@ public class TranscodeSELENIOStep extends JobStep { @StepEntry public Object[] execute(String globalSourcePath, ArchiveItem archiveItem, Media mediaCubeMedia, String transcoderTargetPath, IJobEngine jobEngine, IJobRuntime jobRuntime) throws Exception { - marker = jobRuntime.getMarker(); + marker = jobRuntime.getSessionMarker(); String sourceFileName = null; try { diff --git a/server/user.jobengine.executors/src/user/jobengine/server/steps/TransferFromTSMStep.java b/server/user.jobengine.executors/src/user/jobengine/server/steps/TransferFromTSMStep.java index a74c02b4..3521d3f6 100644 --- a/server/user.jobengine.executors/src/user/jobengine/server/steps/TransferFromTSMStep.java +++ b/server/user.jobengine.executors/src/user/jobengine/server/steps/TransferFromTSMStep.java @@ -8,4 +8,10 @@ public class TransferFromTSMStep extends TransferStep { public Object[] execute(StoreUri sourceStoreUri, String sourceFileName, StoreUri targetStoreUri, String targetFileName) throws Exception { return super.execute(sourceStoreUri, sourceFileName, targetStoreUri, targetFileName); } + + @Override + protected String getTmpExtension() { + return null; + } + } diff --git a/server/user.jobengine.executors/src/user/jobengine/server/steps/TransferStep.java b/server/user.jobengine.executors/src/user/jobengine/server/steps/TransferStep.java index 804c5d26..10270005 100644 --- a/server/user.jobengine.executors/src/user/jobengine/server/steps/TransferStep.java +++ b/server/user.jobengine.executors/src/user/jobengine/server/steps/TransferStep.java @@ -1,5 +1,6 @@ package user.jobengine.server.steps; +import java.nio.file.Path; import java.nio.file.Paths; import org.apache.logging.log4j.LogManager; @@ -7,28 +8,44 @@ import org.apache.logging.log4j.Logger; import user.commons.StoreUri; import user.commons.remotestore.RemoteStoreProtocol; +import user.jobengine.db.Store; public class TransferStep extends JobStep { + private static final String DOT_PART = ".part"; private static final Logger logger = LogManager.getLogger(); private static final boolean simulateTransferToTSM = Boolean.parseBoolean(System.getProperty("test.simulate.transfer.tsm", "false")); @StepEntry public Object[] execute(StoreUri sourceStoreUri, String sourceFileName, StoreUri targetStoreUri, String targetFileName) throws Exception { try { - String description = Paths.get(sourceStoreUri.toString(true), sourceFileName).toString(); + getJobRuntime().setCancelable(false); + + Store sourceStore = getManager().getStore(sourceStoreUri.getStoreId()); + Store targetStore = getManager().getStore(targetStoreUri.getStoreId()); + + String description = String.format("%s -> %s : %s", sourceStore.getName(), targetStore.getName(), sourceFileName); getJobRuntime().setDescription(description); //logger.info(getSessionMarker(), String.valueOf(isSimulateTSMWrite(sourceStoreUri, targetStoreUri))); if (isSimulateTSMWrite(sourceStoreUri, targetStoreUri)) { logger.info(getMarker(), "Skipping real TSM write of {} from {} to {}", sourceFileName, sourceStoreUri, targetStoreUri); - } else { - sourceStoreUri.addProgressListener(e -> setProgress(e.getProgress())); - sourceStoreUri.transferFrom(targetStoreUri, sourceFileName, targetFileName); - logger.info(getMarker(), "Transfer of {} completed from {} to {}", sourceFileName, sourceStoreUri, targetStoreUri); + return null; } + + sourceStoreUri.addProgressListener(e -> setProgress(e.getProgress())); + tryCopy(sourceStoreUri, sourceFileName, targetStoreUri, targetFileName); } catch (Exception e) { - logger.error(getSessionMarker(), "Error in transfer of {} when copying from {} to {}", sourceFileName, sourceStoreUri, targetStoreUri); - throw e; + logger.error(getMarker(), "Error in transfer of {} when copying from {} to {}. Retrying after 3 seconds.", sourceFileName, sourceStoreUri, + targetStoreUri); + + try { + Thread.sleep(3000); + tryCopy(sourceStoreUri, sourceFileName, targetStoreUri, targetFileName); + } catch (Exception e1) { + logger.error(getSessionMarker(), "Error in transfer of {} when copying from {} to {}. System message is: {}", sourceFileName, sourceStoreUri, + targetStoreUri, e1.getMessage()); + throw e1; + } } finally { if (sourceStoreUri != null) sourceStoreUri.cleanUp(); @@ -38,7 +55,30 @@ public class TransferStep extends JobStep { return null; } + protected String getTmpExtension() { + return DOT_PART; + } + private boolean isSimulateTSMWrite(StoreUri sourceStoreUri, StoreUri targetStoreUri) { return simulateTransferToTSM && (RemoteStoreProtocol.TSM.equals(targetStoreUri.getProtocol())); } + + private void tryCopy(StoreUri sourceStoreUri, String sourceFileName, StoreUri targetStoreUri, String targetFileName) throws Exception { + String currentTargetFileName = targetFileName; + + boolean renameAfterCopy = false; + if (getTmpExtension() != null && RemoteStoreProtocol.LOCAL.equals(targetStoreUri.getProtocol())) { + currentTargetFileName += getTmpExtension(); + renameAfterCopy = true; + } + + sourceStoreUri.transferFrom(targetStoreUri, sourceFileName, currentTargetFileName); + logger.info(getMarker(), "Transfer of {} completed from {} to {}", sourceFileName, sourceStoreUri, targetStoreUri); + + if (renameAfterCopy) { + Path tmpTargetFile = Paths.get(targetStoreUri.toString(true), currentTargetFileName); + Path targetFile = Paths.get(targetStoreUri.toString(true), targetFileName); + tmpTargetFile.toFile().renameTo(targetFile.toFile()); + } + } } diff --git a/server/user.jobengine.executors/src/user/jobengine/server/steps/UpdateGhostMediaData.java b/server/user.jobengine.executors/src/user/jobengine/server/steps/UpdateGhostMediaData.java index e6bd1414..b13d4962 100644 --- a/server/user.jobengine.executors/src/user/jobengine/server/steps/UpdateGhostMediaData.java +++ b/server/user.jobengine.executors/src/user/jobengine/server/steps/UpdateGhostMediaData.java @@ -21,7 +21,7 @@ public class UpdateGhostMediaData extends JobStep { @StepEntry public Object[] execute(Media mediaCubeMedia, IJobEngine jobEngine, IJobRuntime jobRuntime) throws Exception { - marker = jobRuntime.getMarker(); + marker = jobRuntime.getSessionMarker(); IItemManager manager = jobEngine.getItemManager(); //Refresh from db diff --git a/server/user.jobengine.executors/src/user/jobengine/server/steps/UploadRecordingToNexioStep.java b/server/user.jobengine.executors/src/user/jobengine/server/steps/UploadRecordingToNexioStep.java index 20796b80..7d9d6795 100644 --- a/server/user.jobengine.executors/src/user/jobengine/server/steps/UploadRecordingToNexioStep.java +++ b/server/user.jobengine.executors/src/user/jobengine/server/steps/UploadRecordingToNexioStep.java @@ -53,7 +53,7 @@ public class UploadRecordingToNexioStep extends JobStep { public Object[] execute(ArchiveItem archiveItem, String targetFileName, int nexioPort, String nexioUserName, String nexioPassword, IJobEngine jobEngine, IJobRuntime jobRuntime) throws Exception { - marker = jobRuntime.getMarker(); + marker = jobRuntime.getSessionMarker(); manager = jobEngine.getItemManager(); setAndCheck(archiveItem, targetFileName, nexioPort, nexioUserName, nexioPassword); File sourceFile = new File(archiveItem.getMediaFile()); diff --git a/server/user.jobengine.osgi.commons/src/user/commons/MediaCubeMarker.java b/server/user.jobengine.osgi.commons/src/user/commons/MediaCubeMarker.java index f5a1ffa3..0a1d0e3f 100644 --- a/server/user.jobengine.osgi.commons/src/user/commons/MediaCubeMarker.java +++ b/server/user.jobengine.osgi.commons/src/user/commons/MediaCubeMarker.java @@ -9,6 +9,7 @@ public class MediaCubeMarker extends Log4jMarker { private String subject; private String sessionID; private String sessionName; + private boolean useSessionLog; public MediaCubeMarker() { super(MEDIACUBE); @@ -33,10 +34,18 @@ public class MediaCubeMarker extends Log4jMarker { return sessionName; } + public String getSubject() { + return subject; + } + public String getTo() { return to; } + public boolean isUseSessionLog() { + return useSessionLog; + } + public void setSessionID(String sessionID) { this.sessionID = sessionID; } @@ -45,16 +54,16 @@ public class MediaCubeMarker extends Log4jMarker { this.sessionName = sessionName; } - public void setTo(String to) { - this.to = to; + public void setSubject(String subject) { + this.subject = subject; } - public String getSubject() { - return subject; + public void setTo(String to) { + this.to = to; } - public void setSubject(String subject) { - this.subject = subject; + public void setUseSessionLog(boolean useSessionLog) { + this.useSessionLog = useSessionLog; } // @Override diff --git a/server/user.jobengine.osgi.commons/src/user/commons/RemoteFile.java b/server/user.jobengine.osgi.commons/src/user/commons/RemoteFile.java index fd810710..59402be7 100644 --- a/server/user.jobengine.osgi.commons/src/user/commons/RemoteFile.java +++ b/server/user.jobengine.osgi.commons/src/user/commons/RemoteFile.java @@ -13,6 +13,11 @@ public class RemoteFile extends EntityBase implements Serializable { private long size; private boolean isFolder = false;; private Object sourceObject = null; + private byte[] content; + + public byte[] getContent() { + return content; + } @XmlJavaTypeAdapter(TimestampAdapter.class) public Timestamp getCreated() { @@ -49,6 +54,10 @@ public class RemoteFile extends EntityBase implements Serializable { return sourceObject; } + public void setContent(byte[] content) { + this.content = content; + } + public void setCreated(Timestamp created) { this.created = created; } diff --git a/server/user.jobengine.osgi.commons/src/user/commons/StoreUri.java b/server/user.jobengine.osgi.commons/src/user/commons/StoreUri.java index 9f0db73e..1947f94c 100644 --- a/server/user.jobengine.osgi.commons/src/user/commons/StoreUri.java +++ b/server/user.jobengine.osgi.commons/src/user/commons/StoreUri.java @@ -26,6 +26,7 @@ import user.commons.remotestore.StatusEvent; @SuppressWarnings("serial") public class StoreUri extends EntityBase implements Serializable { private final static Logger logger = LogManager.getLogger(); + private final int BUFFER_SIZE = 32768; private String uri = null; private long storeId = 0; @@ -81,7 +82,14 @@ public class StoreUri extends EntityBase implements Serializable { RemoteFile result = null; long uploadedBytes = 0; int byteCount = 0; - byte[] buffer = new byte[BUFFER_SIZE]; + byte[] buffer = null; + + try { + String bufferSizeProperty = System.getProperty("mediacube.copybuffer.size"); + buffer = new byte[Integer.parseInt(bufferSizeProperty)]; + } catch (Exception e) { + buffer = new byte[BUFFER_SIZE]; + } InputStream inputStream = inputLister.getInputStream(source); if (inputStream == null) @@ -168,6 +176,13 @@ public class StoreUri extends EntityBase implements Serializable { throw new Exception("InputLister is null."); } + public boolean exists(String name) throws Exception { + ensureLister(); + RemoteFile remoteFile = new RemoteFile(); + remoteFile.setName(name); + return lister.checkAvailability(remoteFile); + } + public boolean fileExists(String fileName) throws Exception { ensureLister(); return lister.exists(fileName); @@ -205,6 +220,11 @@ public class StoreUri extends EntityBase implements Serializable { return fileFilter; } + public RemoteFile getFileWithContent(String name) throws Exception { + ensureLister(); + return lister.getFileWithContent(name); + } + public IDirectoryLister getLister() throws Exception { ensureLister(); return lister; diff --git a/server/user.jobengine.osgi.commons/src/user/commons/harris/VICFileParser.java b/server/user.jobengine.osgi.commons/src/user/commons/harris/VICFileParser.java index 63c8c72b..303c3f5a 100644 --- a/server/user.jobengine.osgi.commons/src/user/commons/harris/VICFileParser.java +++ b/server/user.jobengine.osgi.commons/src/user/commons/harris/VICFileParser.java @@ -1,7 +1,9 @@ package user.commons.harris; +import java.io.BufferedReader; +import java.io.FileInputStream; import java.io.IOException; -import java.nio.file.Files; +import java.io.InputStreamReader; import java.nio.file.Path; import java.util.ArrayList; import java.util.Hashtable; @@ -33,9 +35,25 @@ public class VICFileParser { return; } + // logger.info("Parse format"); parseFormat(lineFormat); - List lines = Files.readAllLines(vicFile); + logger.info("Reading lines"); + + List lines = null; + try (BufferedReader r = new BufferedReader(new InputStreamReader(new FileInputStream(vicFile.toString())));) { + String sLine = r.readLine(); + while ((sLine = r.readLine()) != null) { + if (lines == null) + lines = new ArrayList<>(); + + lines.add(sLine); + } + } catch (Exception e) { + + } + + //List lines = Files.readAllLines(vicFile); if (lines == null) { logger.debug("Skipping empty file {}", vicFile); return; @@ -44,6 +62,7 @@ public class VICFileParser { String typeCodes = validTypeCodes.toUpperCase(); typeCodes = typeCodes.replace("SPACE", " "); + logger.info("Processing lines"); processLines(vicFile.toString(), lines, typeCodes); } @@ -89,11 +108,13 @@ public class VICFileParser { } private void processLines(String vicFileName, List lines, String typeCodes) { + logger.info("Identifying minimum length"); int minLength = columPositions.get("M").end; int nr = 0; for (String line : lines) { nr++; + logger.info("Processing line {}", nr); if (line.length() < minLength) { logger.debug("Skipping line {} from {}, line is too short.", nr, vicFileName); continue; diff --git a/server/user.jobengine.osgi.commons/src/user/commons/nexio/api/MediabaseImpl.java b/server/user.jobengine.osgi.commons/src/user/commons/nexio/api/MediabaseImpl.java index d54ca44d..2c9e4b02 100644 --- a/server/user.jobengine.osgi.commons/src/user/commons/nexio/api/MediabaseImpl.java +++ b/server/user.jobengine.osgi.commons/src/user/commons/nexio/api/MediabaseImpl.java @@ -15,8 +15,8 @@ class MediabaseImpl implements Mediabase { private Clip next; public Itr() throws IOException, ProtocolException { - Id id = protocol.executeListFirstIDHandle(); - next = id != null ? new ClipImpl(protocol, id) : null; + Id id = getProtocol().executeListFirstIDHandle(); + next = id != null ? new ClipImpl(getProtocol(), id) : null; } @Override @@ -32,11 +32,11 @@ class MediabaseImpl implements Mediabase { final Clip c = next; try { - Id id = protocol.executeListNextIDHandle(); + Id id = getProtocol().executeListNextIDHandle(); //next = id != null ? new ClipImpl(protocol, id) : null; if (id != null) { - next = new ClipImpl(protocol, id); + next = new ClipImpl(getProtocol(), id); } else { next = null; } @@ -74,7 +74,7 @@ class MediabaseImpl implements Mediabase { @Override public MediaListener createMediaListener(boolean disabled, ClipEventListener listener) { - mediaListener = new MediaListener(this, protocol.getConnection().getHost(), protocol.getConnection().getPort()); + mediaListener = new MediaListener(this, getProtocol().getConnection().getHost(), getProtocol().getConnection().getPort()); mediaListener.addClipEventListener(listener); return mediaListener; } @@ -84,20 +84,20 @@ class MediabaseImpl implements Mediabase { if (id == null) { throw new NullPointerException(ID); } - byte[] metadata = protocol.executeGetIDMetadata(id); + byte[] metadata = getProtocol().executeGetIDMetadata(id); if (metadata == null) { return null; } - return new ClipImpl(protocol, id, metadata); + return new ClipImpl(getProtocol(), id, metadata); } @Override public Clip getClip(Xid xid) throws IOException, ProtocolException { - Id id = protocol.executeGetIDHandleFromExtendedID(xid); + Id id = getProtocol().executeGetIDHandleFromExtendedID(xid); if (id == null) { return null; } - return new ClipImpl(protocol, id, xid); + return new ClipImpl(getProtocol(), id, xid); } @Override diff --git a/server/user.jobengine.osgi.commons/src/user/commons/nexio/server/protocol/Connection.java b/server/user.jobengine.osgi.commons/src/user/commons/nexio/server/protocol/Connection.java index a9c76690..8ed84319 100644 --- a/server/user.jobengine.osgi.commons/src/user/commons/nexio/server/protocol/Connection.java +++ b/server/user.jobengine.osgi.commons/src/user/commons/nexio/server/protocol/Connection.java @@ -1,31 +1,34 @@ package user.commons.nexio.server.protocol; import java.io.IOException; +import java.io.InputStream; public interface Connection { /** - * + * * @throws IOException */ public void disconnect() throws IOException; public void flush() throws IOException; + public String getHost(); + + InputStream getInput(); + + public int getPort(); + + public int read() throws IOException; + public int read(byte[] b) throws IOException; // Reads up to len bytes of data from the input stream into an array of // bytes. public int read(byte[] b, int off, int len) throws IOException; - public int read() throws IOException; - public void write(byte[] b) throws IOException; public void write(int b) throws IOException; - public String getHost(); - - public int getPort(); - } diff --git a/server/user.jobengine.osgi.commons/src/user/commons/nexio/server/protocol/GenericCommand.java b/server/user.jobengine.osgi.commons/src/user/commons/nexio/server/protocol/GenericCommand.java new file mode 100644 index 00000000..677b43d5 --- /dev/null +++ b/server/user.jobengine.osgi.commons/src/user/commons/nexio/server/protocol/GenericCommand.java @@ -0,0 +1,39 @@ +package user.commons.nexio.server.protocol; + +import java.io.ByteArrayOutputStream; +import java.io.DataInputStream; +import java.io.IOException; + +public class GenericCommand extends Command { + + private byte[] buffer; + + public GenericCommand(Connection connection) { + super(connection); + } + + public byte[] execute(byte[] command) throws IOException, ProtocolException { + byte[] result = null; + buffer = new byte[100]; + + try (ByteArrayOutputStream outputStream = new ByteArrayOutputStream()) { + connection.write(command); + connection.flush(); + DataInputStream in = (DataInputStream) connection.getInput(); + while (in.available() == 0) + Thread.sleep(100); + + while (in.available() > 0) { + int bytesRead = in.read(buffer); + outputStream.write(buffer, 0, bytesRead); + } + + result = outputStream.toByteArray(); + + } catch (Exception e) { + System.out.println(e.getMessage()); + } + return result; + } + +} diff --git a/server/user.jobengine.osgi.commons/src/user/commons/nexio/server/protocol/NexioServerProtocol.java b/server/user.jobengine.osgi.commons/src/user/commons/nexio/server/protocol/NexioServerProtocol.java index 49450e41..a8f09256 100644 --- a/server/user.jobengine.osgi.commons/src/user/commons/nexio/server/protocol/NexioServerProtocol.java +++ b/server/user.jobengine.osgi.commons/src/user/commons/nexio/server/protocol/NexioServerProtocol.java @@ -5,6 +5,8 @@ import java.util.Calendar; public interface NexioServerProtocol { + byte[] executeCommand(byte[] command) throws IOException, ProtocolException; + public byte[] executeGetExtendedFieldCommand(Id id, byte[] fieldNumber) throws IOException, ProtocolException; public Calendar executeGetExtendedFieldGetModifiedTimestamp(Id id) throws IOException, ProtocolException; diff --git a/server/user.jobengine.osgi.commons/src/user/commons/nexio/server/protocol/NexioServerProtocolImpl.java b/server/user.jobengine.osgi.commons/src/user/commons/nexio/server/protocol/NexioServerProtocolImpl.java index 72345194..40b38311 100644 --- a/server/user.jobengine.osgi.commons/src/user/commons/nexio/server/protocol/NexioServerProtocolImpl.java +++ b/server/user.jobengine.osgi.commons/src/user/commons/nexio/server/protocol/NexioServerProtocolImpl.java @@ -15,6 +15,7 @@ public class NexioServerProtocolImpl implements NexioServerProtocol { private GetIDFileSizeCommand getIDFileSizeCommand = null; private PortStatusCommand portStatusCommand = null; private GetExtendedFieldCommand getExtendedFieldCommand = null; + private GenericCommand genericCommand = null; private Connection connection = null; @@ -30,6 +31,12 @@ public class NexioServerProtocolImpl implements NexioServerProtocol { getIDFileSizeCommand = new GetIDFileSizeCommand(connection); portStatusCommand = new PortStatusCommand(connection); getExtendedFieldCommand = new GetExtendedFieldCommand(connection); + genericCommand = new GenericCommand(connection); + } + + @Override + public byte[] executeCommand(byte[] command) throws IOException, ProtocolException { + return genericCommand.execute(command); } @Override diff --git a/server/user.jobengine.osgi.commons/src/user/commons/nexio/server/protocol/TCPConnection.java b/server/user.jobengine.osgi.commons/src/user/commons/nexio/server/protocol/TCPConnection.java index cf542904..74a7479d 100644 --- a/server/user.jobengine.osgi.commons/src/user/commons/nexio/server/protocol/TCPConnection.java +++ b/server/user.jobengine.osgi.commons/src/user/commons/nexio/server/protocol/TCPConnection.java @@ -1,5 +1,6 @@ package user.commons.nexio.server.protocol; +import java.io.DataInputStream; import java.io.IOException; import java.io.InputStream; import java.io.OutputStream; @@ -13,8 +14,8 @@ public class TCPConnection implements Connection { return new TCPConnection(host, port); } - protected InputStream input; - protected OutputStream output; + private final DataInputStream input; + protected final OutputStream output; protected Socket socket = null; private String host = null; @@ -24,12 +25,14 @@ public class TCPConnection implements Connection { this.host = _host; this.port = _port; socket = new Socket(host, port); - input = socket.getInputStream(); + input = new DataInputStream(socket.getInputStream()); output = socket.getOutputStream(); } @Override public void disconnect() throws IOException { + if (input != null) + input.close(); if (socket != null && !socket.isClosed()) { socket.close(); } @@ -45,6 +48,11 @@ public class TCPConnection implements Connection { return this.host; } + @Override + public InputStream getInput() { + return input; + } + @Override public int getPort() { return this.port; @@ -52,13 +60,13 @@ public class TCPConnection implements Connection { @Override public int read() throws IOException { - return input.read(); + return getInput().read(); } @Override public int read(byte[] b) throws IOException { Arrays.fill(b, (byte) 0); - return input.read(b); + return getInput().read(b); } @Override @@ -66,7 +74,7 @@ public class TCPConnection implements Connection { // bytes. public int read(byte[] b, int off, int len) throws IOException { Arrays.fill(b, (byte) 0); - return input.read(b, off, len); + return getInput().read(b, off, len); } @Override diff --git a/server/user.jobengine.osgi.commons/src/user/commons/remotestore/FtpDirectoryLister.java b/server/user.jobengine.osgi.commons/src/user/commons/remotestore/FtpDirectoryLister.java index a9275da1..d2408657 100644 --- a/server/user.jobengine.osgi.commons/src/user/commons/remotestore/FtpDirectoryLister.java +++ b/server/user.jobengine.osgi.commons/src/user/commons/remotestore/FtpDirectoryLister.java @@ -1,23 +1,30 @@ package user.commons.remotestore; +import java.io.IOException; import java.io.InputStream; import java.io.OutputStream; +import java.net.SocketException; import java.sql.Timestamp; import java.util.ArrayList; import java.util.Arrays; import java.util.List; +import org.apache.commons.io.IOUtils; import org.apache.commons.lang.NotImplementedException; import org.apache.commons.lang.StringUtils; import org.apache.commons.net.ftp.FTP; import org.apache.commons.net.ftp.FTPClient; import org.apache.commons.net.ftp.FTPFile; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; import user.commons.IRemoteFileListCallback; import user.commons.RemoteFile; import user.commons.StoreUri; public class FtpDirectoryLister implements IDirectoryLister { + private static Logger logger = LogManager.getLogger(); + private final StoreUri storeUri; private FTPClient ftpClient; private boolean checkCompletePending; @@ -29,23 +36,19 @@ public class FtpDirectoryLister implements IDirectoryLister { @Override public boolean checkAvailability(RemoteFile remoteFile) throws Exception { boolean result = false; - InputStream input = null; + String name = remoteFile.getName(); connect(); if (ftpClient != null) { - input = ftpClient.retrieveFileStream(name); - checkCompletePending = true; - if (input != null) { - input.close(); - completePendingCommand(); - result = updateFileSize(remoteFile); - } else { - List remoteFiles = list(); - for (RemoteFile file : remoteFiles) { - if (file.getName().equals(name)) - if (file.getSize() == remoteFile.getSize()) - result = true; - } + InputStream input = null; + try { + input = ftpClient.retrieveFileStream(name); + result = input != null; + } catch (Exception e) { + logger.error(e.getMessage()); + } finally { + if (input != null) + input.close(); } } return result; @@ -54,15 +57,18 @@ public class FtpDirectoryLister implements IDirectoryLister { @Override public void cleanUp() throws Exception { if (ftpClient != null) { + logger.info("Disconnecting ftp"); + if (checkCompletePending) { completePendingCommand(); } - if (ftpClient.isConnected()) { + try { ftpClient.logout(); + ftpClient.quit(); ftpClient.disconnect(); ftpClient = null; - } else { - ftpClient = null; + } catch (Exception e) { + logger.error(e.getMessage()); } } } @@ -81,13 +87,32 @@ public class FtpDirectoryLister implements IDirectoryLister { public FTPClient connect() throws Exception { if (ftpClient != null) return ftpClient; + + logger.info("Connecting to ftp"); String path = storeUri.toString(true); String[] pathElements = path.split("/"); ftpClient = new FTPClient(); - if (storeUri.getPortNumber() == null || storeUri.getPortNumber() < 1) - ftpClient.connect(pathElements[0]); - else - ftpClient.connect(pathElements[0], storeUri.getPortNumber().intValue()); + //5mp + ftpClient.setConnectTimeout(5000); + + int tryconnect = 3; + boolean connected = false; + + while (tryconnect > 0) { + tryconnect--; + try { + + internalConnect(pathElements); + connected = true; + break; + } catch (Exception e) { + logger.error(e.getMessage()); + } + } + + if (!connected) + throw new Exception("Can not connect to " + storeUri.toString()); + String userName = storeUri.getUserName(); userName = StringUtils.isEmpty(userName) ? "anonymous" : userName; ftpClient.login(userName, storeUri.getPassword()); @@ -155,6 +180,29 @@ public class FtpDirectoryLister implements IDirectoryLister { return result; } + @Override + public RemoteFile getFileWithContent(String name) throws Exception { + RemoteFile result = null; + connect(); + if (ftpClient != null) { + try (InputStream input = ftpClient.retrieveFileStream(name)) { + checkCompletePending = true; + if (input != null) { + byte[] content = IOUtils.toByteArray(input); + input.read(content); + + result = new RemoteFile(); + result.setName(name); + result.setContent(content); + } + + } catch (Exception e) { + throw e; + } + } + return result; + } + @Override public InputStream getInputStream(RemoteFile remoteFile) throws Exception { InputStream input = null; @@ -178,6 +226,13 @@ public class FtpDirectoryLister implements IDirectoryLister { } + private void internalConnect(String[] pathElements) throws SocketException, IOException { + if (storeUri.getPortNumber() == null || storeUri.getPortNumber() < 1) + ftpClient.connect(pathElements[0]); + else + ftpClient.connect(pathElements[0], storeUri.getPortNumber().intValue()); + } + @Override public List list() throws Exception { List result = new ArrayList(); diff --git a/server/user.jobengine.osgi.commons/src/user/commons/remotestore/IDirectoryLister.java b/server/user.jobengine.osgi.commons/src/user/commons/remotestore/IDirectoryLister.java index dfa95641..c4e2d048 100644 --- a/server/user.jobengine.osgi.commons/src/user/commons/remotestore/IDirectoryLister.java +++ b/server/user.jobengine.osgi.commons/src/user/commons/remotestore/IDirectoryLister.java @@ -21,6 +21,8 @@ public interface IDirectoryLister { RemoteFile get(String fileName) throws Exception; + RemoteFile getFileWithContent(String name) throws Exception; + InputStream getInputStream(RemoteFile remoteFile) throws Exception; OutputStream getOutputStream(RemoteFile remoteFile) throws Exception; diff --git a/server/user.jobengine.osgi.commons/src/user/commons/remotestore/LocalDirectoryLister.java b/server/user.jobengine.osgi.commons/src/user/commons/remotestore/LocalDirectoryLister.java index 4e1299dd..2e7ac124 100644 --- a/server/user.jobengine.osgi.commons/src/user/commons/remotestore/LocalDirectoryLister.java +++ b/server/user.jobengine.osgi.commons/src/user/commons/remotestore/LocalDirectoryLister.java @@ -85,6 +85,14 @@ public class LocalDirectoryLister implements IDirectoryLister { return toRemoteFile(file); } + @Override + public RemoteFile getFileWithContent(String name) throws Exception { + Path path = Paths.get(storeUri.toString(true), name); + RemoteFile result = toRemoteFile(path.toFile()); + result.setContent(Files.readAllBytes(path)); + return result; + } + @Override public InputStream getInputStream(RemoteFile remoteFile) throws Exception { String path = storeUri.toString(true); diff --git a/server/user.jobengine.osgi.commons/src/user/commons/remotestore/SambaDirectoryLister.java b/server/user.jobengine.osgi.commons/src/user/commons/remotestore/SambaDirectoryLister.java index e1461ed3..649fe778 100644 --- a/server/user.jobengine.osgi.commons/src/user/commons/remotestore/SambaDirectoryLister.java +++ b/server/user.jobengine.osgi.commons/src/user/commons/remotestore/SambaDirectoryLister.java @@ -71,6 +71,12 @@ public class SambaDirectoryLister implements IDirectoryLister { return result; } + @Override + public RemoteFile getFileWithContent(String name) throws Exception { + // TODO Auto-generated method stub + return null; + } + @Override public InputStream getInputStream(RemoteFile remoteFile) throws Exception { NtlmPasswordAuthentication auth = new NtlmPasswordAuthentication(null, storeUri.getUserName(), storeUri.getPassword()); diff --git a/server/user.jobengine.osgi.commons/src/user/commons/remotestore/TSMLister.java b/server/user.jobengine.osgi.commons/src/user/commons/remotestore/TSMLister.java index 8129cda5..cdd011d3 100644 --- a/server/user.jobengine.osgi.commons/src/user/commons/remotestore/TSMLister.java +++ b/server/user.jobengine.osgi.commons/src/user/commons/remotestore/TSMLister.java @@ -19,15 +19,18 @@ public class TSMLister implements IDirectoryLister { private static final String NODENAME = System.getProperty("tsm.nodename", "JOBENGINE"); private static final String FSNAME = System.getProperty("tsm.fsname", "JOBENGINE"); + private static final String ALTERNATE_FSNAME = System.getProperty("tsm.fsname.alternate", "JOBENGINE"); private static final String HLNAME = System.getProperty("tsm.hlname", "JOBENGINE"); + private static final String LLNAMEQUALIFIER = System.getProperty("tsm.llname.qualifier", File.separator); private static final Logger logger = LogManager.getLogger(); private TSMBufferedClient client; public TSMLister(StoreUri storeUri) throws Exception { client = new TSMBufferedClient(NODENAME); logger.info("TSMBufferedClient created"); + //client.connect(storeUri.getUserName(), storeUri.getPassword(), "\\"); client.connect(storeUri.getUserName(), storeUri.getPassword()); - logger.info("TSMBufferedClient connected, parameters: {} {} {}, separator: {}", NODENAME, FSNAME, HLNAME, File.separator); + logger.info("TSMBufferedClient connected, parameters: {} {}/{} {}, separator: {}", NODENAME, FSNAME, ALTERNATE_FSNAME, HLNAME, File.separator); } @Override @@ -67,13 +70,24 @@ public class TSMLister implements IDirectoryLister { logger.info("Getting {}, {}, {}", FSNAME, HLNAME, currentFileName); TSMBackupFileObject backupFileObject = client.getActiveBackupFileObject(FSNAME, "\\", "\\" + currentFileName); - //probaljuk meg kiterjesztes nelkul is - if (backupFileObject == null && currentFileName.contains(".")) { - currentFileName = fileName.substring(0, fileName.lastIndexOf(".")); - logger.info("Getting {}, {}, {}", FSNAME, HLNAME, currentFileName); - backupFileObject = client.getActiveBackupFileObject(FSNAME, "\\", "\\" + currentFileName); + //probaljuk meg a masik tarbol + if (backupFileObject == null) { + logger.info("Getting {}, {}, {}", ALTERNATE_FSNAME, HLNAME, currentFileName); + backupFileObject = client.getActiveBackupFileObject(ALTERNATE_FSNAME, "\\", "\\" + currentFileName); } + //probaljuk meg kiterjesztes nelkul is + // if (backupFileObject == null && currentFileName.contains(".")) { + // currentFileName = fileName.substring(0, fileName.lastIndexOf(".")); + // logger.info("Getting {}, {}, {}", FSNAME, HLNAME, currentFileName); + // backupFileObject = client.getActiveBackupFileObject(FSNAME, HLNAME, LLNAMEQUALIFIER + currentFileName); + // + // if (backupFileObject == null) { + // logger.info("Getting {}, {}, {}", ALTERNATE_FSNAME, HLNAME, currentFileName); + // backupFileObject = client.getActiveBackupFileObject(ALTERNATE_FSNAME, HLNAME, LLNAMEQUALIFIER + currentFileName); + // } + // } + if (backupFileObject != null) { result = toRemoteFile(backupFileObject); logger.info("Got object {}, {}, {}", FSNAME, HLNAME, currentFileName); @@ -84,6 +98,12 @@ public class TSMLister implements IDirectoryLister { return result; } + @Override + public RemoteFile getFileWithContent(String name) throws Exception { + // TODO Auto-generated method stub + return null; + } + @Override public InputStream getInputStream(RemoteFile remoteFile) throws Exception { TSMInputStream inputStream = new TSMInputStream(client, remoteFile); @@ -94,13 +114,13 @@ public class TSMLister implements IDirectoryLister { @Override public OutputStream getOutputStream(RemoteFile remoteFile) throws Exception { - try { - logger.info("Trying register filespace"); - client.registerFilespace(FSNAME, "JOBENGINE", 'C', "JOBENGINE", 10L * 1024L * 1024L * 1024L, 0); - } catch (Exception e) { - logger.info(e.getMessage()); - } - TSMOutputStream outputStream = new TSMOutputStream(client, FSNAME, HLNAME, remoteFile); + // try { + // logger.info("Trying register filespace"); + // client.registerFilespace(FSNAME, "JOBENGINE", 'C', "JOBENGINE", 10L * 1024L * 1024L * 1024L, 0); + // } catch (Exception e) { + // logger.info(e.getMessage()); + // } + TSMOutputStream outputStream = new TSMOutputStream(client, FSNAME, "\\", remoteFile); outputStream.open(); return outputStream; } diff --git a/server/user.jobengine.osgi.commons/src/user/commons/remotestore/TSMOutputStream.java b/server/user.jobengine.osgi.commons/src/user/commons/remotestore/TSMOutputStream.java index c2b034a4..fba6c442 100644 --- a/server/user.jobengine.osgi.commons/src/user/commons/remotestore/TSMOutputStream.java +++ b/server/user.jobengine.osgi.commons/src/user/commons/remotestore/TSMOutputStream.java @@ -1,6 +1,5 @@ package user.commons.remotestore; -import java.io.File; import java.io.FileNotFoundException; import java.io.OutputStream; @@ -26,8 +25,18 @@ public class TSMOutputStream extends OutputStream { this.sizeEstimate = remoteFile.getSize(); } + @Override + public void close() { + try { + client.finishSend(); + } catch (Exception e) { + e.printStackTrace(); + } + client.disconnect(); + } + public void open() throws TSMException, FileNotFoundException { - backupObject = new TSMBackupFileObject(filespaceName, highLevelName, File.separator + fileName); + backupObject = new TSMBackupFileObject(filespaceName, highLevelName, "\\" + fileName); backupObject.setSizeEstimate(sizeEstimate); } @@ -46,16 +55,6 @@ public class TSMOutputStream extends OutputStream { } } - @Override - public void close() { - try { - client.finishSend(); - } catch (Exception e) { - e.printStackTrace(); - } - client.disconnect(); - } - @Override public void write(int arg0) { } diff --git a/server/user.jobengine.osgi.commons/test/user/common/harris/test/VICParserTest.java b/server/user.jobengine.osgi.commons/test/user/common/harris/test/VICParserTest.java index 94d6dcc4..75cf9330 100644 --- a/server/user.jobengine.osgi.commons/test/user/common/harris/test/VICParserTest.java +++ b/server/user.jobengine.osgi.commons/test/user/common/harris/test/VICParserTest.java @@ -17,7 +17,7 @@ public class VICParserTest { @Test public void testParser() throws Exception { - Path vicFile = Paths.get("/opt/PASA-DB/VIC/EDIT 10.VIC"); + Path vicFile = Paths.get("/opt/EDIT 9.VIC"); String lineFormat = "A TTTTTTTT LLLLLLLLLLL MMMMMMMMMMMMMMMMMMMMMMM X DDDDDDDDDDDDDDDDDDDDDDDDDDDDDDDD"; String validTypeCodes = "SPACE,E"; VICFileParser sut = new VICFileParser(vicFile, lineFormat, validTypeCodes); diff --git a/server/user.jobengine.osgi.db/generated/user/jobengine/db/BreakDAO_SJProfile0.ser b/server/user.jobengine.osgi.db/generated/user/jobengine/db/BreakDAO_SJProfile0.ser index 880386d4..8623276f 100644 Binary files a/server/user.jobengine.osgi.db/generated/user/jobengine/db/BreakDAO_SJProfile0.ser and b/server/user.jobengine.osgi.db/generated/user/jobengine/db/BreakDAO_SJProfile0.ser differ diff --git a/server/user.jobengine.osgi.db/generated/user/jobengine/db/DomainCategoryDAO_SJProfile0.ser b/server/user.jobengine.osgi.db/generated/user/jobengine/db/DomainCategoryDAO_SJProfile0.ser index fe5d6cf1..101941e9 100644 Binary files a/server/user.jobengine.osgi.db/generated/user/jobengine/db/DomainCategoryDAO_SJProfile0.ser and b/server/user.jobengine.osgi.db/generated/user/jobengine/db/DomainCategoryDAO_SJProfile0.ser differ diff --git a/server/user.jobengine.osgi.db/generated/user/jobengine/db/DomainDAO_SJProfile0.ser b/server/user.jobengine.osgi.db/generated/user/jobengine/db/DomainDAO_SJProfile0.ser index 33e78297..eb1ac450 100644 Binary files a/server/user.jobengine.osgi.db/generated/user/jobengine/db/DomainDAO_SJProfile0.ser and b/server/user.jobengine.osgi.db/generated/user/jobengine/db/DomainDAO_SJProfile0.ser differ diff --git a/server/user.jobengine.osgi.db/generated/user/jobengine/db/DomainIndexDAO_SJProfile0.ser b/server/user.jobengine.osgi.db/generated/user/jobengine/db/DomainIndexDAO_SJProfile0.ser index 4b246573..baa37152 100644 Binary files a/server/user.jobengine.osgi.db/generated/user/jobengine/db/DomainIndexDAO_SJProfile0.ser and b/server/user.jobengine.osgi.db/generated/user/jobengine/db/DomainIndexDAO_SJProfile0.ser differ diff --git a/server/user.jobengine.osgi.db/generated/user/jobengine/db/EntityBaseDAO.java b/server/user.jobengine.osgi.db/generated/user/jobengine/db/EntityBaseDAO.java index 9cdfc377..f5286275 100644 --- a/server/user.jobengine.osgi.db/generated/user/jobengine/db/EntityBaseDAO.java +++ b/server/user.jobengine.osgi.db/generated/user/jobengine/db/EntityBaseDAO.java @@ -10,7 +10,7 @@ import org.apache.logging.log4j.Logger; import sqlj.runtime.ref.DefaultContext; import sqlj.runtime.ref.ResultSetIterImpl; /** - * Az entit\ufffd\ufffd\ufffdsok alap DAO oszt\ufffd\ufffd\ufffdlya. + * Az entitasok alap DAO osztalya. */ @SuppressWarnings("unused") public abstract class EntityBaseDAO implements IEntityBaseDAO { @@ -216,6 +216,20 @@ public abstract class EntityBaseDAO implements IEntityBaseDAO { } return result; } + + @Override + public List getAll(IDAOIterProvider provider) { + List result = null; + DefaultContext context = manager.getDbContext(); + try { + result = getAll(context, provider); + } catch (Exception e) { + throw new ItemManagerException(e); + } finally { + manager.putDbContext(context); + } + return result; + } @Override public IEntityBase get(IDAOIterProvider provider) { @@ -246,6 +260,20 @@ public abstract class EntityBaseDAO implements IEntityBaseDAO { return entity; } + public List getAll(DefaultContext context, IDAOIterProvider provider) { + manager.traceIn(); + List result = null; + try { + ResultSetIterImpl iter = provider.get(context, this); + checkNull(iter, ResultSetIterImpl.class); + result = getList(context, iter, false, true); + } catch (Exception e) { + throw new ItemManagerException(e); + } + manager.traceOut(); + return result; + } + public List getAll(DefaultContext context) { manager.traceIn(); List result = null; @@ -348,7 +376,7 @@ public abstract class EntityBaseDAO implements IEntityBaseDAO { manager.setAllCached(getCacheType(), result); } - /*@lineinfo:generated-code*//*@lineinfo:252^3*/ + /*@lineinfo:generated-code*//*@lineinfo:280^3*/ // ************************************************************ // #sql [context] { COMMIT }; @@ -375,10 +403,10 @@ public abstract class EntityBaseDAO implements IEntityBaseDAO { // ************************************************************ -/*@lineinfo:user-code*//*@lineinfo:252^27*/ +/*@lineinfo:user-code*//*@lineinfo:280^27*/ } catch (Exception e) { try { - /*@lineinfo:generated-code*//*@lineinfo:255^4*/ + /*@lineinfo:generated-code*//*@lineinfo:283^4*/ // ************************************************************ // #sql [context] { ROLLBACK }; @@ -405,7 +433,7 @@ public abstract class EntityBaseDAO implements IEntityBaseDAO { // ************************************************************ -/*@lineinfo:user-code*//*@lineinfo:255^30*/ +/*@lineinfo:user-code*//*@lineinfo:283^30*/ } catch(Exception e1) { } manager.throwError(e); @@ -449,7 +477,7 @@ public abstract class EntityBaseDAO implements IEntityBaseDAO { entity.checkParameter("ID", id, false); update(context, entity); afterModify(context, entity); - /*@lineinfo:generated-code*//*@lineinfo:299^3*/ + /*@lineinfo:generated-code*//*@lineinfo:327^3*/ // ************************************************************ // #sql [context] { COMMIT }; @@ -476,12 +504,12 @@ public abstract class EntityBaseDAO implements IEntityBaseDAO { // ************************************************************ -/*@lineinfo:user-code*//*@lineinfo:299^27*/ +/*@lineinfo:user-code*//*@lineinfo:327^27*/ if (useMemoryCache()) manager.storeCached(entity.getId(), entity); } catch (Exception e) { try { - /*@lineinfo:generated-code*//*@lineinfo:304^4*/ + /*@lineinfo:generated-code*//*@lineinfo:332^4*/ // ************************************************************ // #sql [context] { ROLLBACK }; @@ -508,7 +536,7 @@ public abstract class EntityBaseDAO implements IEntityBaseDAO { // ************************************************************ -/*@lineinfo:user-code*//*@lineinfo:304^30*/ +/*@lineinfo:user-code*//*@lineinfo:332^30*/ } catch(Exception e1) { } manager.throwError(e); @@ -537,7 +565,7 @@ public abstract class EntityBaseDAO implements IEntityBaseDAO { beforeDelete(context, entity); delete(context, id); afterDelete(context, entity); - /*@lineinfo:generated-code*//*@lineinfo:333^3*/ + /*@lineinfo:generated-code*//*@lineinfo:361^3*/ // ************************************************************ // #sql [context] { COMMIT }; @@ -564,12 +592,12 @@ public abstract class EntityBaseDAO implements IEntityBaseDAO { // ************************************************************ -/*@lineinfo:user-code*//*@lineinfo:333^27*/ +/*@lineinfo:user-code*//*@lineinfo:361^27*/ if (useMemoryCache()) manager.removeCached(getCacheType(), entity.getId()); } catch (Exception e) { try { - /*@lineinfo:generated-code*//*@lineinfo:338^4*/ + /*@lineinfo:generated-code*//*@lineinfo:366^4*/ // ************************************************************ // #sql [context] { ROLLBACK }; @@ -596,7 +624,7 @@ public abstract class EntityBaseDAO implements IEntityBaseDAO { // ************************************************************ -/*@lineinfo:user-code*//*@lineinfo:338^30*/ +/*@lineinfo:user-code*//*@lineinfo:366^30*/ } catch(Exception e1) { } manager.throwError(e); @@ -620,7 +648,7 @@ public abstract class EntityBaseDAO implements IEntityBaseDAO { try { for (long id : ids) delete(context, id); - /*@lineinfo:generated-code*//*@lineinfo:362^3*/ + /*@lineinfo:generated-code*//*@lineinfo:390^3*/ // ************************************************************ // #sql [context] { COMMIT }; @@ -647,14 +675,14 @@ public abstract class EntityBaseDAO implements IEntityBaseDAO { // ************************************************************ -/*@lineinfo:user-code*//*@lineinfo:362^27*/ +/*@lineinfo:user-code*//*@lineinfo:390^27*/ if (useMemoryCache()) { for (long id : ids) manager.removeCached(getCacheType(), id); } } catch (Exception e) { try { - /*@lineinfo:generated-code*//*@lineinfo:369^4*/ + /*@lineinfo:generated-code*//*@lineinfo:397^4*/ // ************************************************************ // #sql [context] { ROLLBACK }; @@ -681,7 +709,7 @@ public abstract class EntityBaseDAO implements IEntityBaseDAO { // ************************************************************ -/*@lineinfo:user-code*//*@lineinfo:369^30*/ +/*@lineinfo:user-code*//*@lineinfo:397^30*/ } catch(Exception e1) { } manager.throwError(e); @@ -695,7 +723,7 @@ public abstract class EntityBaseDAO implements IEntityBaseDAO { DefaultContext context = manager.getDbContext(); try { truncateTable(context); - /*@lineinfo:generated-code*//*@lineinfo:383^3*/ + /*@lineinfo:generated-code*//*@lineinfo:411^3*/ // ************************************************************ // #sql [context] { COMMIT }; @@ -722,7 +750,7 @@ public abstract class EntityBaseDAO implements IEntityBaseDAO { // ************************************************************ -/*@lineinfo:user-code*//*@lineinfo:383^27*/ +/*@lineinfo:user-code*//*@lineinfo:411^27*/ } catch (Exception e) { throw new ItemManagerException(e); } finally { diff --git a/server/user.jobengine.osgi.db/generated/user/jobengine/db/EntityBaseDAO_SJProfile0.ser b/server/user.jobengine.osgi.db/generated/user/jobengine/db/EntityBaseDAO_SJProfile0.ser index ba588c78..41300e56 100644 Binary files a/server/user.jobengine.osgi.db/generated/user/jobengine/db/EntityBaseDAO_SJProfile0.ser and b/server/user.jobengine.osgi.db/generated/user/jobengine/db/EntityBaseDAO_SJProfile0.ser differ diff --git a/server/user.jobengine.osgi.db/generated/user/jobengine/db/FileTypeDAO_SJProfile0.ser b/server/user.jobengine.osgi.db/generated/user/jobengine/db/FileTypeDAO_SJProfile0.ser index 10a12a48..52589159 100644 Binary files a/server/user.jobengine.osgi.db/generated/user/jobengine/db/FileTypeDAO_SJProfile0.ser and b/server/user.jobengine.osgi.db/generated/user/jobengine/db/FileTypeDAO_SJProfile0.ser differ diff --git a/server/user.jobengine.osgi.db/generated/user/jobengine/db/FolderDAO_SJProfile0.ser b/server/user.jobengine.osgi.db/generated/user/jobengine/db/FolderDAO_SJProfile0.ser index ffb9a79c..1f5baa06 100644 Binary files a/server/user.jobengine.osgi.db/generated/user/jobengine/db/FolderDAO_SJProfile0.ser and b/server/user.jobengine.osgi.db/generated/user/jobengine/db/FolderDAO_SJProfile0.ser differ diff --git a/server/user.jobengine.osgi.db/generated/user/jobengine/db/HelperDAO_SJProfile0.ser b/server/user.jobengine.osgi.db/generated/user/jobengine/db/HelperDAO_SJProfile0.ser index a51a003e..1f14b659 100644 Binary files a/server/user.jobengine.osgi.db/generated/user/jobengine/db/HelperDAO_SJProfile0.ser and b/server/user.jobengine.osgi.db/generated/user/jobengine/db/HelperDAO_SJProfile0.ser differ diff --git a/server/user.jobengine.osgi.db/generated/user/jobengine/db/ItemDAO_SJProfile0.ser b/server/user.jobengine.osgi.db/generated/user/jobengine/db/ItemDAO_SJProfile0.ser index 7199db4e..ffad80e2 100644 Binary files a/server/user.jobengine.osgi.db/generated/user/jobengine/db/ItemDAO_SJProfile0.ser and b/server/user.jobengine.osgi.db/generated/user/jobengine/db/ItemDAO_SJProfile0.ser differ diff --git a/server/user.jobengine.osgi.db/generated/user/jobengine/db/ItemTypeDAO_SJProfile0.ser b/server/user.jobengine.osgi.db/generated/user/jobengine/db/ItemTypeDAO_SJProfile0.ser index 74fea23f..e0e14d26 100644 Binary files a/server/user.jobengine.osgi.db/generated/user/jobengine/db/ItemTypeDAO_SJProfile0.ser and b/server/user.jobengine.osgi.db/generated/user/jobengine/db/ItemTypeDAO_SJProfile0.ser differ diff --git a/server/user.jobengine.osgi.db/generated/user/jobengine/db/JobDAO.java b/server/user.jobengine.osgi.db/generated/user/jobengine/db/JobDAO.java index 6847ddd7..7a194b10 100644 --- a/server/user.jobengine.osgi.db/generated/user/jobengine/db/JobDAO.java +++ b/server/user.jobengine.osgi.db/generated/user/jobengine/db/JobDAO.java @@ -41,6 +41,7 @@ class JobDAO extends EntityBaseDAO { entity.setFinished(iterator.finished()); entity.setTemplate(iterator.template()); entity.setScheduledTime(iterator.scheduledTime()); + entity.setRelated(iterator.related()); if (result == null) result = new ArrayList(); result.add(entity); @@ -52,7 +53,7 @@ class JobDAO extends EntityBaseDAO { @Override public List entities(DefaultContext context, ResultSet rs) throws SQLException { JobIter iter = null; - /*@lineinfo:generated-code*//*@lineinfo:55^2*/ + /*@lineinfo:generated-code*//*@lineinfo:56^2*/ // ************************************************************ // #sql [context] iter = { CAST :rs }; @@ -81,17 +82,17 @@ class JobDAO extends EntityBaseDAO { // ************************************************************ -/*@lineinfo:user-code*//*@lineinfo:55^35*/ +/*@lineinfo:user-code*//*@lineinfo:56^35*/ return entities(context, iter, false); } @Override protected ResultSetIterImpl selectByKey(DefaultContext context, long id) throws SQLException{ JobIter iter = null; - /*@lineinfo:generated-code*//*@lineinfo:62^2*/ + /*@lineinfo:generated-code*//*@lineinfo:63^2*/ // ************************************************************ -// #sql [context] iter = { SELECT ID, NAME, OWNER, PRIORITY, PROGRESS, STATUS, DESCRIPTION, SUBMITTED, FINISHED, TEMPLATE, SCHEDULEDTIME FROM JOB WHERE ID = :id }; +// #sql [context] iter = { SELECT ID, NAME, OWNER, PRIORITY, PROGRESS, STATUS, DESCRIPTION, SUBMITTED, FINISHED, TEMPLATE, SCHEDULEDTIME, RELATED FROM JOB WHERE ID = :id }; // ************************************************************ { @@ -116,17 +117,17 @@ class JobDAO extends EntityBaseDAO { // ************************************************************ -/*@lineinfo:user-code*//*@lineinfo:62^160*/ +/*@lineinfo:user-code*//*@lineinfo:63^169*/ return iter; } @Override protected ResultSetIterImpl selectAll(DefaultContext context) throws SQLException{ JobIter iter = null; - /*@lineinfo:generated-code*//*@lineinfo:69^2*/ + /*@lineinfo:generated-code*//*@lineinfo:70^2*/ // ************************************************************ -// #sql [context] iter = { SELECT ID, NAME, OWNER, PRIORITY, PROGRESS, STATUS, DESCRIPTION, SUBMITTED, FINISHED, TEMPLATE, SCHEDULEDTIME FROM JOB }; +// #sql [context] iter = { SELECT ID, NAME, OWNER, PRIORITY, PROGRESS, STATUS, DESCRIPTION, SUBMITTED, FINISHED, TEMPLATE, SCHEDULEDTIME, RELATED FROM JOB }; // ************************************************************ { @@ -150,13 +151,47 @@ class JobDAO extends EntityBaseDAO { // ************************************************************ -/*@lineinfo:user-code*//*@lineinfo:69^145*/ +/*@lineinfo:user-code*//*@lineinfo:70^154*/ + return iter; + } + + public ResultSetIterImpl selectByRelated(DefaultContext context, String related) throws SQLException{ + JobIter iter = null; + /*@lineinfo:generated-code*//*@lineinfo:76^2*/ + +// ************************************************************ +// #sql [context] iter = { SELECT ID, NAME, OWNER, PRIORITY, PROGRESS, STATUS, DESCRIPTION, SUBMITTED, FINISHED, TEMPLATE, SCHEDULEDTIME, RELATED FROM JOB WHERE LOOKUP(RELATED, :related) > 0 }; +// ************************************************************ + +{ + sqlj.runtime.ConnectionContext __sJT_connCtx = context; + if (__sJT_connCtx == null) sqlj.runtime.error.RuntimeRefErrors.raise_NULL_DEFAULT_CONN_CTX(); + sqlj.runtime.ExecutionContext __sJT_execCtx = __sJT_connCtx.getExecutionContext(); + if (__sJT_execCtx == null) sqlj.runtime.error.RuntimeRefErrors.raise_NULL_EXEC_CTX(); + synchronized (__sJT_execCtx) { + sqlj.runtime.profile.RTStatement __sJT_stmt = __sJT_execCtx.registerStatement(__sJT_connCtx, JobDAO_SJProfileKeys.getKey(0), 3); + try + { + __sJT_stmt.setString(1, related); + iter = new JobIter(__sJT_execCtx.executeQuery(), __sJT_execCtx.getFetchSize(), __sJT_execCtx.getMaxRows()); + } + finally + { + __sJT_execCtx.releaseStatement(); + } + } +} + + +// ************************************************************ + +/*@lineinfo:user-code*//*@lineinfo:76^190*/ return iter; } @Override protected void delete(DefaultContext context, long id) throws SQLException{ - /*@lineinfo:generated-code*//*@lineinfo:75^2*/ + /*@lineinfo:generated-code*//*@lineinfo:82^2*/ // ************************************************************ // #sql [context] { DELETE FROM JOB WHERE ID = :id }; @@ -168,7 +203,7 @@ class JobDAO extends EntityBaseDAO { sqlj.runtime.ExecutionContext __sJT_execCtx = __sJT_connCtx.getExecutionContext(); if (__sJT_execCtx == null) sqlj.runtime.error.RuntimeRefErrors.raise_NULL_EXEC_CTX(); synchronized (__sJT_execCtx) { - sqlj.runtime.profile.RTStatement __sJT_stmt = __sJT_execCtx.registerStatement(__sJT_connCtx, JobDAO_SJProfileKeys.getKey(0), 3); + sqlj.runtime.profile.RTStatement __sJT_stmt = __sJT_execCtx.registerStatement(__sJT_connCtx, JobDAO_SJProfileKeys.getKey(0), 4); try { __sJT_stmt.setLong(1, id); @@ -184,12 +219,12 @@ class JobDAO extends EntityBaseDAO { // ************************************************************ -/*@lineinfo:user-code*//*@lineinfo:75^50*/ +/*@lineinfo:user-code*//*@lineinfo:82^50*/ } @Override protected void truncateTable(DefaultContext context) throws SQLException{ - /*@lineinfo:generated-code*//*@lineinfo:80^2*/ + /*@lineinfo:generated-code*//*@lineinfo:87^2*/ // ************************************************************ // #sql [context] { TRUNCATE TABLE JOB DROP STORAGE IGNORE DELETE TRIGGERS IMMEDIATE }; @@ -201,7 +236,7 @@ class JobDAO extends EntityBaseDAO { sqlj.runtime.ExecutionContext __sJT_execCtx = __sJT_connCtx.getExecutionContext(); if (__sJT_execCtx == null) sqlj.runtime.error.RuntimeRefErrors.raise_NULL_EXEC_CTX(); synchronized (__sJT_execCtx) { - sqlj.runtime.profile.RTStatement __sJT_stmt = __sJT_execCtx.registerStatement(__sJT_connCtx, JobDAO_SJProfileKeys.getKey(0), 4); + sqlj.runtime.profile.RTStatement __sJT_stmt = __sJT_execCtx.registerStatement(__sJT_connCtx, JobDAO_SJProfileKeys.getKey(0), 5); try { __sJT_execCtx.executeUpdate(); @@ -216,7 +251,7 @@ class JobDAO extends EntityBaseDAO { // ************************************************************ -/*@lineinfo:user-code*//*@lineinfo:80^84*/ +/*@lineinfo:user-code*//*@lineinfo:87^84*/ } @Override @@ -233,11 +268,12 @@ class JobDAO extends EntityBaseDAO { Timestamp finished = obj.getFinished(); String template = obj.getTemplate(); Timestamp scheduledTime = obj.getScheduledTime(); - - /*@lineinfo:generated-code*//*@lineinfo:98^2*/ + String related = obj.getRelated(); + + /*@lineinfo:generated-code*//*@lineinfo:106^2*/ // ************************************************************ -// #sql [context] { UPDATE JOB SET NAME = :name, OWNER = :owner, PRIORITY = :priority, PROGRESS = :progress, STATUS = :status, DESCRIPTION = :description, SUBMITTED = :submitted, FINISHED = :finished, TEMPLATE = :template, SCHEDULEDTIME = :scheduledTime WHERE ID = :id }; +// #sql [context] { UPDATE JOB SET NAME = :name, OWNER = :owner, PRIORITY = :priority, PROGRESS = :progress, STATUS = :status, DESCRIPTION = :description, SUBMITTED = :submitted, FINISHED = :finished, TEMPLATE = :template, SCHEDULEDTIME = :scheduledTime, RELATED = :related WHERE ID = :id }; // ************************************************************ { @@ -246,7 +282,7 @@ class JobDAO extends EntityBaseDAO { sqlj.runtime.ExecutionContext __sJT_execCtx = __sJT_connCtx.getExecutionContext(); if (__sJT_execCtx == null) sqlj.runtime.error.RuntimeRefErrors.raise_NULL_EXEC_CTX(); synchronized (__sJT_execCtx) { - sqlj.runtime.profile.RTStatement __sJT_stmt = __sJT_execCtx.registerStatement(__sJT_connCtx, JobDAO_SJProfileKeys.getKey(0), 5); + sqlj.runtime.profile.RTStatement __sJT_stmt = __sJT_execCtx.registerStatement(__sJT_connCtx, JobDAO_SJProfileKeys.getKey(0), 6); try { __sJT_stmt.setString(1, name); @@ -259,7 +295,8 @@ class JobDAO extends EntityBaseDAO { __sJT_stmt.setTimestamp(8, finished); __sJT_stmt.setString(9, template); __sJT_stmt.setTimestamp(10, scheduledTime); - __sJT_stmt.setLong(11, id); + __sJT_stmt.setString(11, related); + __sJT_stmt.setLong(12, id); __sJT_execCtx.executeUpdate(); } finally @@ -272,7 +309,7 @@ class JobDAO extends EntityBaseDAO { // ************************************************************ -/*@lineinfo:user-code*//*@lineinfo:98^268*/ +/*@lineinfo:user-code*//*@lineinfo:106^288*/ } @Override @@ -288,12 +325,13 @@ class JobDAO extends EntityBaseDAO { Timestamp finished = obj.getFinished(); String template = obj.getTemplate(); Timestamp scheduledTime = obj.getScheduledTime(); + String related = obj.getRelated(); - /*@lineinfo:generated-code*//*@lineinfo:115^2*/ + /*@lineinfo:generated-code*//*@lineinfo:124^2*/ // ************************************************************ -// #sql [context] { INSERT INTO JOB (NAME, OWNER, PRIORITY, PROGRESS, STATUS, DESCRIPTION, SUBMITTED, FINISHED, TEMPLATE, SCHEDULEDTIME) -// VALUES (:name, :owner, :priority, :progress, :status, :description, :submitted, :finished, :template, :scheduledTime) }; +// #sql [context] { INSERT INTO JOB (NAME, OWNER, PRIORITY, PROGRESS, STATUS, DESCRIPTION, SUBMITTED, FINISHED, TEMPLATE, SCHEDULEDTIME, RELATED) +// VALUES (:name, :owner, :priority, :progress, :status, :description, :submitted, :finished, :template, :scheduledTime, :related) }; // ************************************************************ { @@ -302,7 +340,7 @@ class JobDAO extends EntityBaseDAO { sqlj.runtime.ExecutionContext __sJT_execCtx = __sJT_connCtx.getExecutionContext(); if (__sJT_execCtx == null) sqlj.runtime.error.RuntimeRefErrors.raise_NULL_EXEC_CTX(); synchronized (__sJT_execCtx) { - sqlj.runtime.profile.RTStatement __sJT_stmt = __sJT_execCtx.registerStatement(__sJT_connCtx, JobDAO_SJProfileKeys.getKey(0), 6); + sqlj.runtime.profile.RTStatement __sJT_stmt = __sJT_execCtx.registerStatement(__sJT_connCtx, JobDAO_SJProfileKeys.getKey(0), 7); try { __sJT_stmt.setString(1, name); @@ -315,6 +353,7 @@ class JobDAO extends EntityBaseDAO { __sJT_stmt.setTimestamp(8, finished); __sJT_stmt.setString(9, template); __sJT_stmt.setTimestamp(10, scheduledTime); + __sJT_stmt.setString(11, related); __sJT_execCtx.executeUpdate(); } finally @@ -327,7 +366,7 @@ class JobDAO extends EntityBaseDAO { // ************************************************************ -/*@lineinfo:user-code*//*@lineinfo:116^120*/ +/*@lineinfo:user-code*//*@lineinfo:125^130*/ } public List getAllIncomplete(String template) { @@ -337,10 +376,10 @@ class JobDAO extends EntityBaseDAO { try { JobIter iter = null; //ResultSetIterImpl iter = null; - /*@lineinfo:generated-code*//*@lineinfo:126^3*/ + /*@lineinfo:generated-code*//*@lineinfo:135^3*/ // ************************************************************ -// #sql [context] iter = { SELECT ID, NAME, OWNER, PRIORITY, PROGRESS, STATUS, DESCRIPTION, SUBMITTED, FINISHED, TEMPLATE, SCHEDULEDTIME +// #sql [context] iter = { SELECT ID, NAME, OWNER, PRIORITY, PROGRESS, STATUS, DESCRIPTION, SUBMITTED, FINISHED, TEMPLATE, SCHEDULEDTIME, RELATED // FROM JOB WHERE TEMPLATE = :template // AND STATUS IN ('EXECUTING', 'WAIT_EXECUTOR', 'WAIT_SUSPEND', 'RUNABLE') }; // ************************************************************ @@ -351,7 +390,7 @@ class JobDAO extends EntityBaseDAO { sqlj.runtime.ExecutionContext __sJT_execCtx = __sJT_connCtx.getExecutionContext(); if (__sJT_execCtx == null) sqlj.runtime.error.RuntimeRefErrors.raise_NULL_EXEC_CTX(); synchronized (__sJT_execCtx) { - sqlj.runtime.profile.RTStatement __sJT_stmt = __sJT_execCtx.registerStatement(__sJT_connCtx, JobDAO_SJProfileKeys.getKey(0), 7); + sqlj.runtime.profile.RTStatement __sJT_stmt = __sJT_execCtx.registerStatement(__sJT_connCtx, JobDAO_SJProfileKeys.getKey(0), 8); try { __sJT_stmt.setString(1, template); @@ -367,7 +406,7 @@ class JobDAO extends EntityBaseDAO { // ************************************************************ -/*@lineinfo:user-code*//*@lineinfo:128^80*/ +/*@lineinfo:user-code*//*@lineinfo:137^80*/ result = getList(context, iter, false); } catch (Exception e) { throw new ItemManagerException(e); diff --git a/server/user.jobengine.osgi.db/generated/user/jobengine/db/JobDAO_SJProfile0.ser b/server/user.jobengine.osgi.db/generated/user/jobengine/db/JobDAO_SJProfile0.ser index 919dd7a9..faa88b59 100644 Binary files a/server/user.jobengine.osgi.db/generated/user/jobengine/db/JobDAO_SJProfile0.ser and b/server/user.jobengine.osgi.db/generated/user/jobengine/db/JobDAO_SJProfile0.ser differ diff --git a/server/user.jobengine.osgi.db/generated/user/jobengine/db/JobIter.java b/server/user.jobengine.osgi.db/generated/user/jobengine/db/JobIter.java index 9763f105..a7bb1677 100644 --- a/server/user.jobengine.osgi.db/generated/user/jobengine/db/JobIter.java +++ b/server/user.jobengine.osgi.db/generated/user/jobengine/db/JobIter.java @@ -12,6 +12,7 @@ public class JobIter extends sqlj.runtime.ref.ResultSetIterImpl implements sqlj.runtime.NamedIterator { + private int relatedNdx; private int scheduledTimeNdx; private int templateNdx; private int finishedNdx; @@ -38,6 +39,7 @@ implements sqlj.runtime.NamedIterator finishedNdx = findColumn("finished"); templateNdx = findColumn("template"); scheduledTimeNdx = findColumn("scheduledTime"); + relatedNdx = findColumn("related"); } public JobIter(sqlj.runtime.profile.RTResultSet resultSet, int fetchSize, int maxRows) throws java.sql.SQLException @@ -54,6 +56,7 @@ implements sqlj.runtime.NamedIterator finishedNdx = findColumn("finished"); templateNdx = findColumn("template"); scheduledTimeNdx = findColumn("scheduledTime"); + relatedNdx = findColumn("related"); } public long id() throws java.sql.SQLException @@ -110,9 +113,14 @@ implements sqlj.runtime.NamedIterator { return resultSet.getTimestamp(scheduledTimeNdx); } + public String related() + throws java.sql.SQLException + { + return resultSet.getString(relatedNdx); + } } // ************************************************************ -/*@lineinfo:user-code*//*@lineinfo:5^209*//*@lineinfo:generated-code*/ \ No newline at end of file +/*@lineinfo:user-code*//*@lineinfo:5^225*//*@lineinfo:generated-code*/ \ No newline at end of file diff --git a/server/user.jobengine.osgi.db/generated/user/jobengine/db/JobParametersDAO_SJProfile0.ser b/server/user.jobengine.osgi.db/generated/user/jobengine/db/JobParametersDAO_SJProfile0.ser index e4e2ef02..30143842 100644 Binary files a/server/user.jobengine.osgi.db/generated/user/jobengine/db/JobParametersDAO_SJProfile0.ser and b/server/user.jobengine.osgi.db/generated/user/jobengine/db/JobParametersDAO_SJProfile0.ser differ diff --git a/server/user.jobengine.osgi.db/generated/user/jobengine/db/MasterIdDAO_SJProfile0.ser b/server/user.jobengine.osgi.db/generated/user/jobengine/db/MasterIdDAO_SJProfile0.ser index bd72e112..68ce6170 100644 Binary files a/server/user.jobengine.osgi.db/generated/user/jobengine/db/MasterIdDAO_SJProfile0.ser and b/server/user.jobengine.osgi.db/generated/user/jobengine/db/MasterIdDAO_SJProfile0.ser differ diff --git a/server/user.jobengine.osgi.db/generated/user/jobengine/db/MediaDAO_SJProfile0.ser b/server/user.jobengine.osgi.db/generated/user/jobengine/db/MediaDAO_SJProfile0.ser index 5a151de7..212468d3 100644 Binary files a/server/user.jobengine.osgi.db/generated/user/jobengine/db/MediaDAO_SJProfile0.ser and b/server/user.jobengine.osgi.db/generated/user/jobengine/db/MediaDAO_SJProfile0.ser differ diff --git a/server/user.jobengine.osgi.db/generated/user/jobengine/db/MediaFileDAO.java b/server/user.jobengine.osgi.db/generated/user/jobengine/db/MediaFileDAO.java index fa1382df..39958450 100644 --- a/server/user.jobengine.osgi.db/generated/user/jobengine/db/MediaFileDAO.java +++ b/server/user.jobengine.osgi.db/generated/user/jobengine/db/MediaFileDAO.java @@ -5,8 +5,9 @@ import sqlj.runtime.ref.*; import java.sql.SQLException; import java.util.ArrayList; import java.util.List; +import java.sql.Timestamp; -/*@lineinfo:generated-code*//*@lineinfo:9^2*/ +/*@lineinfo:generated-code*//*@lineinfo:10^2*/ // ************************************************************ // SQLJ iterator declaration: @@ -16,6 +17,7 @@ class MediaFileIter extends sqlj.runtime.ref.ResultSetIterImpl implements sqlj.runtime.NamedIterator { + private int lastModifiedNdx; private int fileSizeNdx; private int houseIdNdx; private int fileStructInfoNdx; @@ -36,6 +38,7 @@ implements sqlj.runtime.NamedIterator fileStructInfoNdx = findColumn("fileStructInfo"); houseIdNdx = findColumn("houseId"); fileSizeNdx = findColumn("fileSize"); + lastModifiedNdx = findColumn("lastModified"); } public MediaFileIter(sqlj.runtime.profile.RTResultSet resultSet, int fetchSize, int maxRows) throws java.sql.SQLException @@ -49,6 +52,7 @@ implements sqlj.runtime.NamedIterator fileStructInfoNdx = findColumn("fileStructInfo"); houseIdNdx = findColumn("houseId"); fileSizeNdx = findColumn("fileSize"); + lastModifiedNdx = findColumn("lastModified"); } public long id() throws java.sql.SQLException @@ -90,12 +94,17 @@ implements sqlj.runtime.NamedIterator { return resultSet.getLongNoNull(fileSizeNdx); } + public Timestamp lastModified() + throws java.sql.SQLException + { + return resultSet.getTimestamp(lastModifiedNdx); + } } // ************************************************************ -/*@lineinfo:user-code*//*@lineinfo:9^155*/ +/*@lineinfo:user-code*//*@lineinfo:10^179*/ @SuppressWarnings("unused") public class MediaFileDAO extends EntityBaseDAO { @@ -118,6 +127,7 @@ public class MediaFileDAO extends EntityBaseDAO { entity.setFileStructInfo(iterator.fileStructInfo()); entity.setHouseId(iterator.houseId()); entity.setFileSize(iterator.fileSize()); + entity.setLastModified(iterator.lastModified()); if (result == null) result = new ArrayList(); @@ -130,10 +140,10 @@ public class MediaFileDAO extends EntityBaseDAO { @Override protected ResultSetIterImpl selectByKey(DefaultContext context, long id) throws SQLException{ MediaFileIter iter = null; - /*@lineinfo:generated-code*//*@lineinfo:44^2*/ + /*@lineinfo:generated-code*//*@lineinfo:46^2*/ // ************************************************************ -// #sql [context] iter = { SELECT ID, MEDIAID, STOREID, FILETYPEID, RELATIVEPATH, FILESTRUCTINFO, HOUSEID, FILESIZE FROM MEDIAFILE WHERE ID = :id }; +// #sql [context] iter = { SELECT ID, MEDIAID, STOREID, FILETYPEID, RELATIVEPATH, FILESTRUCTINFO, HOUSEID, FILESIZE, LASTMODIFIED FROM MEDIAFILE WHERE ID = :id }; // ************************************************************ { @@ -158,17 +168,17 @@ public class MediaFileDAO extends EntityBaseDAO { // ************************************************************ -/*@lineinfo:user-code*//*@lineinfo:44^145*/ +/*@lineinfo:user-code*//*@lineinfo:46^159*/ return iter; } @Override protected ResultSetIterImpl selectByForeignKey(DefaultContext context, long id) throws SQLException{ MediaFileIter iter = null; - /*@lineinfo:generated-code*//*@lineinfo:51^2*/ + /*@lineinfo:generated-code*//*@lineinfo:53^2*/ // ************************************************************ -// #sql [context] iter = { SELECT ID, MEDIAID, STOREID, FILETYPEID, RELATIVEPATH, FILESTRUCTINFO, HOUSEID, FILESIZE FROM MEDIAFILE WHERE MEDIAID = :id }; +// #sql [context] iter = { SELECT ID, MEDIAID, STOREID, FILETYPEID, RELATIVEPATH, FILESTRUCTINFO, HOUSEID, FILESIZE, LASTMODIFIED FROM MEDIAFILE WHERE MEDIAID = :id }; // ************************************************************ { @@ -193,17 +203,17 @@ public class MediaFileDAO extends EntityBaseDAO { // ************************************************************ -/*@lineinfo:user-code*//*@lineinfo:51^150*/ +/*@lineinfo:user-code*//*@lineinfo:53^164*/ return iter; } @Override protected ResultSetIterImpl selectAll(DefaultContext context) throws SQLException{ MediaFileIter iter = null; - /*@lineinfo:generated-code*//*@lineinfo:58^2*/ + /*@lineinfo:generated-code*//*@lineinfo:60^2*/ // ************************************************************ -// #sql [context] iter = { SELECT ID, MEDIAID, STOREID, FILETYPEID, RELATIVEPATH, FILESTRUCTINFO, HOUSEID, FILESIZE FROM MEDIAFILE }; +// #sql [context] iter = { SELECT ID, MEDIAID, STOREID, FILETYPEID, RELATIVEPATH, FILESTRUCTINFO, HOUSEID, FILESIZE, LASTMODIFIED FROM MEDIAFILE }; // ************************************************************ { @@ -227,13 +237,13 @@ public class MediaFileDAO extends EntityBaseDAO { // ************************************************************ -/*@lineinfo:user-code*//*@lineinfo:58^130*/ +/*@lineinfo:user-code*//*@lineinfo:60^144*/ return iter; } @Override protected void delete(DefaultContext context, long id) throws SQLException{ - /*@lineinfo:generated-code*//*@lineinfo:64^2*/ + /*@lineinfo:generated-code*//*@lineinfo:66^2*/ // ************************************************************ // #sql [context] { DELETE FROM MEDIAFILE WHERE ID = :id }; @@ -261,12 +271,12 @@ public class MediaFileDAO extends EntityBaseDAO { // ************************************************************ -/*@lineinfo:user-code*//*@lineinfo:64^56*/ +/*@lineinfo:user-code*//*@lineinfo:66^56*/ } @Override protected void truncateTable(DefaultContext context) throws SQLException{ - /*@lineinfo:generated-code*//*@lineinfo:69^2*/ + /*@lineinfo:generated-code*//*@lineinfo:71^2*/ // ************************************************************ // #sql [context] { TRUNCATE TABLE MEDIAFILE DROP STORAGE IGNORE DELETE TRIGGERS IMMEDIATE }; @@ -293,7 +303,7 @@ public class MediaFileDAO extends EntityBaseDAO { // ************************************************************ -/*@lineinfo:user-code*//*@lineinfo:69^90*/ +/*@lineinfo:user-code*//*@lineinfo:71^90*/ } @Override @@ -307,11 +317,11 @@ public class MediaFileDAO extends EntityBaseDAO { String fileStructInfo = obj.getFileStructInfo(); String houseId = obj.getHouseId(); long fileSize = obj.getFileSize(); - - /*@lineinfo:generated-code*//*@lineinfo:84^2*/ + Timestamp lastModified = obj.getLastModified(); + /*@lineinfo:generated-code*//*@lineinfo:86^2*/ // ************************************************************ -// #sql [context] { UPDATE MEDIAFILE SET MEDIAID = :mediaId, STOREID = :storeId, FILETYPEID = :fileTypeId, RELATIVEPATH = :relativePath, FILESTRUCTINFO = :fileStructInfo, HOUSEID = :houseId, FILESIZE = :fileSize WHERE ID = :id }; +// #sql [context] { UPDATE MEDIAFILE SET MEDIAID = :mediaId, STOREID = :storeId, FILETYPEID = :fileTypeId, RELATIVEPATH = :relativePath, FILESTRUCTINFO = :fileStructInfo, HOUSEID = :houseId, FILESIZE = :fileSize, LASTMODIFIED = :lastModified WHERE ID = :id }; // ************************************************************ { @@ -330,7 +340,8 @@ public class MediaFileDAO extends EntityBaseDAO { __sJT_stmt.setString(5, fileStructInfo); __sJT_stmt.setString(6, houseId); __sJT_stmt.setLong(7, fileSize); - __sJT_stmt.setLong(8, id); + __sJT_stmt.setTimestamp(8, lastModified); + __sJT_stmt.setLong(9, id); __sJT_execCtx.executeUpdate(); } finally @@ -343,7 +354,7 @@ public class MediaFileDAO extends EntityBaseDAO { // ************************************************************ -/*@lineinfo:user-code*//*@lineinfo:84^226*/ +/*@lineinfo:user-code*//*@lineinfo:86^256*/ } @Override @@ -356,11 +367,12 @@ public class MediaFileDAO extends EntityBaseDAO { String fileStructInfo = obj.getFileStructInfo(); String houseId = obj.getHouseId(); long fileSize = obj.getFileSize(); + Timestamp lastModified = obj.getLastModified(); - /*@lineinfo:generated-code*//*@lineinfo:98^2*/ + /*@lineinfo:generated-code*//*@lineinfo:101^2*/ // ************************************************************ -// #sql [context] { INSERT INTO MEDIAFILE (MEDIAID, STOREID, FILETYPEID, RELATIVEPATH, FILESTRUCTINFO, HOUSEID, FILESIZE) VALUES (:mediaId, :storeId, :fileTypeId, :relativePath, :fileStructInfo, :houseId, :fileSize) }; +// #sql [context] { INSERT INTO MEDIAFILE (MEDIAID, STOREID, FILETYPEID, RELATIVEPATH, FILESTRUCTINFO, HOUSEID, FILESIZE, LASTMODIFIED) VALUES (:mediaId, :storeId, :fileTypeId, :relativePath, :fileStructInfo, :houseId, :fileSize, :lastModified) }; // ************************************************************ { @@ -379,6 +391,7 @@ public class MediaFileDAO extends EntityBaseDAO { __sJT_stmt.setString(5, fileStructInfo); __sJT_stmt.setString(6, houseId); __sJT_stmt.setLong(7, fileSize); + __sJT_stmt.setTimestamp(8, lastModified); __sJT_execCtx.executeUpdate(); } finally @@ -391,7 +404,7 @@ public class MediaFileDAO extends EntityBaseDAO { // ************************************************************ -/*@lineinfo:user-code*//*@lineinfo:98^215*/ +/*@lineinfo:user-code*//*@lineinfo:101^244*/ } void addAll(DefaultContext context, IEntityBase entity) throws SQLException { @@ -412,7 +425,7 @@ public class MediaFileDAO extends EntityBaseDAO { void removeAll(DefaultContext context, long filterId) throws SQLException { manager.traceIn(); - /*@lineinfo:generated-code*//*@lineinfo:119^2*/ + /*@lineinfo:generated-code*//*@lineinfo:122^2*/ // ************************************************************ // #sql [context] { DELETE FROM MEDIAFILE WHERE MEDIAID = :filterId }; @@ -440,16 +453,16 @@ public class MediaFileDAO extends EntityBaseDAO { // ************************************************************ -/*@lineinfo:user-code*//*@lineinfo:119^67*/ +/*@lineinfo:user-code*//*@lineinfo:122^67*/ manager.traceOut(); } private MediaFileIter selectByHouseId(DefaultContext context, String houseId) throws SQLException{ MediaFileIter iter = null; - /*@lineinfo:generated-code*//*@lineinfo:125^2*/ + /*@lineinfo:generated-code*//*@lineinfo:128^2*/ // ************************************************************ -// #sql [context] iter = { SELECT ID, MEDIAID, STOREID, FILETYPEID, RELATIVEPATH, FILESTRUCTINFO, HOUSEID, FILESIZE FROM MEDIAFILE WHERE HOUSEID = :houseId }; +// #sql [context] iter = { SELECT ID, MEDIAID, STOREID, FILETYPEID, RELATIVEPATH, FILESTRUCTINFO, HOUSEID, FILESIZE, LASTMODIFIED FROM MEDIAFILE WHERE HOUSEID = :houseId }; // ************************************************************ { @@ -474,7 +487,7 @@ public class MediaFileDAO extends EntityBaseDAO { // ************************************************************ -/*@lineinfo:user-code*//*@lineinfo:125^155*/ +/*@lineinfo:user-code*//*@lineinfo:128^169*/ return iter; } diff --git a/server/user.jobengine.osgi.db/generated/user/jobengine/db/MediaFileDAO_SJProfile0.ser b/server/user.jobengine.osgi.db/generated/user/jobengine/db/MediaFileDAO_SJProfile0.ser index 2c428e03..bb26aa2a 100644 Binary files a/server/user.jobengine.osgi.db/generated/user/jobengine/db/MediaFileDAO_SJProfile0.ser and b/server/user.jobengine.osgi.db/generated/user/jobengine/db/MediaFileDAO_SJProfile0.ser differ diff --git a/server/user.jobengine.osgi.db/generated/user/jobengine/db/MetadataDAO_SJProfile0.ser b/server/user.jobengine.osgi.db/generated/user/jobengine/db/MetadataDAO_SJProfile0.ser index 64adbf78..b816912b 100644 Binary files a/server/user.jobengine.osgi.db/generated/user/jobengine/db/MetadataDAO_SJProfile0.ser and b/server/user.jobengine.osgi.db/generated/user/jobengine/db/MetadataDAO_SJProfile0.ser differ diff --git a/server/user.jobengine.osgi.db/generated/user/jobengine/db/MetadataElementDAO_SJProfile0.ser b/server/user.jobengine.osgi.db/generated/user/jobengine/db/MetadataElementDAO_SJProfile0.ser index 9e807e6c..7e4128fb 100644 Binary files a/server/user.jobengine.osgi.db/generated/user/jobengine/db/MetadataElementDAO_SJProfile0.ser and b/server/user.jobengine.osgi.db/generated/user/jobengine/db/MetadataElementDAO_SJProfile0.ser differ diff --git a/server/user.jobengine.osgi.db/generated/user/jobengine/db/MetadataTypeDAO_SJProfile0.ser b/server/user.jobengine.osgi.db/generated/user/jobengine/db/MetadataTypeDAO_SJProfile0.ser index a23e1a23..8d5c95ca 100644 Binary files a/server/user.jobengine.osgi.db/generated/user/jobengine/db/MetadataTypeDAO_SJProfile0.ser and b/server/user.jobengine.osgi.db/generated/user/jobengine/db/MetadataTypeDAO_SJProfile0.ser differ diff --git a/server/user.jobengine.osgi.db/generated/user/jobengine/db/RemoteStoreDAO_SJProfile0.ser b/server/user.jobengine.osgi.db/generated/user/jobengine/db/RemoteStoreDAO_SJProfile0.ser index 3afde202..34f04a30 100644 Binary files a/server/user.jobengine.osgi.db/generated/user/jobengine/db/RemoteStoreDAO_SJProfile0.ser and b/server/user.jobengine.osgi.db/generated/user/jobengine/db/RemoteStoreDAO_SJProfile0.ser differ diff --git a/server/user.jobengine.osgi.db/generated/user/jobengine/db/SceneContentDAO_SJProfile0.ser b/server/user.jobengine.osgi.db/generated/user/jobengine/db/SceneContentDAO_SJProfile0.ser index 66e8bc16..1aa06b2b 100644 Binary files a/server/user.jobengine.osgi.db/generated/user/jobengine/db/SceneContentDAO_SJProfile0.ser and b/server/user.jobengine.osgi.db/generated/user/jobengine/db/SceneContentDAO_SJProfile0.ser differ diff --git a/server/user.jobengine.osgi.db/generated/user/jobengine/db/SceneDAO_SJProfile0.ser b/server/user.jobengine.osgi.db/generated/user/jobengine/db/SceneDAO_SJProfile0.ser index 84649387..799e66b5 100644 Binary files a/server/user.jobengine.osgi.db/generated/user/jobengine/db/SceneDAO_SJProfile0.ser and b/server/user.jobengine.osgi.db/generated/user/jobengine/db/SceneDAO_SJProfile0.ser differ diff --git a/server/user.jobengine.osgi.db/generated/user/jobengine/db/SearchDefinitionDAO_SJProfile0.ser b/server/user.jobengine.osgi.db/generated/user/jobengine/db/SearchDefinitionDAO_SJProfile0.ser index 45c69c4c..80dedcfe 100644 Binary files a/server/user.jobengine.osgi.db/generated/user/jobengine/db/SearchDefinitionDAO_SJProfile0.ser and b/server/user.jobengine.osgi.db/generated/user/jobengine/db/SearchDefinitionDAO_SJProfile0.ser differ diff --git a/server/user.jobengine.osgi.db/generated/user/jobengine/db/ShotDAO_SJProfile0.ser b/server/user.jobengine.osgi.db/generated/user/jobengine/db/ShotDAO_SJProfile0.ser index fcd3288c..0dc29210 100644 Binary files a/server/user.jobengine.osgi.db/generated/user/jobengine/db/ShotDAO_SJProfile0.ser and b/server/user.jobengine.osgi.db/generated/user/jobengine/db/ShotDAO_SJProfile0.ser differ diff --git a/server/user.jobengine.osgi.db/generated/user/jobengine/db/StoreDAO_SJProfile0.ser b/server/user.jobengine.osgi.db/generated/user/jobengine/db/StoreDAO_SJProfile0.ser index a1bc7f9d..73f6f721 100644 Binary files a/server/user.jobengine.osgi.db/generated/user/jobengine/db/StoreDAO_SJProfile0.ser and b/server/user.jobengine.osgi.db/generated/user/jobengine/db/StoreDAO_SJProfile0.ser differ diff --git a/server/user.jobengine.osgi.db/generated/user/jobengine/db/StoreUriDAO_SJProfile0.ser b/server/user.jobengine.osgi.db/generated/user/jobengine/db/StoreUriDAO_SJProfile0.ser index 8b02ce0e..6bfd563e 100644 Binary files a/server/user.jobengine.osgi.db/generated/user/jobengine/db/StoreUriDAO_SJProfile0.ser and b/server/user.jobengine.osgi.db/generated/user/jobengine/db/StoreUriDAO_SJProfile0.ser differ diff --git a/server/user.jobengine.osgi.db/generated/user/jobengine/db/UserInfoDAO_SJProfile0.ser b/server/user.jobengine.osgi.db/generated/user/jobengine/db/UserInfoDAO_SJProfile0.ser index e41595b0..636dfe5f 100644 Binary files a/server/user.jobengine.osgi.db/generated/user/jobengine/db/UserInfoDAO_SJProfile0.ser and b/server/user.jobengine.osgi.db/generated/user/jobengine/db/UserInfoDAO_SJProfile0.ser differ diff --git a/server/user.jobengine.osgi.db/generated/user/jobengine/db/WorkflowActionDAO_SJProfile0.ser b/server/user.jobengine.osgi.db/generated/user/jobengine/db/WorkflowActionDAO_SJProfile0.ser index 5f84c81e..d92f4d82 100644 Binary files a/server/user.jobengine.osgi.db/generated/user/jobengine/db/WorkflowActionDAO_SJProfile0.ser and b/server/user.jobengine.osgi.db/generated/user/jobengine/db/WorkflowActionDAO_SJProfile0.ser differ diff --git a/server/user.jobengine.osgi.db/migrations/scripts/011_add_lastmodified_to_mediafile.sql b/server/user.jobengine.osgi.db/migrations/scripts/011_add_lastmodified_to_mediafile.sql new file mode 100644 index 00000000..f7906c11 --- /dev/null +++ b/server/user.jobengine.osgi.db/migrations/scripts/011_add_lastmodified_to_mediafile.sql @@ -0,0 +1,32 @@ +-- +-- Copyright 2010-2016 the original author or authors. +-- +-- Licensed under the Apache License, Version 2.0 (the "License"); +-- you may not use this file except in compliance with the License. +-- You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, software +-- distributed under the License is distributed on an "AS IS" BASIS, +-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +-- See the License for the specific language governing permissions and +-- limitations under the License. +-- + +-- // Alter MEDIAFILE table, add LASTMODIFIED column +-- Migration SQL that makes the change goes here. + +ALTER TABLE MEDIAFILE ADD COLUMN LASTMODIFIED TIMESTAMP DEFAULT CURRENT_TIMESTAMP NOT NULL +@ + +CALL SYSPROC.ADMIN_CMD('REORG TABLE DB2ADMIN.MEDIAFILE') +@ +-- //@UNDO +-- SQL to undo the change goes here. + +ALTER TABLE MEDIAFILE DROP COLUMN LASTMODIFIED +@ + +CALL SYSPROC.ADMIN_CMD('REORG TABLE DB2ADMIN.MEDIAFILE') +@ diff --git a/server/user.jobengine.osgi.db/src/user/jobengine/db/IItemManager.java b/server/user.jobengine.osgi.db/src/user/jobengine/db/IItemManager.java index 18ac3b28..ee909a1a 100644 --- a/server/user.jobengine.osgi.db/src/user/jobengine/db/IItemManager.java +++ b/server/user.jobengine.osgi.db/src/user/jobengine/db/IItemManager.java @@ -316,6 +316,8 @@ public interface IItemManager extends IEntityPersister { StoreUri getStoreUri(String targetStoreName, RemoteStoreProtocol local) throws ItemManagerException; + MediaFile getSystemMediaFile(Media media); + /** * Elkér a StoreDAO-tól, egy olyan Rendszer tárolót, amely a lowres paraméterben kapott érték alapján vagy kisfelbontású videók tárolója, vagy nem. * diff --git a/server/user.jobengine.osgi.db/src/user/jobengine/db/ItemManager.java b/server/user.jobengine.osgi.db/src/user/jobengine/db/ItemManager.java index 2ee4361b..709444e1 100644 --- a/server/user.jobengine.osgi.db/src/user/jobengine/db/ItemManager.java +++ b/server/user.jobengine.osgi.db/src/user/jobengine/db/ItemManager.java @@ -979,6 +979,22 @@ public class ItemManager extends MemoryCache implements IItemManager { return storeUri; } + @Override + public MediaFile getSystemMediaFile(Media media) { + MediaFile result = null; + List mediaFiles = media.getMediaFiles(); + if (mediaFiles == null || mediaFiles.size() == 0) + return null; + + Store store = getSystemStore(false); + for (MediaFile mediaFile : mediaFiles) { + if (mediaFile.getStoreId() == store.getId()) + return mediaFile; + } + + return result; + } + @Override public Store getSystemStore(boolean lowres) { Store result = null; diff --git a/server/user.jobengine.osgi.db/src/user/jobengine/db/Media.java b/server/user.jobengine.osgi.db/src/user/jobengine/db/Media.java index a88400cb..c638f706 100644 --- a/server/user.jobengine.osgi.db/src/user/jobengine/db/Media.java +++ b/server/user.jobengine.osgi.db/src/user/jobengine/db/Media.java @@ -183,4 +183,5 @@ public class Media extends DynamicAttributes { checkParameter("Title", title); this.title = title; } + } \ No newline at end of file diff --git a/server/user.jobengine.osgi.db/src/user/jobengine/db/MediaFile.java b/server/user.jobengine.osgi.db/src/user/jobengine/db/MediaFile.java index 11b373f0..d8a977d4 100644 --- a/server/user.jobengine.osgi.db/src/user/jobengine/db/MediaFile.java +++ b/server/user.jobengine.osgi.db/src/user/jobengine/db/MediaFile.java @@ -1,6 +1,7 @@ package user.jobengine.db; import java.io.Serializable; +import java.sql.Timestamp; import user.commons.EntityBase; @@ -17,6 +18,11 @@ public class MediaFile extends EntityBase implements Serializable { private long storeId = 0; private long fileSize = 0; private String houseId = null; + private Timestamp lastModified; + + public long getFileSize() { + return fileSize; + } public String getFileStructInfo() { return this.fileStructInfo; @@ -36,6 +42,10 @@ public class MediaFile extends EntityBase implements Serializable { return houseId; } + public Timestamp getLastModified() { + return lastModified; + } + public Media getMedia() { if (this.media == null && this.mediaId != 0) { this.media = ItemManager.getInstance().getMedia(this.mediaId); @@ -63,6 +73,10 @@ public class MediaFile extends EntityBase implements Serializable { return this.storeId; } + public void setFileSize(long fileSize) { + this.fileSize = fileSize; + } + public void setFileStructInfo(String fileStructInfo) { this.fileStructInfo = fileStructInfo; } @@ -82,6 +96,10 @@ public class MediaFile extends EntityBase implements Serializable { this.houseId = houseId; } + public void setLastModified(Timestamp lastModified) { + this.lastModified = lastModified; + } + public void setMedia(Media media) { //checkParameter("Media", media); this.media = media; @@ -109,12 +127,4 @@ public class MediaFile extends EntityBase implements Serializable { this.storeId = id; } - public long getFileSize() { - return fileSize; - } - - public void setFileSize(long fileSize) { - this.fileSize = fileSize; - } - } \ No newline at end of file diff --git a/server/user.jobengine.osgi.db/src/user/jobengine/db/MediaFileDAO.sqlj b/server/user.jobengine.osgi.db/src/user/jobengine/db/MediaFileDAO.sqlj index 1d5f3c15..697c69df 100644 --- a/server/user.jobengine.osgi.db/src/user/jobengine/db/MediaFileDAO.sqlj +++ b/server/user.jobengine.osgi.db/src/user/jobengine/db/MediaFileDAO.sqlj @@ -5,8 +5,9 @@ import sqlj.runtime.ref.*; import java.sql.SQLException; import java.util.ArrayList; import java.util.List; +import java.sql.Timestamp; -#sql iterator MediaFileIter(long id, long mediaId, long storeId, long fileTypeId, String relativePath, String fileStructInfo, String houseId, long fileSize); +#sql iterator MediaFileIter(long id, long mediaId, long storeId, long fileTypeId, String relativePath, String fileStructInfo, String houseId, long fileSize, Timestamp lastModified); @SuppressWarnings("unused") public class MediaFileDAO extends EntityBaseDAO { @@ -29,6 +30,7 @@ public class MediaFileDAO extends EntityBaseDAO { entity.setFileStructInfo(iterator.fileStructInfo()); entity.setHouseId(iterator.houseId()); entity.setFileSize(iterator.fileSize()); + entity.setLastModified(iterator.lastModified()); if (result == null) result = new ArrayList(); @@ -41,21 +43,21 @@ public class MediaFileDAO extends EntityBaseDAO { @Override protected ResultSetIterImpl selectByKey(DefaultContext context, long id) throws SQLException{ MediaFileIter iter = null; - #sql [context] iter = { SELECT ID, MEDIAID, STOREID, FILETYPEID, RELATIVEPATH, FILESTRUCTINFO, HOUSEID, FILESIZE FROM MEDIAFILE WHERE ID = :id }; + #sql [context] iter = { SELECT ID, MEDIAID, STOREID, FILETYPEID, RELATIVEPATH, FILESTRUCTINFO, HOUSEID, FILESIZE, LASTMODIFIED FROM MEDIAFILE WHERE ID = :id }; return iter; } @Override protected ResultSetIterImpl selectByForeignKey(DefaultContext context, long id) throws SQLException{ MediaFileIter iter = null; - #sql [context] iter = { SELECT ID, MEDIAID, STOREID, FILETYPEID, RELATIVEPATH, FILESTRUCTINFO, HOUSEID, FILESIZE FROM MEDIAFILE WHERE MEDIAID = :id }; + #sql [context] iter = { SELECT ID, MEDIAID, STOREID, FILETYPEID, RELATIVEPATH, FILESTRUCTINFO, HOUSEID, FILESIZE, LASTMODIFIED FROM MEDIAFILE WHERE MEDIAID = :id }; return iter; } @Override protected ResultSetIterImpl selectAll(DefaultContext context) throws SQLException{ MediaFileIter iter = null; - #sql [context] iter = { SELECT ID, MEDIAID, STOREID, FILETYPEID, RELATIVEPATH, FILESTRUCTINFO, HOUSEID, FILESIZE FROM MEDIAFILE }; + #sql [context] iter = { SELECT ID, MEDIAID, STOREID, FILETYPEID, RELATIVEPATH, FILESTRUCTINFO, HOUSEID, FILESIZE, LASTMODIFIED FROM MEDIAFILE }; return iter; } @@ -80,8 +82,8 @@ public class MediaFileDAO extends EntityBaseDAO { String fileStructInfo = obj.getFileStructInfo(); String houseId = obj.getHouseId(); long fileSize = obj.getFileSize(); - - #sql [context] { UPDATE MEDIAFILE SET MEDIAID = :mediaId, STOREID = :storeId, FILETYPEID = :fileTypeId, RELATIVEPATH = :relativePath, FILESTRUCTINFO = :fileStructInfo, HOUSEID = :houseId, FILESIZE = :fileSize WHERE ID = :id }; + Timestamp lastModified = obj.getLastModified(); + #sql [context] { UPDATE MEDIAFILE SET MEDIAID = :mediaId, STOREID = :storeId, FILETYPEID = :fileTypeId, RELATIVEPATH = :relativePath, FILESTRUCTINFO = :fileStructInfo, HOUSEID = :houseId, FILESIZE = :fileSize, LASTMODIFIED = :lastModified WHERE ID = :id }; } @Override @@ -94,8 +96,9 @@ public class MediaFileDAO extends EntityBaseDAO { String fileStructInfo = obj.getFileStructInfo(); String houseId = obj.getHouseId(); long fileSize = obj.getFileSize(); + Timestamp lastModified = obj.getLastModified(); - #sql [context] { INSERT INTO MEDIAFILE (MEDIAID, STOREID, FILETYPEID, RELATIVEPATH, FILESTRUCTINFO, HOUSEID, FILESIZE) VALUES (:mediaId, :storeId, :fileTypeId, :relativePath, :fileStructInfo, :houseId, :fileSize) }; + #sql [context] { INSERT INTO MEDIAFILE (MEDIAID, STOREID, FILETYPEID, RELATIVEPATH, FILESTRUCTINFO, HOUSEID, FILESIZE, LASTMODIFIED) VALUES (:mediaId, :storeId, :fileTypeId, :relativePath, :fileStructInfo, :houseId, :fileSize, :lastModified) }; } void addAll(DefaultContext context, IEntityBase entity) throws SQLException { @@ -122,7 +125,7 @@ public class MediaFileDAO extends EntityBaseDAO { private MediaFileIter selectByHouseId(DefaultContext context, String houseId) throws SQLException{ MediaFileIter iter = null; - #sql [context] iter = { SELECT ID, MEDIAID, STOREID, FILETYPEID, RELATIVEPATH, FILESTRUCTINFO, HOUSEID, FILESIZE FROM MEDIAFILE WHERE HOUSEID = :houseId }; + #sql [context] iter = { SELECT ID, MEDIAID, STOREID, FILETYPEID, RELATIVEPATH, FILESTRUCTINFO, HOUSEID, FILESIZE, LASTMODIFIED FROM MEDIAFILE WHERE HOUSEID = :houseId }; return iter; } diff --git a/server/user.jobengine.osgi.db/src/user/jobengine/search/MediaFinder.java b/server/user.jobengine.osgi.db/src/user/jobengine/search/MediaFinder.java index 3152ba10..1f7bf393 100644 --- a/server/user.jobengine.osgi.db/src/user/jobengine/search/MediaFinder.java +++ b/server/user.jobengine.osgi.db/src/user/jobengine/search/MediaFinder.java @@ -53,8 +53,16 @@ public class MediaFinder implements IMediaFinder { sql.append(typeCriteria.toString()); } + // Ha csak LIKE van media.title-re, nem kell (AMC) + // String criteria = formatSearchValue(criteria); + + // if (criteria != null && criteria.trim().length() > 0) + // sql.append(" AND CONTAINS(md.description, '" + criteria + "') >= 1"); + + if (criteria.endsWith("*")) + criteria = criteria.replace("*", ""); if (criteria != null && criteria.trim().length() > 0) - sql.append(" AND CONTAINS(md.description, '" + criteria + "') >= 1"); + sql.append(" AND m.title LIKE('%" + criteria + "%')"); if (options.getSearchFrom() != null) { sql.append(String.format(" AND m.archived > '%s'", df.format(options.getSearchFrom()))); @@ -151,8 +159,7 @@ public class MediaFinder implements IMediaFinder { DefaultContext context = manager.getDbContext(); Connection connection = context.getConnection(); try { - String text = formatSearchValue(options.getText()); - st = createStatement(connection, text, typeIDs, options); + st = createStatement(connection, options.getText(), typeIDs, options); rs = st.executeQuery(); rs.last(); result.setItemCount(rs.getRow()); diff --git a/server/user.jobengine.osgi.server/pages/joblist.zul b/server/user.jobengine.osgi.server/pages/joblist.zul index d5f9b2ec..0f1b6aa6 100644 --- a/server/user.jobengine.osgi.server/pages/joblist.zul +++ b/server/user.jobengine.osgi.server/pages/joblist.zul @@ -15,7 +15,7 @@ - + @@ -43,7 +43,7 @@ - + diff --git a/server/user.jobengine.osgi.server/pages/menu.zul b/server/user.jobengine.osgi.server/pages/menu.zul index 0f09ca02..a25c2674 100644 --- a/server/user.jobengine.osgi.server/pages/menu.zul +++ b/server/user.jobengine.osgi.server/pages/menu.zul @@ -55,8 +55,9 @@ -

+ + diff --git a/server/user.jobengine.osgi.server/src/user/jobengine/server/IJobEngine.java b/server/user.jobengine.osgi.server/src/user/jobengine/server/IJobEngine.java index ffdd6e5f..eb9e8f6e 100644 --- a/server/user.jobengine.osgi.server/src/user/jobengine/server/IJobEngine.java +++ b/server/user.jobengine.osgi.server/src/user/jobengine/server/IJobEngine.java @@ -118,7 +118,7 @@ public interface IJobEngine { void removeSpanwChild(IJobRuntime jobRuntime); - void removeSuspended(); + void removeGarbage(); ClusteredJob requestJob(String className) throws Exception; diff --git a/server/user.jobengine.osgi.server/src/user/jobengine/server/IJobRuntime.java b/server/user.jobengine.osgi.server/src/user/jobengine/server/IJobRuntime.java index 7d6f935f..d58c0c8b 100644 --- a/server/user.jobengine.osgi.server/src/user/jobengine/server/IJobRuntime.java +++ b/server/user.jobengine.osgi.server/src/user/jobengine/server/IJobRuntime.java @@ -52,7 +52,7 @@ public interface IJobRuntime extends IJob { IJobEngine getJobEngine(); - Marker getMarker(); + Marker getSessionMarker(); IInstruction getNextInstruction() throws NoSuchElementException; diff --git a/server/user.jobengine.osgi.server/src/user/jobengine/server/JobEngine.java b/server/user.jobengine.osgi.server/src/user/jobengine/server/JobEngine.java index 2f6f3fac..fdb40335 100644 --- a/server/user.jobengine.osgi.server/src/user/jobengine/server/JobEngine.java +++ b/server/user.jobengine.osgi.server/src/user/jobengine/server/JobEngine.java @@ -279,7 +279,7 @@ public class JobEngine implements IJobEngine { try { logger.debug("{} adding to run queue", jobRuntime); if (jobRuntime.getIp() == 0 && !jobRuntime.isService() && jobRuntime.getParentJobId() == 0) - logger.info(jobRuntime.getMarker(), "A '{}' folyamat elindult.", jobRuntime.getName()); + logger.info(jobRuntime.getSessionMarker(), "A '{}' folyamat elindult.", jobRuntime.getName()); runQueue.put(jobRuntime); } catch (Exception e) { logger.error(e.getMessage(), e); @@ -878,10 +878,11 @@ public class JobEngine implements IJobEngine { } @Override - public void removeSuspended() { + public void removeGarbage() { List removeId = new ArrayList<>(); for (Long id : submittedJobs.keySet()) { - if (submittedJobs.get(id).getStatus() == JobStatus.SUSPENDED) + IJobRuntime runtime = submittedJobs.get(id); + if (runtime != null && (JobStatus.SUSPENDED.equals(runtime.getStatus()) || JobStatus.CANCELED.equals(runtime.getStatus()))) removeId.add(id); } for (Long id : removeId) @@ -990,7 +991,7 @@ public class JobEngine implements IJobEngine { @Override public void startup() { try { - removeSuspended(); + removeGarbage(); loadPrograms(); loadExecutors(); diff --git a/server/user.jobengine.osgi.server/src/user/jobengine/server/JobRuntime.java b/server/user.jobengine.osgi.server/src/user/jobengine/server/JobRuntime.java index b0cb919c..d887cc80 100644 --- a/server/user.jobengine.osgi.server/src/user/jobengine/server/JobRuntime.java +++ b/server/user.jobengine.osgi.server/src/user/jobengine/server/JobRuntime.java @@ -22,6 +22,7 @@ import org.apache.commons.lang.StringUtils; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.apache.logging.log4j.Marker; +import org.apache.logging.log4j.MarkerManager; import org.apache.logging.log4j.message.Message; import user.commons.IEntityPersister; @@ -416,7 +417,7 @@ public class JobRuntime extends Job implements IJobRuntime { @Override public Marker getFinishMarker() { if (finishMarker == null) - finishMarker = new MediaCubeFinishMarker((MediaCubeMarker) getMarker()); + finishMarker = new MediaCubeFinishMarker((MediaCubeMarker) getSessionMarker()); return finishMarker; } @@ -430,21 +431,6 @@ public class JobRuntime extends Job implements IJobRuntime { return jobEngine; } - /*** - * Log session marker. A teljes folyamat osszes naplobejegyzese osszegyujtheto a segitsegevel. MediaCubeMarker tipusu, folyamatonkent uj peldany jon letre. - */ - @Override - public Marker getMarker() { - if (sessionMarker == null) { - // Marker parentMarker = MarkerManager.getMarker(template); - // sessionMarker.setParents(parentMarker); - sessionMarker = new MediaCubeMarker(); - sessionMarker.setSessionID(getParentJobId() == 0 ? String.valueOf(getId()) : String.valueOf(getParentJobId())); - sessionMarker.setSessionName(name); - } - return sessionMarker; - } - @Override public IInstruction getNextInstruction() throws NoSuchElementException { if (!hasNextInstruction()) @@ -486,6 +472,22 @@ public class JobRuntime extends Job implements IJobRuntime { return savedStatus; } + /*** + * Log session marker. A teljes folyamat osszes naplobejegyzese osszegyujtheto a segitsegevel. MediaCubeMarker tipusu, folyamatonkent uj peldany jon letre. + */ + @Override + public Marker getSessionMarker() { + if (sessionMarker == null) { + sessionMarker = new MediaCubeMarker(); + sessionMarker.setSessionID(getParentJobId() == 0 ? String.valueOf(getId()) : String.valueOf(getParentJobId())); + sessionMarker.setSessionName(name); + sessionMarker.setUseSessionLog(getProgram().getTemplate().isUseSessionLog()); + Marker parentMarker = MarkerManager.getMarker(template); + sessionMarker.setParents(parentMarker); + } + return sessionMarker; + } + @Override public int getSpawnOrder() { return spawnOrder; diff --git a/server/user.jobengine.osgi.server/src/user/jobengine/server/JobStepExecutor.java b/server/user.jobengine.osgi.server/src/user/jobengine/server/JobStepExecutor.java index 1045305e..7f5e385c 100644 --- a/server/user.jobengine.osgi.server/src/user/jobengine/server/JobStepExecutor.java +++ b/server/user.jobengine.osgi.server/src/user/jobengine/server/JobStepExecutor.java @@ -130,7 +130,9 @@ public class JobStepExecutor implements IJobStepExecutor { Object[] outputs = step.run(jobEngine, jobRuntime, inputs); //TODO itt lekezelni a remote notification-t - jobEngine.sendMessage(new JobStepCompletedMessage(jobRuntime.getId(), outputs)); + if (!jobRuntime.isWaitingCancel()) { + jobEngine.sendMessage(new JobStepCompletedMessage(jobRuntime.getId(), outputs)); + } } public void shutdown() { diff --git a/server/user.jobengine.osgi.server/src/user/jobengine/server/ast/JobTemplate.java b/server/user.jobengine.osgi.server/src/user/jobengine/server/ast/JobTemplate.java index d1f85504..3ceb5b4d 100644 --- a/server/user.jobengine.osgi.server/src/user/jobengine/server/ast/JobTemplate.java +++ b/server/user.jobengine.osgi.server/src/user/jobengine/server/ast/JobTemplate.java @@ -14,6 +14,7 @@ public class JobTemplate extends AST { private DeclarationSequence declarationSequence = null; private String fileName = null; private String name = null; + private boolean useSessionLog = true; private String description = null; private boolean multiInstance; private boolean service; @@ -54,6 +55,10 @@ public class JobTemplate extends AST { return service; } + public boolean isUseSessionLog() { + return useSessionLog; + } + public void setCommandSequence(CommandSequence commandSequence) { this.commandSequence = commandSequence; } @@ -82,6 +87,10 @@ public class JobTemplate extends AST { this.service = service; } + public void setUseSessionLog(boolean useSessionLog) { + this.useSessionLog = useSessionLog; + } + //TODO foreach ellenorzes public List validate() throws Exception { Map parameters = new HashMap<>(); diff --git a/server/user.jobengine.osgi.server/src/user/jobengine/server/steps/JobStep.java b/server/user.jobengine.osgi.server/src/user/jobengine/server/steps/JobStep.java index b3f8980f..42de2048 100644 --- a/server/user.jobengine.osgi.server/src/user/jobengine/server/steps/JobStep.java +++ b/server/user.jobengine.osgi.server/src/user/jobengine/server/steps/JobStep.java @@ -107,7 +107,7 @@ public class JobStep implements IJobStep { * @return */ protected Marker getMarker() { - Marker parentMarker = MarkerManager.getMarker("MEDIACUBE"); + Marker parentMarker = MarkerManager.getMarker("MEDIACUBE FILELOG"); Marker result = MarkerManager.getMarker(getRuntimeName()); result.setParents(parentMarker); return result; @@ -140,7 +140,7 @@ public class JobStep implements IJobStep { * @return */ protected MediaCubeMarker getSessionMarker() { - return (MediaCubeMarker) jobRuntime.getMarker(); + return (MediaCubeMarker) jobRuntime.getSessionMarker(); } @Override diff --git a/server/user.jobengine.osgi.server/src/user/jobengine/zk/model/AlternateRetrieveBatchSelectorModel.java b/server/user.jobengine.osgi.server/src/user/jobengine/zk/model/AlternateRetrieveBatchSelectorModel.java index c98b395b..b7591864 100644 --- a/server/user.jobengine.osgi.server/src/user/jobengine/zk/model/AlternateRetrieveBatchSelectorModel.java +++ b/server/user.jobengine.osgi.server/src/user/jobengine/zk/model/AlternateRetrieveBatchSelectorModel.java @@ -116,8 +116,19 @@ public class AlternateRetrieveBatchSelectorModel extends BaseModel { public void init() { setEmail(SessionUtil.getUserPrincipal().getEmail()); List targetStores = ItemManager.getInstance().getTargetStores(); - if (targetStores != null) - getStores().addAll(targetStores); + if (targetStores != null) { + + List filters = SessionUtil.getMediaCubeConfig().getTargetRestoreFilters(); + if (filters == null || filters.size() == 0) { + stores.addAll(targetStores); + } else { + for (Store store : targetStores) { + if (filters.contains(store.getName())) + stores.add(store); + } + } + + } } public boolean isDisabled() { diff --git a/server/user.jobengine.osgi.server/src/user/jobengine/zk/model/JobListModel.java b/server/user.jobengine.osgi.server/src/user/jobengine/zk/model/JobListModel.java index 2d29bdf6..e2fe4a41 100644 --- a/server/user.jobengine.osgi.server/src/user/jobengine/zk/model/JobListModel.java +++ b/server/user.jobengine.osgi.server/src/user/jobengine/zk/model/JobListModel.java @@ -54,7 +54,7 @@ public class JobListModel extends AsyncBaseModel implements IJobChangedListener return; try { for (IJob job : jobList) { - if (job instanceof IJobRuntime) + if (job instanceof IJobRuntime && ((IJobRuntime) job).isCancelable()) jobEngine.sendMessage(new CancelRequest(job.getId())); } } catch (Exception e) { @@ -91,20 +91,11 @@ public class JobListModel extends AsyncBaseModel implements IJobChangedListener @Command public void cleanupSuspended() { - jobEngine.removeSuspended(); + jobEngine.removeGarbage(); initializeList(); } - private int compare(IJobRuntime o1, IJobRuntime o2) { - // return o1.getStatus().compareTo(o2.getStatus()); - int ret = (o2.getPriority() - o1.getPriority()); - if ((ret == 0) && (o1.getSubmitted() != null) && (o2.getSubmitted() != null)) { - ret = (int) (o2.getSubmitted().getTime() - o1.getSubmitted().getTime()); - } - return ret; - } - @Command public void executeJob() { String template = "/pages/jobselector.zul"; diff --git a/server/user.jobengine.osgi.server/src/user/jobengine/zk/model/MediaCubeConfig.java b/server/user.jobengine.osgi.server/src/user/jobengine/zk/model/MediaCubeConfig.java index c8e5c110..d75bf3c9 100644 --- a/server/user.jobengine.osgi.server/src/user/jobengine/zk/model/MediaCubeConfig.java +++ b/server/user.jobengine.osgi.server/src/user/jobengine/zk/model/MediaCubeConfig.java @@ -7,6 +7,7 @@ public class MediaCubeConfig { private MediaCubeAuthentication authentication; private List topTypeFilters; private List bottomTypeFilters; + private List targetRestoreFilters; public MediaCubeAuthentication getAuthentication() { return authentication; @@ -20,6 +21,10 @@ public class MediaCubeConfig { return jobQueuePollInterval; } + public List getTargetRestoreFilters() { + return targetRestoreFilters; + } + public List getTopTypeFilters() { return topTypeFilters; } @@ -36,6 +41,10 @@ public class MediaCubeConfig { this.jobQueuePollInterval = jobQueuePollInterval; } + public void setTargetRestoreFilters(List targetRestoreFilters) { + this.targetRestoreFilters = targetRestoreFilters; + } + public void setTopTypeFilters(List topTypeFilters) { this.topTypeFilters = topTypeFilters; } diff --git a/server/user.jobengine.osgi.server/src/user/jobengine/zk/model/MenuModel.java b/server/user.jobengine.osgi.server/src/user/jobengine/zk/model/MenuModel.java index 59172e69..5c1e8130 100644 --- a/server/user.jobengine.osgi.server/src/user/jobengine/zk/model/MenuModel.java +++ b/server/user.jobengine.osgi.server/src/user/jobengine/zk/model/MenuModel.java @@ -94,7 +94,7 @@ public class MenuModel extends BaseModel { if (select) { if (!basketItems.contains(item)) { basketItems.add(item); - Clients.showNotification(String.format("Hozzáadva: %s", item.getMedia().getMediaFilesName()), "info", basketIMenuItem, "start_center", 2000); + Clients.showNotification(String.format("Hozzáadva: %s", item.getMedia().getMediaFileRealName()), "info", basketIMenuItem, "start_center", 2000); } } else { if (basketItems.contains(item)) diff --git a/server/user.jobengine.osgi.server/src/user/jobengine/zk/util/ADHandler.java b/server/user.jobengine.osgi.server/src/user/jobengine/zk/util/ADHandler.java index f93b7388..b9603101 100644 --- a/server/user.jobengine.osgi.server/src/user/jobengine/zk/util/ADHandler.java +++ b/server/user.jobengine.osgi.server/src/user/jobengine/zk/util/ADHandler.java @@ -106,113 +106,113 @@ public class ADHandler { * Minden AD csoport els� CN nev�t adja vissza. */ - static public void getAllADGroupName(Vector ret) throws Exception { - - LdapContext ctx = null; - NamingEnumeration answer = null; - try { - //Create the initial directory context - ctx = getContext(); - - //Create the search controls - SearchControls searchCtls = new SearchControls(); - - //Specify the search scope - searchCtls.setSearchScope(SearchControls.SUBTREE_SCOPE); - - //specify the LDAP search filter - String searchFilter = "(&(objectClass=group))"; //(CN=All Research) - - //Specify the Base for the search - String searchBase = cfg.getProperty("ad_base_dn"); - - //initialize counter to total the group members - int totalResults = 0; - - //Specify the attributes to return - String returnedAtts[] = { "member" }; - searchCtls.setReturningAttributes(returnedAtts); - - //Search for objects using the filter - answer = ctx.search(searchBase, searchFilter, searchCtls); - - //Loop through the search results - while (answer.hasMoreElements()) { - SearchResult sr = (SearchResult) answer.next(); - String grpname = sr.getName(); - String[] parts = grpname.split(","); - String[] firstCn = parts[0].split("="); - ret.add(firstCn[1]); - } - - } catch (Exception e) { - } finally { - try { - if (answer != null) - answer.close(); - if (ctx != null) - ctx.close(); - } catch (Exception exc) { - } - } - } - - synchronized static public String getAllFromAD(String name) throws Exception { - - //if(1==1) return "-"; - - String fullName = ""; - LdapContext ctx = null; - NamingEnumeration answer = null; - try { - //Create the initial directory context - ctx = getContext(); - - //Create the search controls - SearchControls searchCtls = new SearchControls(); - - //Specify the search scope - searchCtls.setSearchScope(SearchControls.SUBTREE_SCOPE); - - //Specify the LDAP search filter - String searchFilter = "(&(objectClass=person)(objectCategory=Person)(CN=" + name + "))"; //p�ld�k: "(&(objectClass=person)(objectCategory=Person)(CN=y011216s))"; "(&(objectClass=group)(objectCategory=Group)(CN=Users))"; - - //Specify the Base for the search - String searchBase = cfg.getProperty("ad_base_dn"); - - //Specify the attributes to return - //String returnedAtts[]={"memberOf"}; //"member" - //searchCtls.setReturningAttributes(returnedAtts); - - //Search for objects using the filter - answer = ctx.search(searchBase, searchFilter, searchCtls); - - //Loop through the search results - if (answer.hasMoreElements()) { - SearchResult sr = (SearchResult) answer.next(); - Attributes srAttrs = sr.getAttributes(); - - for (NamingEnumeration enu = srAttrs.getAll(); enu.hasMore();) { - Attribute attr = (Attribute) enu.next(); - //System.out.println(attr); - if (attr != null) { - System.out.println("attrib: " + attr.getID() + " : " + attr.get(0).toString()); - } - } - } - } catch (Exception e) { - System.out.println(e.getMessage()); - } finally { - try { - if (answer != null) - answer.close(); - if (ctx != null) - ctx.close(); - } catch (Exception exc) { - } - } - return fullName; - } + // static public void getAllADGroupName(Vector ret) throws Exception { + // + // LdapContext ctx = null; + // NamingEnumeration answer = null; + // try { + // //Create the initial directory context + // ctx = getContext(); + // + // //Create the search controls + // SearchControls searchCtls = new SearchControls(); + // + // //Specify the search scope + // searchCtls.setSearchScope(SearchControls.SUBTREE_SCOPE); + // + // //specify the LDAP search filter + // String searchFilter = "(&(objectClass=group))"; //(CN=All Research) + // + // //Specify the Base for the search + // String searchBase = cfg.getProperty("ad_base_dn"); + // + // //initialize counter to total the group members + // int totalResults = 0; + // + // //Specify the attributes to return + // String returnedAtts[] = { "member" }; + // searchCtls.setReturningAttributes(returnedAtts); + // + // //Search for objects using the filter + // answer = ctx.search(searchBase, searchFilter, searchCtls); + // + // //Loop through the search results + // while (answer.hasMoreElements()) { + // SearchResult sr = (SearchResult) answer.next(); + // String grpname = sr.getName(); + // String[] parts = grpname.split(","); + // String[] firstCn = parts[0].split("="); + // ret.add(firstCn[1]); + // } + // + // } catch (Exception e) { + // } finally { + // try { + // if (answer != null) + // answer.close(); + // if (ctx != null) + // ctx.close(); + // } catch (Exception exc) { + // } + // } + // } + + // synchronized static public String getAllFromAD(String name) throws Exception { + // + // //if(1==1) return "-"; + // + // String fullName = ""; + // LdapContext ctx = null; + // NamingEnumeration answer = null; + // try { + // //Create the initial directory context + // ctx = getContext(); + // + // //Create the search controls + // SearchControls searchCtls = new SearchControls(); + // + // //Specify the search scope + // searchCtls.setSearchScope(SearchControls.SUBTREE_SCOPE); + // + // //Specify the LDAP search filter + // String searchFilter = "(&(objectClass=person)(objectCategory=Person)(CN=" + name + "))"; //p�ld�k: "(&(objectClass=person)(objectCategory=Person)(CN=y011216s))"; "(&(objectClass=group)(objectCategory=Group)(CN=Users))"; + // + // //Specify the Base for the search + // String searchBase = cfg.getProperty("ad_base_dn"); + // + // //Specify the attributes to return + // //String returnedAtts[]={"memberOf"}; //"member" + // //searchCtls.setReturningAttributes(returnedAtts); + // + // //Search for objects using the filter + // answer = ctx.search(searchBase, searchFilter, searchCtls); + // + // //Loop through the search results + // if (answer.hasMoreElements()) { + // SearchResult sr = (SearchResult) answer.next(); + // Attributes srAttrs = sr.getAttributes(); + // + // for (NamingEnumeration enu = srAttrs.getAll(); enu.hasMore();) { + // Attribute attr = (Attribute) enu.next(); + // //System.out.println(attr); + // if (attr != null) { + // System.out.println("attrib: " + attr.getID() + " : " + attr.get(0).toString()); + // } + // } + // } + // } catch (Exception e) { + // System.out.println(e.getMessage()); + // } finally { + // try { + // if (answer != null) + // answer.close(); + // if (ctx != null) + // ctx.close(); + // } catch (Exception exc) { + // } + // } + // return fullName; + // } synchronized static public LdapContext getContext() throws Exception { @@ -235,89 +235,89 @@ public class ADHandler { return ctx; } - synchronized static public String getDisplayName(String name) throws Exception { - String fullName = ""; - LdapContext ctx = null; - NamingEnumeration answer = null; - try { - ctx = getContext(); - SearchControls searchCtls = new SearchControls(); - searchCtls.setSearchScope(SearchControls.SUBTREE_SCOPE); - String searchFilter = "(&(objectClass=person)(objectCategory=Person)(CN=" + name + "))"; - String searchBase = cfg.getProperty("ad_base_dn"); - answer = ctx.search(searchBase, searchFilter, searchCtls); - if (answer.hasMoreElements()) { - SearchResult sr = answer.next(); - Attribute attr = sr.getAttributes().get("displayName"); - if (attr != null) { - return attr.get(0).toString(); - } - } - } catch (Exception e) { - System.out.println(e.getMessage()); - } finally { - try { - if (answer != null) - answer.close(); - if (ctx != null) - ctx.close(); - } catch (Exception exc) { - } - } - return fullName; - } - - synchronized static public String getMail(String name) throws Exception { - String fullName = ""; - LdapContext ctx = null; - NamingEnumeration answer = null; - try { - ctx = getContext(); - SearchControls searchCtls = new SearchControls(); - searchCtls.setSearchScope(SearchControls.SUBTREE_SCOPE); - String searchFilter = "(&(objectClass=person)(objectCategory=Person)(CN=" + name + "))"; - String searchBase = cfg.getProperty("ad_base_dn"); - answer = ctx.search(searchBase, searchFilter, searchCtls); - if (answer.hasMoreElements()) { - SearchResult sr = answer.next(); - Attribute attr = sr.getAttributes().get("userPrincipalName"); - if (attr != null) { - return attr.get(0).toString(); - } - } - } catch (Exception e) { - System.out.println(e.getMessage()); - } finally { - try { - if (answer != null) - answer.close(); - if (ctx != null) - ctx.close(); - } catch (Exception exc) { - } - } - return fullName; - } - - //TEST CODE - public static void main(String[] args) { - try { - Properties config = new Properties(); - config.setProperty("ad_host", "10.10.254.11"); - config.setProperty("ad_nonsecureport", "389"); - config.setProperty("ad_base_dn", "DC=intra,DC=echotv,DC=hu"); - config.setProperty("ad_nonsecure_user_dn", "CN=echotest,CN=Users"); - config.setProperty("ad_nonsecure_user_password", "aA123456+"); - ADHandler.setConfig(config); - - ADHandler.getMail("echotest"); - System.out.println("Display name: " + ADHandler.getDisplayName("echotest")); - Vector groups = new Vector(); - ADHandler.getADGroupsForCN("echotest", groups); - } catch (Exception e) { - e.printStackTrace(); - } - } + // synchronized static public String getDisplayName(String name) throws Exception { + // String fullName = ""; + // LdapContext ctx = null; + // NamingEnumeration answer = null; + // try { + // ctx = getContext(); + // SearchControls searchCtls = new SearchControls(); + // searchCtls.setSearchScope(SearchControls.SUBTREE_SCOPE); + // String searchFilter = "(&(objectClass=person)(objectCategory=Person)(CN=" + name + "))"; + // String searchBase = cfg.getProperty("ad_base_dn"); + // answer = ctx.search(searchBase, searchFilter, searchCtls); + // if (answer.hasMoreElements()) { + // SearchResult sr = answer.next(); + // Attribute attr = sr.getAttributes().get("displayName"); + // if (attr != null) { + // return attr.get(0).toString(); + // } + // } + // } catch (Exception e) { + // System.out.println(e.getMessage()); + // } finally { + // try { + // if (answer != null) + // answer.close(); + // if (ctx != null) + // ctx.close(); + // } catch (Exception exc) { + // } + // } + // return fullName; + // } + // + // synchronized static public String getMail(String name) throws Exception { + // String fullName = ""; + // LdapContext ctx = null; + // NamingEnumeration answer = null; + // try { + // ctx = getContext(); + // SearchControls searchCtls = new SearchControls(); + // searchCtls.setSearchScope(SearchControls.SUBTREE_SCOPE); + // String searchFilter = "(&(objectClass=person)(objectCategory=Person)(CN=" + name + "))"; + // String searchBase = cfg.getProperty("ad_base_dn"); + // answer = ctx.search(searchBase, searchFilter, searchCtls); + // if (answer.hasMoreElements()) { + // SearchResult sr = answer.next(); + // Attribute attr = sr.getAttributes().get("userPrincipalName"); + // if (attr != null) { + // return attr.get(0).toString(); + // } + // } + // } catch (Exception e) { + // System.out.println(e.getMessage()); + // } finally { + // try { + // if (answer != null) + // answer.close(); + // if (ctx != null) + // ctx.close(); + // } catch (Exception exc) { + // } + // } + // return fullName; + // } + // + // //TEST CODE + // public static void main(String[] args) { + // try { + // Properties config = new Properties(); + // config.setProperty("ad_host", "10.10.254.11"); + // config.setProperty("ad_nonsecureport", "389"); + // config.setProperty("ad_base_dn", "DC=intra,DC=echotv,DC=hu"); + // config.setProperty("ad_nonsecure_user_dn", "CN=echotest,CN=Users"); + // config.setProperty("ad_nonsecure_user_password", "aA123456+"); + // ADHandler.setConfig(config); + // + // ADHandler.getMail("echotest"); + // System.out.println("Display name: " + ADHandler.getDisplayName("echotest")); + // Vector groups = new Vector(); + // ADHandler.getADGroupsForCN("echotest", groups); + // } catch (Exception e) { + // e.printStackTrace(); + // } + // } static public void setConfig(Properties _cfg) { ADHandler.cfg = _cfg; diff --git a/server/user.jobengine.osgi.server/src/user/jobengine/zk/util/SessionUtil.java b/server/user.jobengine.osgi.server/src/user/jobengine/zk/util/SessionUtil.java index eeaed293..969387e6 100644 --- a/server/user.jobengine.osgi.server/src/user/jobengine/zk/util/SessionUtil.java +++ b/server/user.jobengine.osgi.server/src/user/jobengine/zk/util/SessionUtil.java @@ -1,14 +1,11 @@ package user.jobengine.zk.util; -import java.io.FileInputStream; -import java.io.InputStream; import java.security.MessageDigest; import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; import java.util.List; import java.util.Map; -import java.util.Properties; import javax.servlet.http.HttpServletRequest; import javax.servlet.http.HttpSession; @@ -34,6 +31,7 @@ import user.jobengine.zk.model.MaestroConfig; import user.jobengine.zk.model.MediaCubeConfig; public class SessionUtil { + private static final String MEDIACUBE_CONFIG = "mediaCubeConfig"; private static Logger logger = LogManager.getLogger(); private static final String JOBENGINE_MEDIACUBE_CONFIG = "jobengine.mediacube.config"; private static final String JOBENGINE_MAESTRO_CONFIG = "jobengine.maestro.config"; @@ -52,19 +50,18 @@ public class SessionUtil { public static final String GUI_STAT_DISABLE = "gui.statistics.disable"; public static final String GUI_HELP_DISABLE = "gui.help.disable"; public static final String GUI_EDITOR_DISABLE = "gui.editor.disable"; - private static MediaCubeConfig mediaCubeConfig; - - static { - try { - mediaCubeConfig = JSONConfig.read(JOBENGINE_MEDIACUBE_CONFIG, MediaCubeConfig.class); - } catch (Exception e) { - logger.error("Critical error", e); - } - } + // private static MediaCubeConfig mediaCubeConfig; + // + // static { + // try { + // mediaCubeConfig = JSONConfig.read(JOBENGINE_MEDIACUBE_CONFIG, MediaCubeConfig.class); + // } catch (Exception e) { + // logger.error("Critical error", e); + // } + // } public static boolean authenticate(String account, String password) { - //Properties cfg = getConfiguration(); - // MediaCubeConfig mediaCubeConfig = getMediaCubeConfig(); + MediaCubeConfig mediaCubeConfig = getMediaCubeConfig(); LocalUserHandler localUserHandler = new LocalUserHandler(account, password, mediaCubeConfig.getAuthentication()); UserPrincipal userPrincipal = localUserHandler.getUserPrincipal(); // Object nexioStatus = System.getProperty(NEXIO); @@ -148,20 +145,6 @@ public class SessionUtil { return session.getAttribute(name); } - private static Properties getConfiguration() { - - mediaCubeConfig = getMediaCubeConfig(); - // setMediaCubeAuthentication(mediaCubeConfig.getAuthentication()); - Properties cfg = new Properties(); - try (InputStream is = new FileInputStream(System.getProperty(AUTH_LOCATION))) { - cfg.load(is); - } catch (Exception e) { - logger.catching(e); - } - - return cfg; - } - public static MaestroConfig getMaestroConfig() { MaestroConfig result = null; try { @@ -177,7 +160,19 @@ public class SessionUtil { } public static MediaCubeConfig getMediaCubeConfig() { - return mediaCubeConfig; + MediaCubeConfig result = null; + + result = (MediaCubeConfig) getAttribute(MEDIACUBE_CONFIG); + + if (result == null) { + try { + result = JSONConfig.read(JOBENGINE_MEDIACUBE_CONFIG, MediaCubeConfig.class); + setAttribute(MEDIACUBE_CONFIG, result); + } catch (Exception e) { + logger.error(e.getMessage()); + } + } + return result; } static public String getQueryParameter(String key) { @@ -268,7 +263,7 @@ public class SessionUtil { } public static boolean isAuthenticationEnabled() { - return mediaCubeConfig.getAuthentication().isAuthEnabled(); + return getMediaCubeConfig().getAuthentication().isAuthEnabled(); } static public boolean isEditor() { diff --git a/server/user.jobengine.osgi.server/test/user/jobengine/server/scheduler/CronExpressionTest.java b/server/user.jobengine.osgi.server/test/user/jobengine/server/scheduler/CronExpressionTest.java index c6b2dbaf..526c1bce 100644 --- a/server/user.jobengine.osgi.server/test/user/jobengine/server/scheduler/CronExpressionTest.java +++ b/server/user.jobengine.osgi.server/test/user/jobengine/server/scheduler/CronExpressionTest.java @@ -17,4 +17,16 @@ public class CronExpressionTest { System.out.println(nextTime); } } + + @Test + public void test2() throws Exception { + + CronExpression ce = new CronExpression("0 */1 * * * ?"); + Date nextTime = new Date(); + for (int i = 0; i < 10; i++) { + nextTime = ce.getNextValidTimeAfter(nextTime); + System.out.println(nextTime); + } + } + } diff --git a/server/user.jobengine.osgi.services/deploy.launch b/server/user.jobengine.osgi.services/build-services-module.launch similarity index 100% rename from server/user.jobengine.osgi.services/deploy.launch rename to server/user.jobengine.osgi.services/build-services-module.launch diff --git a/server/user.jobengine.osgi.services/src/user/jobengine/osgi/rest/ComponentBinder.java b/server/user.jobengine.osgi.services/src/user/jobengine/osgi/rest/ComponentBinder.java index 72f0b0ac..aef9a803 100644 --- a/server/user.jobengine.osgi.services/src/user/jobengine/osgi/rest/ComponentBinder.java +++ b/server/user.jobengine.osgi.services/src/user/jobengine/osgi/rest/ComponentBinder.java @@ -12,7 +12,8 @@ import user.jobengine.osgi.ws.nexio.NexioWSServlet; import user.jobengine.server.IJobEngine; public class ComponentBinder { - private static final String MEDIACUBE = "MEDIACUBE"; + public static final String MEDIACUBE = "MEDIACUBE"; + public static final String MEDIACUBE_FILELOG = "MEDIACUBE FILELOG"; private static final Logger logger = LogManager.getLogger(); private static IJobEngine jobengine; private static IItemManager itemManager; @@ -86,7 +87,7 @@ public class ComponentBinder { dispatcher.setNexioChangeListener(nexioServlet); dispatcher.startup(); - logger.info(MarkerManager.getMarker(MEDIACUBE), "A MediaCube server elindult."); + logger.info(MarkerManager.getMarker(MEDIACUBE_FILELOG), "A MediaCube server elindult."); } public synchronized void unbindService(Object service) { diff --git a/server/user.peablebeach.api/target/classes/hu/user/peablebeach/api/PBMissingMaterialSrc.class b/server/user.peablebeach.api/target/classes/hu/user/peablebeach/api/PBMissingMaterialSrc.class index d960ad49..5c83a640 100644 Binary files a/server/user.peablebeach.api/target/classes/hu/user/peablebeach/api/PBMissingMaterialSrc.class and b/server/user.peablebeach.api/target/classes/hu/user/peablebeach/api/PBMissingMaterialSrc.class differ diff --git a/server/user.peablebeach.api/target/local-artifacts.properties b/server/user.peablebeach.api/target/local-artifacts.properties index 01a0781c..ba73dee3 100644 --- a/server/user.peablebeach.api/target/local-artifacts.properties +++ b/server/user.peablebeach.api/target/local-artifacts.properties @@ -1,4 +1,4 @@ -#Mon Jul 06 11:16:29 CEST 2020 +#Fri Aug 28 13:48:08 CEST 2020 artifact.attached.p2metadata=C\:\\_workspace\\USER\\MediaCube\\server\\user.peablebeach.api\\target\\p2content.xml artifact.main=C\:\\_workspace\\USER\\MediaCube\\server\\user.peablebeach.api\\target\\user.peablebeach.api_1.0.0.jar artifact.attached.p2artifacts=C\:\\_workspace\\USER\\MediaCube\\server\\user.peablebeach.api\\target\\p2artifacts.xml diff --git a/server/user.peablebeach.api/target/maven-archiver/pom.properties b/server/user.peablebeach.api/target/maven-archiver/pom.properties index 47e50a2d..ae9a7463 100644 --- a/server/user.peablebeach.api/target/maven-archiver/pom.properties +++ b/server/user.peablebeach.api/target/maven-archiver/pom.properties @@ -1,5 +1,5 @@ #Generated by Maven -#Mon Jul 06 11:16:28 CEST 2020 +#Fri Aug 28 13:48:07 CEST 2020 version=1.0.0 groupId=user.jobengine artifactId=user.peablebeach.api diff --git a/server/user.peablebeach.api/target/p2artifacts.xml b/server/user.peablebeach.api/target/p2artifacts.xml index 8c896e17..29d39e64 100644 --- a/server/user.peablebeach.api/target/p2artifacts.xml +++ b/server/user.peablebeach.api/target/p2artifacts.xml @@ -3,8 +3,8 @@ - - + + diff --git a/server/user.peablebeach.api/target/user.peablebeach.api_1.0.0.jar b/server/user.peablebeach.api/target/user.peablebeach.api_1.0.0.jar index 06cc346e..b3cd08c3 100644 Binary files a/server/user.peablebeach.api/target/user.peablebeach.api_1.0.0.jar and b/server/user.peablebeach.api/target/user.peablebeach.api_1.0.0.jar differ