From: Vásáry Dániel Date: Mon, 14 Sep 2020 08:45:06 +0000 (+0000) Subject: git-tfs-id: [http://tfs.userrendszerhaz.hu:8080/tfs/DefaultCollection]$/MediaCube... X-Git-Url: http://git.useribm.hu/?a=commitdiff_plain;h=cacb5db5236f883cad196a5e9dd3a8c1c89131f8;p=mediacube.git git-tfs-id: [tfs.userrendszerhaz.hu:8080/tfs/DefaultCollection]$/MediaCube;C31954 --- diff --git a/server/hu.user.mediacube.executors.tests/src/hu/user/mediacube/executors/tests/HSMMigrateStepTest.java b/server/hu.user.mediacube.executors.tests/src/hu/user/mediacube/executors/tests/HSMMigrateStepTest.java index 5fa41f5f..2655f7ac 100644 --- a/server/hu.user.mediacube.executors.tests/src/hu/user/mediacube/executors/tests/HSMMigrateStepTest.java +++ b/server/hu.user.mediacube.executors.tests/src/hu/user/mediacube/executors/tests/HSMMigrateStepTest.java @@ -5,7 +5,6 @@ import static org.junit.Assert.assertNull; import java.nio.file.Files; import java.nio.file.Paths; -import java.util.LinkedHashSet; import java.util.List; import org.junit.BeforeClass; @@ -30,42 +29,29 @@ public class HSMMigrateStepTest { @Test public void createMigrateReport() throws Exception { - LinkedHashSet success = new LinkedHashSet<>(); - List migrated = Files.readAllLines(Paths.get("/opt/MV-migrate-status/migrated.txt")); + List migrated = Files.readAllLines(Paths.get("/opt/MV-migrate-status/select_distinct_houseid__from_MEDIAFILE.csv")); System.out.println("Starting"); - for (String m : migrated) { - String name = m.trim(); - int indexOf = name.indexOf("-"); - if (indexOf > -1) { - name = name.substring(9); - if (success.contains(name)) - System.out.println("Error " + m.trim()); - else - success.add(name); - } - // System.out.println(name); - } + //Files.write(Paths.get("/opt/MV-migrate-status/migrated-clean.txt"), success); + + StringBuilder sb = new StringBuilder(); List tomigrate = Files.readAllLines(Paths.get("/opt/MV-migrate-status/to-migrate.txt")); int nosuccesscount = 0; for (String t : tomigrate) { String name = t.trim(); name = name.substring(2); - int indexOf = name.indexOf("."); - if (indexOf > -1) { - indexOf = name.lastIndexOf("/"); - name = name.substring(indexOf + 1); - if (!success.contains(name)) { - System.out.println(name); - nosuccesscount++; - } + int indexOf = name.lastIndexOf("/"); + name = name.substring(indexOf + 1); + if (!migrated.contains(name)) { + sb.append("Missing " + name + "\r\n"); + nosuccesscount++; } } + Files.write(Paths.get("/opt/MV-migrate-status/report.txt"), sb.toString().getBytes()); System.out.println("Success " + migrated.size()); System.out.println("No success " + nosuccesscount); - System.out.println(100 - (nosuccesscount * 100 / migrated.size())); - + System.out.println((float) nosuccesscount * 100 / migrated.size() + "% missing"); } private HSMMigrateStep createSUT() { diff --git a/server/hu.user.mediacube.executors.tests/src/hu/user/mediacube/executors/tests/SmallTests.java b/server/hu.user.mediacube.executors.tests/src/hu/user/mediacube/executors/tests/SmallTests.java index 622fd8dc..8ac626b5 100644 --- a/server/hu.user.mediacube.executors.tests/src/hu/user/mediacube/executors/tests/SmallTests.java +++ b/server/hu.user.mediacube.executors.tests/src/hu/user/mediacube/executors/tests/SmallTests.java @@ -10,12 +10,15 @@ import java.util.Arrays; import java.util.Collection; import java.util.Collections; import java.util.Date; +import java.util.HashMap; import java.util.List; +import java.util.Map; import java.util.Set; import java.util.TreeMap; import java.util.TreeSet; import org.apache.commons.io.FilenameUtils; +import org.apache.commons.lang.StringUtils; import org.junit.Test; import user.jobengine.db.Media; @@ -281,4 +284,84 @@ public class SmallTests { String mexio = "2020-09-03T17:52:59.098+0200"; System.out.println(mc.compareTo(mexio)); } + + @Test + public void test94() throws Exception { + List uploads = Files.readAllLines(Paths.get("/opt/AMC/upload-errors.csv")); + //List downloads = Files.readAllLines(Paths.get("/opt/AMC/download-errors.csv")); + + Map ups = new HashMap<>(); + + boolean first = true; + for (String upload : uploads) { + if (first) { + first = false; + continue; + } + String[] upfields = upload.split(";"); + String related = upfields[3]; + + String content = ups.get(related); + + if (upload.contains("FINISHED") && upload.contains("NEXIO")) + continue; + + if (content == null) + content = upload + "\r\n"; + else + content += upload + "\r\n"; + + ups.put(related, content); + } + + System.out.println("Start"); + int countTouched = 0; + int countMultiTouched = 0; + for (String related : ups.keySet()) { + String content = ups.get(related); + int countMatches = StringUtils.countMatches(content, "EXECUTING"); + if (countMatches == 0) + continue; + + countMatches = StringUtils.countMatches(content, "NEXIO"); + if (countMatches == 0) + continue; + + countMatches = StringUtils.countMatches(content, "MEDIACUBE_UPLOADS ->"); + if (countMatches == 0) + continue; + + System.out.println(">>>" + related); + countTouched++; + System.out.println(content); + System.out.println("..."); + + // if (countMatches > 1) { + // countMultiTouched++; + // } + + } + System.out.println(countTouched); + // System.out.println(countMultiTouched); + } + + @Test + public void test95() throws Exception { + DirectoryStream directoryStream = null; + + List escortFiles = new ArrayList<>(); + directoryStream = Files.newDirectoryStream(Paths.get("/_video")); + for (Path escortFile : directoryStream) { + if (escortFile.toFile().isDirectory()) + continue; + escortFiles.add(escortFile); + } + escortFiles.sort((p1, p2) -> { + return Long.valueOf(p1.toFile().lastModified()).compareTo(p2.toFile().lastModified()); + }); + + for (Path p : escortFiles) { + System.out.println(p.getFileName()); + } + } } diff --git a/server/user.jobengine.executors/src/user/jobengine/server/steps/CancelableStep.java b/server/user.jobengine.executors/src/user/jobengine/server/steps/CancelableStep.java index dae68860..cf9d0358 100644 --- a/server/user.jobengine.executors/src/user/jobengine/server/steps/CancelableStep.java +++ b/server/user.jobengine.executors/src/user/jobengine/server/steps/CancelableStep.java @@ -29,8 +29,9 @@ public class CancelableStep extends JobStep { // getJobRuntime().setDescription(String.valueOf(param)); try { + //getJobRuntime().setRelated("xxx"); //logger.info(new MediaCubeMarker("vasary@elgekko.net", "TESZT"), "Heloka"); - logger.error(getSessionMarker(), "Heloka"); + //logger.error(getSessionMarker(), "Heloka"); for (int i = 0; i < count; i++) { if (getJobRuntime().isWaitingCancel()) break; diff --git a/server/user.jobengine.executors/src/user/jobengine/server/steps/FILEZILLA_PASARESTORETransferToStep.java b/server/user.jobengine.executors/src/user/jobengine/server/steps/FILEZILLA_PASARESTORETransferToStep.java new file mode 100644 index 00000000..d69a6ed4 --- /dev/null +++ b/server/user.jobengine.executors/src/user/jobengine/server/steps/FILEZILLA_PASARESTORETransferToStep.java @@ -0,0 +1,11 @@ +package user.jobengine.server.steps; + +import user.commons.StoreUri; + +public class FILEZILLA_PASARESTORETransferToStep extends TransferStep { + @Override + @StepEntry + public Object[] execute(StoreUri sourceStoreUri, String sourceFileName, StoreUri targetStoreUri, String targetFileName) throws Exception { + return super.execute(sourceStoreUri, sourceFileName, targetStoreUri, targetFileName); + } +} diff --git a/server/user.jobengine.executors/src/user/jobengine/server/steps/ForkDownloadStep.java b/server/user.jobengine.executors/src/user/jobengine/server/steps/ForkDownloadStep.java index 7923ae65..addb9f75 100644 --- a/server/user.jobengine.executors/src/user/jobengine/server/steps/ForkDownloadStep.java +++ b/server/user.jobengine.executors/src/user/jobengine/server/steps/ForkDownloadStep.java @@ -9,14 +9,17 @@ import java.util.ArrayList; import java.util.HashMap; import java.util.List; import java.util.Map; +import java.util.Set; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import user.commons.DownloadableMedia; import user.commons.JobStatus; +import user.commons.MediaCubeMarker; import user.commons.StoreUri; import user.commons.remotestore.RemoteStoreProtocol; +import user.jobengine.db.Store; import user.jobengine.server.IJobEngine; import user.jobengine.server.IJobRuntime; @@ -25,6 +28,9 @@ public class ForkDownloadStep extends JobStep { @StepEntry public Object[] execute(String tempStoreName, String template, String expectedColorSpace, int limit) throws Exception { + + sendStatusReport(); + DirectoryStream directoryStream = null; int count = limit; int allCount = 0; @@ -44,14 +50,8 @@ public class ForkDownloadStep extends JobStep { escortFiles.add(escortFile); } allCount = escortFiles.size(); - //Comparison method violates its general contract! miatt try-catch escortFiles.sort((p1, p2) -> { - int result = 0; - try { - result = p1.toFile().lastModified() > p2.toFile().lastModified() ? 1 : -1; - } catch (Exception e) { - } - return result; + return Long.valueOf(p1.toFile().lastModified()).compareTo(p2.toFile().lastModified()); }); if (allCount > 0) { @@ -104,9 +104,9 @@ public class ForkDownloadStep extends JobStep { if (EscortFiles.isCatchedFileExists(escortFile)) return false; - String fileName = escortFile.getFileName().toString(); + String escortFileName = escortFile.getFileName().toString(); //minimum 3 karakter: STORENAME.FILENAME.MXF.JSON - long dotCount = fileName.chars().filter(ch -> ch == '.').count(); + long dotCount = escortFileName.chars().filter(ch -> ch == '.').count(); if (dotCount < 3) return false; @@ -117,13 +117,17 @@ public class ForkDownloadStep extends JobStep { return false; } - logger.info(getMarker(), "Starting {} for {}", template, fileName); - StoreUri sourceStoreUri = getManager().getStoreUri(downloadable.getLong("sourceStoreUriId")); StoreUri targetStoreUri = getManager().getStoreUri(downloadable.getLong("targetStoreUriId")); + + Store targetStore = getManager().getStore(targetStoreUri.getStoreId()); + template = targetStore.getName() + "-" + template; + logger.info(getMarker(), "Starting {} for {}", template, escortFileName); + + String sourceFileName = downloadable.getString("fileName"); Map parameters = new HashMap<>(); parameters.put("sourceStoreUri", sourceStoreUri); - parameters.put("fileName", downloadable.getString("fileName")); + parameters.put("fileName", sourceFileName); parameters.put("tempStoreUri", tempStoreUri); parameters.put("targetStoreUri", targetStoreUri); parameters.put("expectedFrameNumber", downloadable.getLong("frames")); @@ -131,15 +135,21 @@ public class ForkDownloadStep extends JobStep { parameters.put("expectedColorSpace", downloadable.getString("colorSpace")); parameters.put("escortFile", escortFile.toAbsolutePath().toString()); - //IJobRuntime parent, String template, String name, int priority, Map parameters + int priority = 50; + + //megszakitasok miatt a mar lejott mozikat vegyuk elore 20.09.09 + if (targetExists(tempStoreUri, sourceFileName)) + priority = 51; + + //parameterek: IJobRuntime parent, String template, String name, int priority, Map parameters IJobRuntime child = getEngine().submit(null, e -> { if (e.getStatus().equals(JobStatus.CANCELED)) EscortFiles.removeCatchedFile(escortFile); - }, template, "Archiválás", 0, IJobEngine.DEFAULT_OWNER, parameters); + }, template, "Archiválás", priority, IJobEngine.DEFAULT_OWNER, parameters); if (child == null) throw new Exception("Submit error."); - child.setDescription(fileName); + child.setDescription(sourceFileName); child.setRelated(downloadable.getString("title")); EscortFiles.createCatchedFile(escortFile); @@ -150,4 +160,37 @@ public class ForkDownloadStep extends JobStep { return true; } + + private void sendStatusReport() { + IJobEngine engine = getEngine(); + int jobCount = 0; + StringBuilder sb = new StringBuilder(); + Map jobs = engine.getJobs(); + if (jobs != null) { + Set keys = jobs.keySet(); + jobCount = keys.size(); + for (Long key : keys) { + IJobRuntime runtime = jobs.get(key); + if (JobStatus.SUSPENDED.equals(runtime.getStatus())) + sb.append(runtime.getRelated() + " SUSPENDED: " + runtime.getDescription() + "
"); + } + } + MediaCubeMarker marker = new MediaCubeMarker(); + marker.setSubject("AMC MediaCube feldolgozási sor " + jobCount); + logger.info(marker, sb.toString()); + } + + private boolean targetExists(StoreUri storeUri, String fileName) { + try { + Path targetFile = Paths.get(storeUri.toString(true), fileName); + if (targetFile.toFile().exists()) { + logger.info(getMarker(), "Retrieved file {} exists", fileName); + return true; + } + } catch (Exception e) { + logger.error(getMarker(), e.getMessage()); + } + logger.info(getMarker(), "Retrieved file {} not exists yet", fileName); + return false; + } } diff --git a/server/user.jobengine.executors/src/user/jobengine/server/steps/ForkUploadStep.java b/server/user.jobengine.executors/src/user/jobengine/server/steps/ForkUploadStep.java index c6544ce6..70da7b5e 100644 --- a/server/user.jobengine.executors/src/user/jobengine/server/steps/ForkUploadStep.java +++ b/server/user.jobengine.executors/src/user/jobengine/server/steps/ForkUploadStep.java @@ -19,6 +19,7 @@ import user.commons.StoreUri; import user.commons.remotestore.RemoteStoreProtocol; import user.jobengine.db.Media; import user.jobengine.db.MediaFile; +import user.jobengine.db.Store; import user.jobengine.server.IJobEngine; import user.jobengine.server.IJobRuntime; @@ -47,8 +48,13 @@ public class ForkUploadStep extends JobStep { } allCount = escortFiles.size(); //a file neve az adasbakerules idopontjaval kezdodik + // escortFiles.sort((p1, p2) -> { + // return p1.compareTo(p2); + // }); + + //200911 megis inkabb a letrehozas idopontja legyen escortFiles.sort((p1, p2) -> { - return p1.compareTo(p2); + return Long.valueOf(p1.toFile().lastModified()).compareTo(p2.toFile().lastModified()); }); if (allCount > 0) { @@ -114,9 +120,9 @@ public class ForkUploadStep extends JobStep { if (EscortFiles.isCatchedFileExists(escortFile)) return false; - String fileName = escortFile.getFileName().toString(); + String escortFileName = escortFile.getFileName().toString(); //minimum 3 karakter: STORENAME.FILENAME.MXF.JSON - long dotCount = fileName.chars().filter(ch -> ch == '.').count(); + long dotCount = escortFileName.chars().filter(ch -> ch == '.').count(); if (dotCount < 3) return false; @@ -124,20 +130,24 @@ public class ForkUploadStep extends JobStep { DownloadableMedia downloadable = EscortFiles.decode(escortFile); if (downloadable == null) { - logger.error(getSessionMarker(), "Can't encode {}", escortFile); + logger.error(getSessionMarker(), "Can't decode {}", escortFile); return false; } - logger.info(getMarker(), "Starting {} for {}", template, fileName); StoreUri sourceStoreUri = getManager().getStoreUri("TSM", RemoteStoreProtocol.TSM); StoreUri targetStoreUri = getManager().getStoreUri(downloadable.getLong("targetStoreUriId")); + Store targetStore = getManager().getStore(targetStoreUri.getStoreId()); + template = targetStore.getName() + "-" + template; + logger.info(getMarker(), "Starting {} for {}", template, escortFileName); + String tsmFileName = getTSMFileName(downloadable.getLong("mediaId"), sourceStoreUri); + String targetFileName = downloadable.getString("fileName"); Map parameters = new HashMap<>(); parameters.put("sourceStoreUri", sourceStoreUri); parameters.put("tsmFileName", tsmFileName); - parameters.put("fileName", downloadable.getString("fileName")); + parameters.put("fileName", targetFileName); parameters.put("tempStoreUri", tempStoreUri); parameters.put("targetStoreUri", targetStoreUri); parameters.put("expectedFrameNumber", downloadable.getLong("frames")); @@ -145,8 +155,17 @@ public class ForkUploadStep extends JobStep { parameters.put("expectedColorSpace", downloadable.getString("colorSpace")); parameters.put("escortFile", escortFile.toAbsolutePath().toString()); - int priority = downloadable.containsKey("priority") ? downloadable.getInt("priority") : 0; - //IJobRuntime parent, String template, String name, int priority, Map parameters + //int priority = downloadable.containsKey("priority") ? downloadable.getInt("priority") : 0; + //mindenki egyforman induljon 20.09.07 + //a NEXIO egy kicsit fontosabb 20.09.08. megsem 20.09.09 + //int priority = escortFile.toString().contains("NEXIO") ? 51 : 50; + //megszakitasok miatt a mar lejott mozikat vegyuk elore 20.09.09 + + int priority = 50; + if (targetExists(tempStoreUri, targetFileName)) + priority = 51; + + //parameterek: IJobRuntime parent, String template, String name, int priority, Map parameters IJobRuntime child = getEngine().submit(null, e -> { if (e.getStatus().equals(JobStatus.CANCELED) || e.getStatus().equals(JobStatus.SUSPENDED)) EscortFiles.removeCatchedFile(escortFile); @@ -154,7 +173,7 @@ public class ForkUploadStep extends JobStep { if (child == null) throw new Exception("Submit error."); - child.setDescription(fileName); + child.setDescription(escortFileName); child.setRelated(downloadable.getString("title")); EscortFiles.createCatchedFile(escortFile); @@ -165,4 +184,18 @@ public class ForkUploadStep extends JobStep { return true; } + + private boolean targetExists(StoreUri storeUri, String fileName) { + try { + Path targetFile = Paths.get(storeUri.toString(true), fileName); + if (targetFile.toFile().exists()) { + logger.info(getMarker(), "Retrieved file {} exists", fileName); + return true; + } + } catch (Exception e) { + logger.error(getMarker(), e.getMessage()); + } + logger.info(getMarker(), "Retrieved file {} not exists yet", fileName); + return false; + } } diff --git a/server/user.jobengine.executors/src/user/jobengine/server/steps/MetadataPersisterStep.java b/server/user.jobengine.executors/src/user/jobengine/server/steps/MetadataPersisterStep.java index c8cb85f2..007ad6c0 100644 --- a/server/user.jobengine.executors/src/user/jobengine/server/steps/MetadataPersisterStep.java +++ b/server/user.jobengine.executors/src/user/jobengine/server/steps/MetadataPersisterStep.java @@ -36,6 +36,10 @@ public class MetadataPersisterStep extends JobStep { IItemManager manager = getManager(); + if (mediaId == 0) { + mediaId = tryGetMediaId(title, mediaId); + } + //TODO ha van mediaId, akkor update!!!!!!!!! if (mediaId == 0) { @@ -59,6 +63,8 @@ public class MetadataPersisterStep extends JobStep { media.add(); mediaFile.setMediaId(media.getId()); mediaFile.setLastModified(modified); + //TODO + //mediaFile.setFileSize(fileSize); mediaFile.add(); } else { Media media = manager.getMedia(mediaId); @@ -68,6 +74,8 @@ public class MetadataPersisterStep extends JobStep { media.setCreated(created); manager.modify(media); MediaFile mediaFile = manager.getSystemMediaFile(media); + //TODO + //mediaFile.setFileSize(fileSize); mediaFile.setLastModified(modified); manager.modify(mediaFile); } @@ -81,4 +89,17 @@ public class MetadataPersisterStep extends JobStep { return null; } + private long tryGetMediaId(String title, long mediaId) { + Media media = null; + //duplikatum miatt + try { + media = getManager().getMedia(title); + if (media != null) + mediaId = media.getId(); + } catch (Exception e) { + logger.error(getMarker(), "Media with title {} is not available. System message is: {}", title, e.getMessage()); + } + return mediaId; + } + } diff --git a/server/user.jobengine.executors/src/user/jobengine/server/steps/PeableBeachMissingMaterialCheckerStep.java b/server/user.jobengine.executors/src/user/jobengine/server/steps/PeableBeachMissingMaterialCheckerStep.java index 1c64232f..a99e9abb 100644 --- a/server/user.jobengine.executors/src/user/jobengine/server/steps/PeableBeachMissingMaterialCheckerStep.java +++ b/server/user.jobengine.executors/src/user/jobengine/server/steps/PeableBeachMissingMaterialCheckerStep.java @@ -50,13 +50,22 @@ public class PeableBeachMissingMaterialCheckerStep extends JobStep { return null; } String lookupStoreName = String.valueOf(lookup); + StoreUri lookupStoreUri = getManager().getStoreUri(lookupStoreName, RemoteStoreProtocol.LOCAL); - List lookupContent = getPoolContent(lookupStoreUri); + logger.info(getMarker(), "Checking pool {}", lookupStoreName); + List lookupContent = getPoolContent(lookupStoreName, lookupStoreUri); poolContent.addAll(lookupContent); processed++; progress = processed * 50 / count; setProgress(progress); } + /* + if (1 == 1) { + + logger.info(getMarker(), "Kikapcsolva"); + return null; + } + */ PBQuery query = new PBQuery(); query.init(primaryEndPoint, primaryUserName, primaryPassword, secondaryEndPoint, secondaryUserName, secondaryPassword, rangeForwardHours); @@ -65,6 +74,11 @@ public class PeableBeachMissingMaterialCheckerStep extends JobStep { if (medias != null && medias.size() > 0) { count = medias.size(); + + for (MMMedia media : medias) { + logger.info(getMarker(), "API response contains {}", media.getName()); + } + for (MMMedia media : medias) { if (getJobRuntime().isWaitingCancel()) { cancel(); @@ -89,16 +103,26 @@ public class PeableBeachMissingMaterialCheckerStep extends JobStep { return null; } - private List getPoolContent(StoreUri targetStoreUri) throws Exception { + private List getPoolContent(String storeName, StoreUri targetStoreUri) throws Exception { List poolContent = new ArrayList<>(); List remoteFiles = targetStoreUri.getRemoteFiles(); - if (remoteFiles != null) - remoteFiles.forEach(i -> { - String name = i.getName(); - poolContent.add(name.substring(0, name.lastIndexOf("."))); - }); + if (remoteFiles != null) { + for (RemoteFile rf : remoteFiles) { + try { + String name = rf.getName(); + int pos = name.lastIndexOf("."); + if (pos > -1) { + name = name.substring(0, pos); + //logger.info(getMarker(), "Adding {}", name); + poolContent.add(name); + } + } catch (Exception e) { + logger.error(e.getMessage()); + } - logger.info(getMarker(), "Pool contains {} items", poolContent == null ? 0 : poolContent.size()); + } + } + logger.info(getMarker(), "Pool {} contains {} items", storeName, poolContent == null ? 0 : poolContent.size()); return poolContent; } @@ -115,7 +139,15 @@ public class PeableBeachMissingMaterialCheckerStep extends JobStep { private void processRecord(MMMedia mmedia, StoreUri sourceStoreUri, String targetStoreName, StoreUri targetStoreUri, StoreUri escortStoreUri) { //TODO kisbetu/nagybetu problema kezelese - Media media = getManager().getMedia(mmedia.getName()); + + Media media = null; + //duplikatum miatt + try { + media = getManager().getMedia(mmedia.getName()); + } catch (Exception e) { + logger.error(getMarker(), "Can't identify missing material {}. System message is: {}", mmedia.getName(), e.getMessage()); + return; + } String fileName = normalizeName(mmedia.getName()); diff --git a/server/user.jobengine.executors/src/user/jobengine/server/steps/TransferFromFTPStep.java b/server/user.jobengine.executors/src/user/jobengine/server/steps/TransferFromFTPStep.java new file mode 100644 index 00000000..3142133b --- /dev/null +++ b/server/user.jobengine.executors/src/user/jobengine/server/steps/TransferFromFTPStep.java @@ -0,0 +1,11 @@ +package user.jobengine.server.steps; + +import user.commons.StoreUri; + +public class TransferFromFTPStep extends TransferStep { + @Override + @StepEntry + public Object[] execute(StoreUri sourceStoreUri, String sourceFileName, StoreUri targetStoreUri, String targetFileName) throws Exception { + return super.execute(sourceStoreUri, sourceFileName, targetStoreUri, targetFileName); + } +} diff --git a/server/user.jobengine.executors/src/user/jobengine/server/steps/TransferFromTSMStep.java b/server/user.jobengine.executors/src/user/jobengine/server/steps/TransferFromTSMStep.java index 3521d3f6..8eb982d2 100644 --- a/server/user.jobengine.executors/src/user/jobengine/server/steps/TransferFromTSMStep.java +++ b/server/user.jobengine.executors/src/user/jobengine/server/steps/TransferFromTSMStep.java @@ -1,17 +1,31 @@ package user.jobengine.server.steps; +import java.nio.file.Path; +import java.nio.file.Paths; + +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + import user.commons.StoreUri; public class TransferFromTSMStep extends TransferStep { + private static final Logger logger = LogManager.getLogger(); + @Override @StepEntry public Object[] execute(StoreUri sourceStoreUri, String sourceFileName, StoreUri targetStoreUri, String targetFileName) throws Exception { - return super.execute(sourceStoreUri, sourceFileName, targetStoreUri, targetFileName); - } - - @Override - protected String getTmpExtension() { - return null; + Object[] result = null; + try { + Path targetFile = Paths.get(targetStoreUri.toString(true), targetFileName); + if (targetFile.toFile().exists()) { + logger.info(getMarker(), "Using previously retrived file {}", targetFile); + } else { + result = super.execute(sourceStoreUri, sourceFileName, targetStoreUri, targetFileName); + } + } catch (Exception e) { + logger.error(getMarker(), e.getMessage()); + } + return result; } } diff --git a/server/user.jobengine.executors/src/user/jobengine/server/steps/TransferToFTPStep.java b/server/user.jobengine.executors/src/user/jobengine/server/steps/TransferToFTPStep.java new file mode 100644 index 00000000..b42d5d53 --- /dev/null +++ b/server/user.jobengine.executors/src/user/jobengine/server/steps/TransferToFTPStep.java @@ -0,0 +1,11 @@ +package user.jobengine.server.steps; + +import user.commons.StoreUri; + +public class TransferToFTPStep extends TransferStep { + @Override + @StepEntry + public Object[] execute(StoreUri sourceStoreUri, String sourceFileName, StoreUri targetStoreUri, String targetFileName) throws Exception { + return super.execute(sourceStoreUri, sourceFileName, targetStoreUri, targetFileName); + } +} diff --git a/server/user.jobengine.osgi.commons/src/user/commons/StoreUri.java b/server/user.jobengine.osgi.commons/src/user/commons/StoreUri.java index 1947f94c..e7c0f8ad 100644 --- a/server/user.jobengine.osgi.commons/src/user/commons/StoreUri.java +++ b/server/user.jobengine.osgi.commons/src/user/commons/StoreUri.java @@ -157,7 +157,8 @@ public class StoreUri extends EntityBase implements Serializable { if (outputLister != null) outputLister.cleanUp(); } catch (Exception e) { - logger.error("Additional error!", e); + logger.error("Error closing copy: {}", targetName); + throw e; } } result.setSize(uploadedBytes); diff --git a/server/user.jobengine.osgi.commons/src/user/commons/pool/orig/ConnectionPool.java b/server/user.jobengine.osgi.commons/src/user/commons/pool/orig/ConnectionPool.java deleted file mode 100644 index bcae09f6..00000000 --- a/server/user.jobengine.osgi.commons/src/user/commons/pool/orig/ConnectionPool.java +++ /dev/null @@ -1,70 +0,0 @@ -package user.commons.pool.orig; - -import java.sql.Connection; -import java.sql.DriverManager; -import java.sql.SQLException; - - -/** - * Adatbázis kapcsolat erõforrás kezelõ. - * - */ -public class ConnectionPool extends ResourcePool { - public ConnectionPool(ConnectionPoolProperties properties) { - super(properties); - } - - public Connection borrowObject() { - return (Connection) super.borrowObject(); - } - - protected boolean checkObject(Object resourceObject) { - Connection connection = (Connection) resourceObject; - boolean closed = false; - try { - closed = connection.isClosed(); - } catch (SQLException e) { - } - return closed; - } - - protected Connection createObject() { - ConnectionPoolProperties cpp = (ConnectionPoolProperties) this.getProperties(); - Connection connection = null; - try { - Class.forName(cpp.getDriverName()); - DriverManager.setLoginTimeout(cpp.getTimeout()); - connection = DriverManager.getConnection(cpp.getUrl(), cpp.getUserName(), cpp.getPassword()); - connection.setAutoCommit(cpp.getAutoCommit()); - } catch (Exception e) { - throw new PoolException(e.getMessage()); - } - return connection; - } - - protected void releaseObject(Object resourceObject) { - validateObject(resourceObject); - Connection connection = (Connection) resourceObject; - try { - if (connection == null) - return; - if (connection.isClosed() == false) { - connection.close(); - } - connection = null; - } catch (Exception e) { - throw new PoolException(e.getMessage()); - } - } - - public void lendObject(Connection connection) { - super.lendObject(connection); - } - - protected void validateObject(Object resourceObject) { - super.validateObject(resourceObject); - if (!(resourceObject instanceof Connection)) { - throw new IllegalArgumentException("Only Connection class is acceptable."); - } - } -} diff --git a/server/user.jobengine.osgi.commons/src/user/commons/pool/orig/ConnectionPoolProperties.java b/server/user.jobengine.osgi.commons/src/user/commons/pool/orig/ConnectionPoolProperties.java deleted file mode 100644 index 94b4f75d..00000000 --- a/server/user.jobengine.osgi.commons/src/user/commons/pool/orig/ConnectionPoolProperties.java +++ /dev/null @@ -1,51 +0,0 @@ -package user.commons.pool.orig; - -import user.commons.CommonsProperties; - -public class ConnectionPoolProperties extends CommonsProperties { - static public final String AUTOCOMMIT = "AutoCommit"; - static public final String DRIVERNAME = "DriverName"; - static public final String PASSWORD = "Password"; - static public final String TIMEOUT = "Timeout"; - static public final String URL = "ConnectionUrl"; - static public final String USERNAME = "UserName"; - - public ConnectionPoolProperties(String driver, String url, String userName, String password, boolean autoCommit, int timeout) { - CommonsProperties.checkParameter(DRIVERNAME, driver); - CommonsProperties.checkParameter(URL, url); - CommonsProperties.checkParameter(USERNAME, userName); - CommonsProperties.checkParameter(PASSWORD, password); - CommonsProperties.checkParameter(AUTOCOMMIT, autoCommit); - CommonsProperties.checkParameter(TIMEOUT, timeout); - super.add(DRIVERNAME, driver); - super.add(URL, url); - super.add(USERNAME, userName); - super.add(PASSWORD, password); - super.add(AUTOCOMMIT, autoCommit); - super.add(TIMEOUT, timeout); - } - - public boolean getAutoCommit(){ - return getBool(AUTOCOMMIT); - } - - public String getDriverName(){ - return getString(DRIVERNAME); - } - - public String getPassword(){ - return getString(PASSWORD); - } - - public int getTimeout() { - return getInt(TIMEOUT); - } - - public String getUrl(){ - return getString(URL); - } - - public String getUserName(){ - return getString(USERNAME); - } -} diff --git a/server/user.jobengine.osgi.commons/src/user/commons/pool/orig/ContextPool.java b/server/user.jobengine.osgi.commons/src/user/commons/pool/orig/ContextPool.java deleted file mode 100644 index 5deca666..00000000 --- a/server/user.jobengine.osgi.commons/src/user/commons/pool/orig/ContextPool.java +++ /dev/null @@ -1,103 +0,0 @@ -package user.commons.pool.orig; - - -/* -import java.lang.reflect.InvocationTargetException; -import java.sql.Connection; -import java.sql.DriverManager; -import java.sql.SQLException; - -import sqlj.runtime.ref.DefaultContext; -*/ -/** - * Adatbázis kontextus erõforrás kezelõ. - * - */ -public class ContextPool extends ResourcePool { - public ContextPool(ConnectionPoolProperties properties) { - super(properties); - } -/* - public DefaultContext borrowObject() { - DefaultContext context = (DefaultContext) super.borrowObject(); - DefaultContext.setDefaultContext(context); - return context; - } - - protected boolean checkObject(Object resourceObject) { - DefaultContext context = (DefaultContext) resourceObject; - return context.isClosed(); - } - - protected DefaultContext createObject() { - ConnectionPoolProperties cpp = (ConnectionPoolProperties) this.getProperties(); - Connection connection = null; - DefaultContext context = null; - try { - Class.forName(cpp.getDriverName()); - } catch (ClassNotFoundException e) { - throw new PoolException(e.getMessage()); - } - - try { - DriverManager.setLoginTimeout(cpp.getTimeout()); - connection = DriverManager.getConnection(cpp.getUrl(), cpp.getUserName(), cpp.getPassword()); - connection.setAutoCommit(cpp.getAutoCommit()); - context = new DefaultContext(connection); - } catch (SQLException e) { - throw new PoolException(e.getMessage()); - } catch (RuntimeException e) { - Throwable cause = e.getCause(); - if(cause instanceof InvocationTargetException) { - Throwable targetException = ((InvocationTargetException)cause).getTargetException(); - targetException.printStackTrace(); - } - } - return context; - } - - protected void releaseObject(Object value) { - validateObject(value); - DefaultContext context = (DefaultContext) value; - try { - if (context == null) - return; - if (context.isClosed() == false) { - context.close(); - } - context = null; - } catch (Exception e) { - throw new PoolException(e.getMessage()); - } - } - - public void lendObject(DefaultContext context) { - super.lendObject(context); - } - - protected void validateObject(Object resourceObject) { - super.validateObject(resourceObject); - if (!(resourceObject instanceof DefaultContext)) { - throw new IllegalArgumentException("Only DefaultContext class is acceptable."); - } - } -*/ - - @Override - protected boolean checkObject(Object resourceObject) { - // TODO Auto-generated method stub - return false; - } - - @Override - protected Object createObject() { - // TODO Auto-generated method stub - return null; - } - - @Override - protected void releaseObject(Object resourceObject) { - // TODO Auto-generated method stub - - } -} diff --git a/server/user.jobengine.osgi.commons/src/user/commons/pool/orig/PoolException.java b/server/user.jobengine.osgi.commons/src/user/commons/pool/orig/PoolException.java deleted file mode 100644 index ed59f502..00000000 --- a/server/user.jobengine.osgi.commons/src/user/commons/pool/orig/PoolException.java +++ /dev/null @@ -1,12 +0,0 @@ -package user.commons.pool.orig; - -/** - * Pool kivétel. - */ -public class PoolException extends RuntimeException { - private static final long serialVersionUID = 874798532700307318L; - - public PoolException(String message) { - super(message); - } -} diff --git a/server/user.jobengine.osgi.commons/src/user/commons/pool/orig/ResourcePool.java b/server/user.jobengine.osgi.commons/src/user/commons/pool/orig/ResourcePool.java deleted file mode 100644 index fbd845c6..00000000 --- a/server/user.jobengine.osgi.commons/src/user/commons/pool/orig/ResourcePool.java +++ /dev/null @@ -1,166 +0,0 @@ -package user.commons.pool.orig; - -import java.util.LinkedList; - -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; - -import user.commons.CommonsProperties; - -/** - * Er�forr�s kezel�. - *
- * Az implement�ci�ban meghat�rozott t�pus� er�forr�s objektumb�l - * ig�ny szerint l�trehoz annyit, amennyi enged�lyezett. - *
- * Az er�forr�s objektumok k�lcs�n�zhet�ek, visszaadhat�ak - * thread-safe m�don. - *
- * Mind�g a legr�gebben haszn�lt objektumot k�lcs�nzi ki. - *
- * Ha elfogy a k�lcs�n�zhet� objektum, v�rakoztat. - */ - -abstract public class ResourcePool { - private static final Logger logger = LogManager.getLogger(); - static public final String POOLSIZE = "PoolSize"; - protected LinkedList pool = null; - private CommonsProperties properties = null; - private int resourceCounter = 0; - - /** - * Konstruktor a pool l�trehoz�s�ra. - * @param properties - * Be�ll�t�sok. - */ - protected ResourcePool(CommonsProperties properties) { - if (properties == null) - throw new NullPointerException("Pool property map cannot be null."); - this.properties = properties; - resourceCounter = this.properties.getInt(POOLSIZE); - } - - /** - * Er�forr�s objektum k�lcs�nz�se. - * @return - * Kik�lcs�nz�tt er�forr�s objektum. - * Visszet�r�s el�tt a checkObject-el ellen�rzi, hogy haszn�lhat� e az objektum. - * Ha nem �jat, hoz l�tre. - */ - protected Object borrowObject() { - if (pool == null) - pool = new LinkedList(); - - Object resourceObject = null; - - for (;;) { - synchronized (this) { - resourceObject = pool.poll(); - } - if (resourceObject != null) - break; - if (resourceCounter > 0) { - synchronized (this) { - resourceObject = createObject(); - resourceCounter--; - } - break; - } - - System.out.println("Waiting for resource."); - try { - Thread.sleep(1000); - } catch (InterruptedException e) { - } - } - - if (resourceObject != null && checkObject(resourceObject) == true) { - synchronized (this) { - System.out.println("Resource not passed the check. Recreating."); - if (pool.contains(resourceObject)) - pool.remove(resourceObject); - resourceObject = createObject(); - } - } - logger.trace(String.format("Borrow, pool size: %1$d, counter: %2$d", pool.size(), resourceCounter)); - return resourceObject; - } - - /** - * Ellen�rzi, hogy haszn�lhat� e az objektum. - * Implement�ci� f�gg�! - * @param resourceObject - * Ellen�rizend� objektum. - * @return - * true ha m�r nem haszn�lhat� - */ - abstract protected boolean checkObject(Object resourceObject); - - /** - * Az �sszes nem kik�lcs�nz�tt er�forr�s objektumot, - * a releaseObject h�v�ssal felszabad�tja. - */ - public void close() { - if (pool == null) - return; - Object resourceObject = null; - synchronized (this) { - while ((resourceObject = pool.poll()) != null) { - releaseObject(resourceObject); - } - } - pool = null; - } - - /** - * L�trehozza az er�forr�s objektumot. - * Implement�ci� f�gg�! - * @return - * A l�trehozott er�forr�s objektum. - */ - abstract protected Object createObject(); - - /** - * Param�terek lek�rdez�se. - * @return - * CommonsProperties t�pus� param�ter objektum. - */ - protected CommonsProperties getProperties() { - return properties; - } - - /** - * K�lcs�nvett er�forr�s objektum visszaszolg�ltat�sa. - * T�pus �s null pointer elle�rz�s a validateObject seg�ts�g�vel. - * @param value - * Er�forr�s objektum. - */ - protected void lendObject(Object value) { - if (pool == null) { - throw new IllegalStateException("Pool never initialized, or closed. Did you missed something?"); - } - validateObject(value); - synchronized (this) { - pool.add(value); - } - logger.trace(String.format("Lend, pool size: %1$d, counter: %2$d", pool.size(), resourceCounter)); - } - - /** - * Er�forr�s objektum felszabad�t�sa (kapcsolat lez�r�s, - * haszn�lt er�forr�sok felszabad�t�sa). - * Implement�ci� f�gg�! - * @param resourceObject - * A sz�ban forg� objektum. - */ - abstract protected void releaseObject(Object resourceObject); - - /** - * Ellen�rzi, hogy az objektum t�pusa megfelel� e. - */ - protected void validateObject(Object value) { - if (value == null) { - throw new NullPointerException("Object is null."); - } - } -} diff --git a/server/user.jobengine.osgi.commons/src/user/commons/remotestore/FtpDirectoryLister.java b/server/user.jobengine.osgi.commons/src/user/commons/remotestore/FtpDirectoryLister.java index d2408657..57eea5dc 100644 --- a/server/user.jobengine.osgi.commons/src/user/commons/remotestore/FtpDirectoryLister.java +++ b/server/user.jobengine.osgi.commons/src/user/commons/remotestore/FtpDirectoryLister.java @@ -64,7 +64,6 @@ public class FtpDirectoryLister implements IDirectoryLister { } try { ftpClient.logout(); - ftpClient.quit(); ftpClient.disconnect(); ftpClient = null; } catch (Exception e) { diff --git a/server/user.jobengine.osgi.db/src/user/commons/pool/ContextPool.java b/server/user.jobengine.osgi.db/src/user/commons/pool/ContextPool.java index ab9e44fe..64504ee0 100644 --- a/server/user.jobengine.osgi.db/src/user/commons/pool/ContextPool.java +++ b/server/user.jobengine.osgi.db/src/user/commons/pool/ContextPool.java @@ -4,6 +4,7 @@ import java.lang.reflect.InvocationTargetException; import java.sql.Connection; import java.sql.DriverManager; import java.sql.ResultSet; +import java.sql.SQLException; import java.sql.Statement; import java.util.Properties; @@ -35,14 +36,27 @@ public class ContextPool extends ResourcePool { DefaultContext context = (DefaultContext) resourceObject; boolean result = false; String sql = "SELECT 1 FROM SYSIBM.SYSDUMMY1"; - try (Statement stmt = context.getConnection().createStatement()) { - try (ResultSet resultSet = stmt.executeQuery(sql)) { - result = true; - } catch (Exception e2) { - throw e2; - } + Statement stmt = null; + ResultSet rs = null; + try { + stmt = context.getConnection().createStatement(); + rs = stmt.executeQuery(sql); + result = true; } catch (Exception e1) { logger.catching(e1); + if (rs != null) + try { + rs.close(); + } catch (SQLException e) { + logger.catching(e); + } + if (stmt != null) + try { + stmt.close(); + } catch (SQLException e) { + logger.catching(e); + } + } return result; } @@ -91,13 +105,10 @@ public class ContextPool extends ResourcePool { try { if (context == null) return; - - if (context.isClosed() == false) { - context.close(); - } - context = null; + context.close(); } catch (Exception e) { - throw new PoolException(e.getMessage()); + logger.catching(e); + //throw new PoolException(e.getMessage()); } } diff --git a/server/user.jobengine.osgi.db/src/user/commons/pool/ResourcePool.java b/server/user.jobengine.osgi.db/src/user/commons/pool/ResourcePool.java index dec0e491..24e686d1 100644 --- a/server/user.jobengine.osgi.db/src/user/commons/pool/ResourcePool.java +++ b/server/user.jobengine.osgi.db/src/user/commons/pool/ResourcePool.java @@ -68,9 +68,10 @@ abstract public class ResourcePool { } } - if (resourceObject != null && checkObject(resourceObject) == true) { + if (resourceObject != null && !checkObject(resourceObject)) { + logger.debug("Resource not passed the check. Recreating."); + releaseObject(resourceObject); synchronized (this) { - logger.debug("Resource not passed the check. Recreating."); if (pool.contains(resourceObject)) pool.remove(resourceObject); resourceObject = createObject(); diff --git a/server/user.jobengine.osgi.db/src/user/jobengine/db/ItemManager.java b/server/user.jobengine.osgi.db/src/user/jobengine/db/ItemManager.java index 709444e1..26464c20 100644 --- a/server/user.jobengine.osgi.db/src/user/jobengine/db/ItemManager.java +++ b/server/user.jobengine.osgi.db/src/user/jobengine/db/ItemManager.java @@ -72,7 +72,7 @@ public class ItemManager extends MemoryCache implements IItemManager { private Map, IEntityBaseDAO> baseDAOs = new Hashtable, IEntityBaseDAO>(); private IChangeHandler itemTypeChangeFactory; - private int connectionPoolSize = 5; + private int connectionPoolSize = 10; private ContextPool contextPool = null; String dbDriver = "com.ibm.db2.jcc.DB2Driver"; @@ -84,6 +84,7 @@ public class ItemManager extends MemoryCache implements IItemManager { private SignalServer signalServer; private boolean useLucene; private IMediaFinder finder; + private Connection connection; public ItemManager() { initializeItemManager(signalServer); @@ -124,7 +125,7 @@ public class ItemManager extends MemoryCache implements IItemManager { @Override public void connect() { traceIn(); - ConnectionPoolProperties properties = new ConnectionPoolProperties(dbDriver, dbUrl, dbUser, dbPassword, false, 1000); + ConnectionPoolProperties properties = new ConnectionPoolProperties(dbDriver, dbUrl, dbUser, dbPassword, false, 2000); properties.add(ResourcePool.POOLSIZE, connectionPoolSize); contextPool = new ContextPool(properties); @@ -522,12 +523,13 @@ public class ItemManager extends MemoryCache implements IItemManager { @Override public void executeQuery(String query, IResultSetConsumer consumer, IStatementDecorator decorator) { DefaultContext context = getDbContext(); - Connection connection = context.getConnection(); try { + connection = context.getConnection(); executeQuery(connection, query, consumer, decorator); } catch (Exception e) { } finally { - putDbContext(context); + if (context != null) + putDbContext(context); } } @@ -560,8 +562,9 @@ public class ItemManager extends MemoryCache implements IItemManager { @Override public void executeUpdate(String query, IStatementDecorator decorator) { DefaultContext context = getDbContext(); - Connection connection = context.getConnection(); + Connection connection = null; try { + connection = context.getConnection(); executeUpdate(connection, query, decorator); } catch (Exception e) { } finally { diff --git a/server/user.jobengine.osgi.server/pages/joblist.zul b/server/user.jobengine.osgi.server/pages/joblist.zul index 0f1b6aa6..a8ed2672 100644 --- a/server/user.jobengine.osgi.server/pages/joblist.zul +++ b/server/user.jobengine.osgi.server/pages/joblist.zul @@ -1,15 +1,17 @@ - + + +
@@ -31,6 +33,7 @@ + @@ -38,22 +41,20 @@
- - - - - - - - - - - - - + + + + + + + + + + @@ -63,7 +64,7 @@ - + diff --git a/server/user.jobengine.osgi.server/pages/jobselector.zul b/server/user.jobengine.osgi.server/pages/jobselector.zul index a3980f00..ed19775e 100644 --- a/server/user.jobengine.osgi.server/pages/jobselector.zul +++ b/server/user.jobengine.osgi.server/pages/jobselector.zul @@ -80,6 +80,7 @@      +