git-tfs-id: [http://tfs.userrendszerhaz.hu:8080/tfs/DefaultCollection]$/MediaCube...
authorVásáry Dániel <daniel.vasary@userrendszerhaz.hu>
Thu, 12 Nov 2020 10:36:37 +0000 (10:36 +0000)
committerVásáry Dániel <daniel.vasary@userrendszerhaz.hu>
Thu, 12 Nov 2020 10:36:37 +0000 (10:36 +0000)
14 files changed:
server/-modules/pom.xml
server/hu.user.mediacube.executors.tests/src/hu/user/mediacube/executors/tests/SmallTests.java
server/user.jobengine.executors/jobtemplates/register-vod-restore.xml [new file with mode: 0644]
server/user.jobengine.executors/src/user/jobengine/server/steps/CreateMissingLowresStep.java
server/user.jobengine.executors/src/user/jobengine/server/steps/ForkUploadStep.java
server/user.jobengine.executors/src/user/jobengine/server/steps/HSMMigrateStep.java
server/user.jobengine.executors/src/user/jobengine/server/steps/RegisterVODRestoreStep.java [new file with mode: 0644]
server/user.jobengine.executors/src/user/jobengine/server/steps/TransferStep.java
server/user.jobengine.executors/src/user/jobengine/server/steps/VODTransferToStep.java [new file with mode: 0644]
server/user.jobengine.osgi.commons/src/user/commons/StoreUri.java
server/user.jobengine.osgi.commons/src/user/commons/remotestore/TSMLister.java
server/user.jobengine.osgi.db/migrations/scripts/021_create_dropitem_sp.sql
server/user.jobengine.osgi.db/migrations/scripts/025_create_dropitem_bymediafile_sp.sql [new file with mode: 0644]
server/user.jobengine.osgi.server/resources/i3-label_hu.properties

index 15af2c72950b2d58c747cdba1f3ef48d12e1216a..ebf9f9bb9d685786b67be5eb96c2b013ab5ad512 100644 (file)
@@ -14,7 +14,6 @@
                <module>../user.jobengine.osgi.server</module>\r
                <module>../user.jobengine.osgi.services</module>\r
                <module>../user.commons.log4j2</module>\r
-               <module>../user.peablebeach.api</module>\r
                <module>../user.tsm.client</module>\r
                <module>../-product</module>\r
        </modules>\r
index 10a0267f11fdc7d540fe4c035f9a94bb61e48e52..9127677aa90fbc90a5e3ef269a73a7e7d0d439b4 100644 (file)
@@ -8,6 +8,7 @@ import java.nio.file.Files;
 import java.nio.file.Path;\r
 import java.nio.file.Paths;\r
 import java.sql.Timestamp;\r
+import java.text.DecimalFormat;\r
 import java.time.Duration;\r
 import java.time.Instant;\r
 import java.time.LocalDateTime;\r
@@ -51,6 +52,14 @@ public class SmallTests {
        class PojoRoot {\r
        }\r
 \r
+       public static String readableFileSize(long size) {\r
+               if (size <= 0)\r
+                       return "0";\r
+               final String[] units = new String[] { "B", "kB", "MB", "GB", "TB" };\r
+               int digitGroups = (int) (Math.log10(size) / Math.log10(1024));\r
+               return new DecimalFormat("#,##0.#").format(size / Math.pow(1024, digitGroups)) + " " + units[digitGroups];\r
+       }\r
+\r
        @Test\r
        public void test0() throws Exception {\r
                File f = new File("/opt/test.txt");\r
@@ -573,4 +582,229 @@ public class SmallTests {
                }\r
 \r
        }\r
+\r
+       //mv migracio report\r
+       @Test\r
+       public void test995() throws Exception {\r
+               String tsm = new String(Files.readAllBytes(Paths.get("/opt/mv-tsm.txt")));\r
+               List<String> om = Files.readAllLines(Paths.get("/opt/mv-omarchive.txt"));\r
+               List<String> exclude = Files.readAllLines(Paths.get("/opt/mv-migrate-exclude.txt"));\r
+\r
+               int omCount = 0;\r
+               int archivedCount = 0;\r
+               int errorCount = 0;\r
+               for (String omFilePath : om) {\r
+                       if (omFilePath.indexOf("/") == omFilePath.lastIndexOf("/")) {\r
+                               System.out.println("Skipping dir " + omFilePath);\r
+                               continue;\r
+                       }\r
+\r
+                       omCount++;\r
+\r
+                       String fileName = omFilePath.trim().substring(omFilePath.lastIndexOf("/") + 1);\r
+                       String fileId = fileName.substring(0, fileName.lastIndexOf("."));\r
+\r
+                       boolean excludeContains = exclude.contains(fileId);\r
+                       if (excludeContains) {\r
+                               System.out.println("Skipping exclude " + omFilePath);\r
+                               continue;\r
+                       }\r
+\r
+                       boolean tsmContains = tsm.contains(fileName);\r
+                       if (tsmContains) {\r
+                               archivedCount++;\r
+                               //System.out.println("Skipping archived " + omFilePath);\r
+                               continue;\r
+                       }\r
+\r
+                       System.out.println("Not archived " + omFilePath);\r
+                       errorCount++;\r
+               }\r
+               float archivePercent = archivedCount * 100 / omCount;\r
+               System.out.println(String.format("Archiválandó %d, archivált %d, nem archivált %d, százalék %f", omCount, archivedCount, errorCount, archivePercent));\r
+               //              Files.write(Paths.get("/opt/AMC/mcmissing.txt"), mcmissing);\r
+       }\r
+\r
+       //mv mc-tsm kulonbseg report\r
+       //select relativepath from mediafile;\r
+       // dsmadmc -id=support -password=userkft1q2 -TABdelimited "select LL_NAME from backups where FILESPACE_NAME LIKE '%JOBENGINE%'" > /opt/mv-tsm.txt\r
+       @Test\r
+       public void test996() throws Exception {\r
+               String tsm = new String(Files.readAllBytes(Paths.get("/opt/mv-tsm.txt")));\r
+               List<String> mc = Files.readAllLines(Paths.get("/opt/mv-mc.txt"));\r
+               List<String> exclude = Files.readAllLines(Paths.get("/opt/PTV_all_ID.txt"));\r
+               List<String> sb = new ArrayList<>();\r
+               int errorCount = 0;\r
+               for (String fileName : mc) {\r
+\r
+                       //TODO idezojelek eltavolitasa\r
+                       int lastIndexOf = fileName.lastIndexOf("-");\r
+                       if (lastIndexOf > -1) {\r
+                               String fileId = fileName.substring(0, lastIndexOf);\r
+                               if (exclude.contains(fileId)) {\r
+                                       //System.out.println("Excluded " + fileName);\r
+                                       continue;\r
+                               }\r
+                       }\r
+\r
+                       boolean tsmContains = tsm.contains(fileName);\r
+                       if (!tsmContains) {\r
+                               errorCount++;\r
+                               sb.add(String.format("delete from mediafile where relativepath = ('%s')@", fileName));\r
+                               System.out.println("Missing " + fileName);\r
+                               continue;\r
+                       }\r
+               }\r
+               System.out.println(String.format("All %d, missing %d", mc.size(), errorCount));\r
+               Files.write(Paths.get("/opt/mc-drop.sql"), sb);\r
+       }\r
+\r
+       //mv exclude size by ID\r
+       @Test\r
+       public void test997() throws Exception {\r
+               List<String> om = Files.readAllLines(Paths.get("/opt/mv-omarchive-size.txt"));\r
+               List<String> exclude = Files.readAllLines(Paths.get("/opt/PTV_all_ID.txt"));\r
+\r
+               int omExcludeCount = 0;\r
+               long omExcludeSizeSum = 0;\r
+               List<String> out = new ArrayList<>();\r
+               for (String omToken : om) {\r
+                       int firstSpace = omToken.indexOf(" ");\r
+                       String omPath = omToken.substring(firstSpace, omToken.length());\r
+                       int lastSlash = omPath.lastIndexOf("/");\r
+                       if (lastSlash < 0)\r
+                               System.out.println(omPath);\r
+                       if (omPath.indexOf("/") == lastSlash) {\r
+                               System.out.println("Skipping dir " + omPath);\r
+                               continue;\r
+                       }\r
+                       long omSize = Long.parseLong(omToken.substring(0, firstSpace));\r
+                       String fileName = omPath.substring(lastSlash + 1, omPath.length());\r
+                       int lastHyphen = fileName.lastIndexOf("-");\r
+                       String fileId = (lastHyphen < 0) ? fileName : fileName.substring(0, lastHyphen);\r
+                       if (exclude.contains(fileId)) {\r
+                               out.add(omPath);\r
+                               //System.out.println(omPath);\r
+                               omExcludeCount++;\r
+                               omExcludeSizeSum += omSize;\r
+                       }\r
+\r
+               }\r
+               String excludedReport = String.format("Kihagyva %d db, összméret %s", omExcludeCount, readableFileSize(omExcludeSizeSum));\r
+               System.out.println(excludedReport);\r
+\r
+               Files.write(Paths.get("/opt/mv-excluded-parlament-byid.txt"), out);\r
+       }\r
+\r
+       //mv exclude size by name\r
+       @Test\r
+       public void test998() throws Exception {\r
+               List<String> om = Files.readAllLines(Paths.get("/opt/mv-omarchive-size.txt"));\r
+\r
+               int omExcludeCount = 0;\r
+               long omExcludeSizeSum = 0;\r
+               List<String> out = new ArrayList<>();\r
+               for (String omToken : om) {\r
+                       int firstSpace = omToken.indexOf(" ");\r
+                       String omPath = omToken.substring(firstSpace, omToken.length());\r
+                       int lastSlash = omPath.lastIndexOf("/");\r
+                       if (lastSlash < 0)\r
+                               System.out.println(omPath);\r
+                       if (omPath.indexOf("/") == lastSlash) {\r
+                               System.out.println("Skipping dir " + omPath);\r
+                               continue;\r
+                       }\r
+                       long omSize = Long.parseLong(omToken.substring(0, firstSpace));\r
+                       String fileName = omPath.substring(lastSlash + 1, omPath.length());\r
+                       if (fileName.toLowerCase().contains("parlament")) {\r
+                               out.add(omPath);\r
+                               System.out.println(omPath);\r
+                               omExcludeCount++;\r
+                               omExcludeSizeSum += omSize;\r
+                       }\r
+\r
+               }\r
+               String excludedReport = String.format("%d db, összméret %s", omExcludeCount, readableFileSize(omExcludeSizeSum));\r
+               System.out.println(excludedReport);\r
+\r
+               Files.write(Paths.get("/opt/mv-excluded-parlament-name.txt"), out);\r
+       }\r
+\r
+       //lastminute\r
+       @Test\r
+       public void test999() throws Exception {\r
+               List<String> lastMinute = Files.readAllLines(Paths.get("/opt/mv-lastminute.txt"));\r
+               List<String> exclude = Files.readAllLines(Paths.get("/opt/PTV_all_ID.txt"));\r
+               String tsm = new String(Files.readAllBytes(Paths.get("/opt/mv-tsm.txt")));\r
+               String om = new String(Files.readAllBytes(Paths.get("/opt/mv-omarchive.txt")));\r
+\r
+               List<String> out = new ArrayList<>();\r
+               int count = 0;\r
+               for (String lastMinuteToken : lastMinute) {\r
+                       String fileName = lastMinuteToken.substring(lastMinuteToken.lastIndexOf("/") + 1);\r
+                       String fileId = fileName.substring(0, fileName.lastIndexOf("-"));\r
+                       if (exclude.contains(fileId)) {\r
+                               System.out.println("Excluded " + fileName);\r
+                               continue;\r
+                       }\r
+                       if (!om.contains(fileName)) {\r
+                               System.out.println("Not OM " + fileName);\r
+                               continue;\r
+                       }\r
+\r
+                       if (!tsm.contains(fileName)) {\r
+                               //System.out.println("Missing " + fileName);\r
+                               out.add(lastMinuteToken);\r
+                               count++;\r
+                       }\r
+\r
+               }\r
+               System.out.println("All " + count);\r
+               //              String excludedReport = String.format("Kihagyva %d db, összméret %s", omExcludeCount, readableFileSize(omExcludeSizeSum));\r
+               //              System.out.println(excludedReport);\r
+               //\r
+               Files.write(Paths.get("/opt/mv-lastminute-missing.txt"), out);\r
+       }\r
+\r
+       //migration report\r
+       @Test\r
+       public void test9990() throws Exception {\r
+               List<String> mc = Files.readAllLines(Paths.get("/opt/mv-mc-houseid.txt"));\r
+               List<String> exclude = Files.readAllLines(Paths.get("/opt/PTV_all_ID.txt"));\r
+               List<String> om = Files.readAllLines(Paths.get("/opt/mv-omarchive.txt"));\r
+\r
+               List<String> out = new ArrayList<>();\r
+               int count = 0;\r
+               int omcount = 0;\r
+               for (String omToken : om) {\r
+                       if (omToken.indexOf("/") == omToken.lastIndexOf("/")) {\r
+                               //System.out.println("Skipping " + omToken);\r
+                               continue;\r
+                       }\r
+\r
+                       String fileName = omToken.substring(omToken.lastIndexOf("/") + 1);\r
+                       int lastIndexOf = fileName.lastIndexOf("-");\r
+                       if (lastIndexOf > -1) {\r
+                               String fileId = fileName.substring(0, lastIndexOf);\r
+                               if (exclude.contains(fileId)) {\r
+                                       //System.out.println("Excluded " + fileName);\r
+                                       continue;\r
+                               }\r
+                       }\r
+\r
+                       omcount++;\r
+                       if (!mc.contains(fileName)) {\r
+                               //System.out.println("Missing " + fileName);\r
+                               out.add(omToken);\r
+                               count++;\r
+                               continue;\r
+                       }\r
+               }\r
+               System.out.println("All (no PTV) " + omcount + " missing " + count + " completed " + ((count * 100) / omcount) + "%");\r
+               //              String excludedReport = String.format("Kihagyva %d db, összméret %s", omExcludeCount, readableFileSize(omExcludeSizeSum));\r
+               //              System.out.println(excludedReport);\r
+               //\r
+               Files.write(Paths.get("/opt/mv-missing-noptv.txt"), out);\r
+       }\r
+\r
 }\r
diff --git a/server/user.jobengine.executors/jobtemplates/register-vod-restore.xml b/server/user.jobengine.executors/jobtemplates/register-vod-restore.xml
new file mode 100644 (file)
index 0000000..6097d32
--- /dev/null
@@ -0,0 +1,17 @@
+<?xml version="1.0" encoding="UTF-8"?>\r
+<jobtemplate useSessionLog="false">\r
+       <declarations>\r
+               <parameters>\r
+                       <parameter name="basket" type="com.ibm.nosql.json.api.BasicDBList" />\r
+               </parameters>\r
+       </declarations>\r
+       <commands>\r
+               <calljobstep type="user.jobengine.server.steps.RegisterVODRestoreStep" weight="1">\r
+                       <inputs>\r
+                               <input>\r
+                                       <parameter name="basket"/>\r
+                               </input>\r
+                       </inputs>\r
+               </calljobstep>\r
+       </commands>\r
+</jobtemplate>
\ No newline at end of file
index dadb7f722e95c60e91d3be1367b6f02e6b4023b8..832b4df8c35e7354a83ed5a258b1cbcdd18601a1 100644 (file)
@@ -58,7 +58,7 @@ public class CreateMissingLowresStep extends JobStep {
 \r
        private Media getFirstUntranscodedMedia(IItemManager manager, DBCollection collection) {\r
                Media[] result = { null };\r
-               String query = "SELECT mediaid FROM VW_MISSING_PROXY_IDS";\r
+               String query = "SELECT mediaid FROM VW_MISSING_PROXY_IDS WHERE LCASE(HOUSEID) LIKE '%.mxf' OR LCASE(HOUSEID) LIKE '%.mov' ORDER BY modified DESC FETCH FIRST ROW ONLY";\r
                manager.executeQuery(query, rs -> {\r
                        try {\r
                                long mediaId = rs.getLong(1);\r
index d580e05f2c6e640960d8629b2ab6c12dc542c0f7..67010d4c647e427e63c89787ec678e24e995ab59 100644 (file)
@@ -162,6 +162,7 @@ public class ForkUploadStep extends JobStep {
                        parameters.put("fileName", targetFileName);\r
                        parameters.put("tempStoreUri", tempStoreUri);\r
                        parameters.put("targetStoreUri", targetStoreUri);\r
+                       parameters.put("relativeTargetPath", downloadable.getString("relativeTargetPath"));\r
                        parameters.put("expectedFrameNumber", downloadable.getLong("frames"));\r
                        parameters.put("expectedSize", downloadable.getLong("size"));\r
                        parameters.put("expectedColorSpace", downloadable.getString("colorSpace"));\r
index 507a804e94cbd108dd190e402b451b559ac7a270..6b85f1d7bf300c7914240873494423c7ec9f1050 100644 (file)
@@ -74,8 +74,10 @@ public class HSMMigrateStep extends JobStep {
                try {\r
                        result = getPlanAirMetadata(mediaHouseId);\r
                } catch (Exception e) {\r
+\r
                        logger.error("PlanAir metadata error", e);\r
-                       throw e;\r
+                       //nem latja a drivert pl.\r
+                       //throw e;\r
                }\r
 \r
                if (result == null) {\r
@@ -283,7 +285,11 @@ public class HSMMigrateStep extends JobStep {
                                }\r
                        }\r
                }\r
-               if (successCopy) {\r
+               String metadataFileName = sourceFilePath.getFileName() + EscortFiles.DOT_JSON;\r
+               Path metadataPath = Paths.get(targetFilePath.getParent().toString(), EscortFiles.STATUSFOLDER, metadataFileName);\r
+               boolean createMetadata = Files.exists(targetFilePath) && !Files.exists(metadataPath);\r
+\r
+               if (successCopy || createMetadata) {\r
                        String metadata = null;\r
                        try {\r
                                metadata = createMetadata(volumeName, hsmFileName).toPrettyString("");\r
diff --git a/server/user.jobengine.executors/src/user/jobengine/server/steps/RegisterVODRestoreStep.java b/server/user.jobengine.executors/src/user/jobengine/server/steps/RegisterVODRestoreStep.java
new file mode 100644 (file)
index 0000000..45ee400
--- /dev/null
@@ -0,0 +1,76 @@
+package user.jobengine.server.steps;\r
+\r
+import java.nio.file.Paths;\r
+\r
+import org.apache.commons.lang.StringUtils;\r
+import org.apache.logging.log4j.LogManager;\r
+import org.apache.logging.log4j.Logger;\r
+\r
+import com.ibm.nosql.json.api.BasicDBList;\r
+import com.ibm.nosql.json.api.BasicDBObject;\r
+\r
+import user.commons.DownloadableMedia;\r
+import user.commons.StoreUri;\r
+import user.commons.remotestore.RemoteStoreProtocol;\r
+import user.jobengine.db.Media;\r
+\r
+public class RegisterVODRestoreStep extends JobStep {\r
+       private static final Logger logger = LogManager.getLogger();\r
+       private static final String appendExtension = System.getProperty("missingmaterial.appendextension", ".mxf");\r
+\r
+       @StepEntry\r
+       public Object[] execute(BasicDBList basket) throws Exception {\r
+               try {\r
+                       StoreUri sourceStoreUri = getManager().getStoreUri("TSM", RemoteStoreProtocol.TSM);\r
+\r
+                       if (true)\r
+                               throw new Exception("Teszt success");\r
+\r
+                       StoreUri targetStoreUri = getManager().getStore("FILEZILLA_PASARESTORE").getTargetStoreUri();\r
+                       StoreUri escortStoreUri = getManager().getStoreUri("MEDIACUBE_UPLOADS", RemoteStoreProtocol.LOCAL);\r
+                       String targetStoreName = getManager().getStore(targetStoreUri.getStoreId()).getName();\r
+                       int processed = 0;\r
+                       for (int i = 0; i < basket.size(); i++) {\r
+                               BasicDBObject item = (BasicDBObject) basket.get(i);\r
+                               processRecord(targetStoreName, sourceStoreUri, targetStoreUri, escortStoreUri, item);\r
+                               processed++;\r
+                               int progress = processed * 100 / basket.size();\r
+                               setProgress(progress);\r
+                       }\r
+               } catch (Exception e) {\r
+                       logger.error(getSessionMarker(), e.getMessage());\r
+                       throw e;\r
+               } finally {\r
+               }\r
+\r
+               return null;\r
+       }\r
+\r
+       private void processRecord(String targetStoreName, StoreUri sourceStoreUri, StoreUri targetStoreUri, StoreUri escortStoreUri, BasicDBObject item) {\r
+               String mediaTitle = item.getString("title");\r
+               String relativeTargetPath = item.getString("path");\r
+\r
+               Media media = getManager().getMedia(mediaTitle);\r
+               String fileName = media.getTitle();\r
+               if (StringUtils.isNotBlank(appendExtension))\r
+                       fileName += appendExtension;\r
+\r
+               DownloadableMedia downloadable = DownloadableMedia.create(media.getTitle(), fileName, media.getModified(), media.getCreated(), media.getLength(), 0L,\r
+                               sourceStoreUri.getId(), targetStoreUri.getId(), media.getId());\r
+               downloadable.put("skipValidation", true);\r
+               downloadable.put("relativeTargetPath", relativeTargetPath);\r
+               String escortFileName = targetStoreName + "." + downloadable.getString("fileName");\r
+               String outputPath = null;\r
+               try {\r
+                       outputPath = Paths.get(escortStoreUri.toString(true)).toString();\r
+                       if (EscortFiles.createMetadataIfNotExists(outputPath, escortFileName, downloadable.toPrettyString("")))\r
+                               logger.info(getSessionMarker(), "Status file created {}", escortFileName);\r
+                       else\r
+                               logger.info(getSessionMarker(), "Status file already exists {}", escortFileName);\r
+               } catch (Exception e) {\r
+                       logger.error("Can't create escort file {}", Paths.get(outputPath.toString(), escortFileName));\r
+               }\r
+\r
+       }\r
+\r
+}\r
index 10270005a546d402ad4ee13523a95c8aa2e1bc1c..332b2c65b7fca5e91f64b35477103f2330ea2162 100644 (file)
@@ -15,6 +15,10 @@ public class TransferStep extends JobStep {
        private static final Logger logger = LogManager.getLogger();\r
        private static final boolean simulateTransferToTSM = Boolean.parseBoolean(System.getProperty("test.simulate.transfer.tsm", "false"));\r
 \r
+       protected StoreUri configureTargetUri(StoreUri targetStoreUri) {\r
+               return targetStoreUri;\r
+       }\r
+\r
        @StepEntry\r
        public Object[] execute(StoreUri sourceStoreUri, String sourceFileName, StoreUri targetStoreUri, String targetFileName) throws Exception {\r
                try {\r
@@ -33,19 +37,23 @@ public class TransferStep extends JobStep {
                        }\r
 \r
                        sourceStoreUri.addProgressListener(e -> setProgress(e.getProgress()));\r
+\r
+                       configureTargetUri(targetStoreUri);\r
                        tryCopy(sourceStoreUri, sourceFileName, targetStoreUri, targetFileName);\r
                } catch (Exception e) {\r
-                       logger.error(getMarker(), "Error in transfer of {} when copying from {} to {}. Retrying after 3 seconds.", sourceFileName, sourceStoreUri,\r
-                                       targetStoreUri);\r
-\r
-                       try {\r
-                               Thread.sleep(3000);\r
-                               tryCopy(sourceStoreUri, sourceFileName, targetStoreUri, targetFileName);\r
-                       } catch (Exception e1) {\r
-                               logger.error(getSessionMarker(), "Error in transfer of {} when copying from {} to {}. System message is: {}", sourceFileName, sourceStoreUri,\r
-                                               targetStoreUri, e1.getMessage());\r
-                               throw e1;\r
-                       }\r
+                       logger.error(getMarker(), "Error in transfer of {} when copying from {} to {}.", sourceFileName, sourceStoreUri, targetStoreUri);\r
+                       throw e;\r
+                       //                      logger.error(getMarker(), "Error in transfer of {} when copying from {} to {}. Retrying after 3 seconds.", sourceFileName, sourceStoreUri,\r
+                       //                                      targetStoreUri);\r
+\r
+                       //                      try {\r
+                       //                              Thread.sleep(3000);\r
+                       //                              tryCopy(sourceStoreUri, sourceFileName, targetStoreUri, targetFileName);\r
+                       //                      } catch (Exception e1) {\r
+                       //                              logger.error(getSessionMarker(), "Error in transfer of {} when copying from {} to {}. System message is: {}", sourceFileName, sourceStoreUri,\r
+                       //                                              targetStoreUri, e1.getMessage());\r
+                       //                              throw e1;\r
+                       //                      }\r
                } finally {\r
                        if (sourceStoreUri != null)\r
                                sourceStoreUri.cleanUp();\r
@@ -66,6 +74,7 @@ public class TransferStep extends JobStep {
        private void tryCopy(StoreUri sourceStoreUri, String sourceFileName, StoreUri targetStoreUri, String targetFileName) throws Exception {\r
                String currentTargetFileName = targetFileName;\r
 \r
+               //TODO a .tmp masolas most csak local lehet, kellene ftp-re is, kiveve NEXIO\r
                boolean renameAfterCopy = false;\r
                if (getTmpExtension() != null && RemoteStoreProtocol.LOCAL.equals(targetStoreUri.getProtocol())) {\r
                        currentTargetFileName += getTmpExtension();\r
diff --git a/server/user.jobengine.executors/src/user/jobengine/server/steps/VODTransferToStep.java b/server/user.jobengine.executors/src/user/jobengine/server/steps/VODTransferToStep.java
new file mode 100644 (file)
index 0000000..7bf240f
--- /dev/null
@@ -0,0 +1,21 @@
+package user.jobengine.server.steps;\r
+\r
+import user.commons.StoreUri;\r
+\r
+public class VODTransferToStep extends TransferStep {\r
+       private String relativeTargetPath;\r
+\r
+       @Override\r
+       protected StoreUri configureTargetUri(StoreUri targetStoreUri) {\r
+               targetStoreUri.setRootPath(relativeTargetPath);\r
+               return targetStoreUri;\r
+       }\r
+\r
+       @StepEntry\r
+       public Object[] execute(StoreUri sourceStoreUri, String sourceFileName, StoreUri targetStoreUri, String relativeTargetPath, String targetFileName)\r
+                       throws Exception {\r
+               this.relativeTargetPath = relativeTargetPath;\r
+               return super.execute(sourceStoreUri, sourceFileName, targetStoreUri, targetFileName);\r
+       }\r
+\r
+}\r
index e7c0f8ad3bfd526a7f08253912b833f9eaeceea1..3dd2019fe26939c798f1734d3dce5c515e6ae021 100644 (file)
@@ -2,6 +2,7 @@ package user.commons;
 
 import java.io.BufferedInputStream;
 import java.io.BufferedOutputStream;
+import java.io.FileNotFoundException;
 import java.io.InputStream;
 import java.io.OutputStream;
 import java.io.Serializable;
@@ -26,6 +27,7 @@ import user.commons.remotestore.StatusEvent;
 @SuppressWarnings("serial")
 public class StoreUri extends EntityBase implements Serializable {
        private final static Logger logger = LogManager.getLogger();
+       private static final String validateTransfers = System.getProperty("jobengine.validateTransfers", "false");
 
        private final int BUFFER_SIZE = 32768;
        private String uri = null;
@@ -45,6 +47,7 @@ public class StoreUri extends EntityBase implements Serializable {
        private EventListenerList progressListenerList;
        private EventListenerList statusListenerList;
        private int percent;
+       //TODO do not serialize, set transient
        private IDirectoryLister lister;
        private boolean showDirectories;
        //1-magas, n-alacsony
@@ -100,14 +103,6 @@ public class StoreUri extends EntityBase implements Serializable {
                if (StringUtils.isEmpty(targetFileName))
                        targetFileName = source.getName();
 
-               // Nexio feltöltésnél nem jó ha automatikusan másoljuk a kiterjesztést
-               //              else {
-               //                      String name = source.getName();
-               //                      int sep1 = name.lastIndexOf(".");
-               //                      int sep2 = targetFileName.lastIndexOf(".");
-               //                      if (sep2 == -1)
-               //                              targetFileName = targetFileName + name.substring(sep1);
-               //              }
                result.setName(targetFileName);
                result.setSize(source.getSize());
 
@@ -146,6 +141,7 @@ public class StoreUri extends EntityBase implements Serializable {
                        progressEvent.setProgress(100);
                        fireProgressEvent(progressEvent);
                        out.flush();
+
                } finally {
                        try {
                                if (out != null)
@@ -162,6 +158,30 @@ public class StoreUri extends EntityBase implements Serializable {
                        }
                }
                result.setSize(uploadedBytes);
+
+               if ("true".equals(validateTransfers)) {
+                       //validate
+                       try {
+                               RemoteFile remoteFile = outputLister.get(targetFileName);
+                               if (remoteFile == null)
+                                       throw new FileNotFoundException("Remote file " + targetFileName + " not exists!");
+                               else
+                                       logger.info("File {} succesfully validated", targetName);
+                       } catch (Exception e) {
+                               logger.error("Error in copy validation! System message is: {}", e.getMessage());
+                               throw e;
+                       } finally {
+                               try {
+                                       if (outputLister != null)
+                                               outputLister.cleanUp();
+                               } catch (Exception e) {
+                                       logger.error("Error closing validation: {}", targetName);
+                                       throw e;
+                               }
+
+                       }
+               }
+
                return result;
        }
 
index 266ace4430d57e0c72d088958e821bd4d017b091..60513a3f32f807e86bbb8f7267713106a691bb29 100644 (file)
@@ -24,13 +24,10 @@ public class TSMLister implements IDirectoryLister {
        private static final String HLNAME = System.getProperty("tsm.hlname", "/JOBENGINE");\r
        private static final Logger logger = LogManager.getLogger();\r
        private TSMBufferedClient client;\r
+       private StoreUri storeUri;\r
 \r
        public TSMLister(StoreUri storeUri) throws Exception {\r
-               client = new TSMBufferedClient(NODENAME);\r
-               logger.info("TSMBufferedClient created");\r
-               //client.connect(storeUri.getUserName(), storeUri.getPassword(), "\\");\r
-               client.connect(storeUri.getUserName(), storeUri.getPassword());\r
-               logger.info("TSMBufferedClient connected, parameters: {} {}/{} {}, separator: {}", NODENAME, FSNAME, ALTERNATE_FSNAME, HLNAME, File.separator);\r
+               this.storeUri = storeUri;\r
        }\r
 \r
        @Override\r
@@ -41,7 +38,19 @@ public class TSMLister implements IDirectoryLister {
 \r
        @Override\r
        public void cleanUp() throws Exception {\r
-               client.disconnect();\r
+               if (client != null) {\r
+                       client.disconnect();\r
+                       client = null;\r
+               }\r
+       }\r
+\r
+       private void connect() throws Exception {\r
+               if (client != null)\r
+                       return;\r
+               client = new TSMBufferedClient(NODENAME);\r
+               //client.connect(storeUri.getUserName(), storeUri.getPassword(), "\\");\r
+               client.connect(storeUri.getUserName(), storeUri.getPassword());\r
+               logger.info("TSMBufferedClient connected, parameters: {} {}/{} {}, separator: {}", NODENAME, FSNAME, ALTERNATE_FSNAME, HLNAME, File.separator);\r
        }\r
 \r
        @Override\r
@@ -64,15 +73,16 @@ public class TSMLister implements IDirectoryLister {
 \r
        @Override\r
        public RemoteFile get(String fileName) throws Exception {\r
-               RemoteFile result = null;\r
 \r
+               RemoteFile result = null;\r
+               connect();\r
                String currentFileName = fileName;\r
-               logger.info("Getting {}, {}, {}", FSNAME, HLNAME, currentFileName);\r
+               logger.info("Getting {}, {}, {}", FSNAME, HLNAME, DELIMITER + currentFileName);\r
                TSMBackupFileObject backupFileObject = client.getActiveBackupFileObject(FSNAME, HLNAME, DELIMITER + currentFileName);\r
 \r
                //probaljuk meg a masik tarbol\r
                if (backupFileObject == null) {\r
-                       logger.info("Getting {}, {}, {}", ALTERNATE_FSNAME, HLNAME, currentFileName);\r
+                       logger.info("Getting {}, {}, {}", ALTERNATE_FSNAME, HLNAME, DELIMITER + currentFileName);\r
                        backupFileObject = client.getActiveBackupFileObject(ALTERNATE_FSNAME, HLNAME, DELIMITER + currentFileName);\r
                }\r
 \r
@@ -106,6 +116,7 @@ public class TSMLister implements IDirectoryLister {
 \r
        @Override\r
        public InputStream getInputStream(RemoteFile remoteFile) throws Exception {\r
+               connect();\r
                TSMInputStream inputStream = new TSMInputStream(client, remoteFile);\r
                logger.info("TSM InputStream created");\r
                inputStream.open();\r
@@ -120,6 +131,8 @@ public class TSMLister implements IDirectoryLister {
                //              } catch (Exception e) {\r
                //                      logger.info(e.getMessage());\r
                //              }\r
+\r
+               connect();\r
                TSMOutputStream outputStream = new TSMOutputStream(client, FSNAME, HLNAME, DELIMITER, remoteFile);\r
                outputStream.open();\r
                return outputStream;\r
index 19ef07df2e8cde96bdecb58db9e00c2e5f1ba0c9..a5cbd976d85f0862e45d02e79c081eef7d4a085b 100644 (file)
@@ -1,3 +1,4 @@
+
 --
 --    Copyright 2010-2016 the original author or authors.
 --
diff --git a/server/user.jobengine.osgi.db/migrations/scripts/025_create_dropitem_bymediafile_sp.sql b/server/user.jobengine.osgi.db/migrations/scripts/025_create_dropitem_bymediafile_sp.sql
new file mode 100644 (file)
index 0000000..d88b2fe
--- /dev/null
@@ -0,0 +1,39 @@
+--
+--    Copyright 2010-2016 the original author or authors.
+--
+--    Licensed under the Apache License, Version 2.0 (the "License");
+--    you may not use this file except in compliance with the License.
+--    You may obtain a copy of the License at
+--
+--       http://www.apache.org/licenses/LICENSE-2.0
+--
+--    Unless required by applicable law or agreed to in writing, software
+--    distributed under the License is distributed on an "AS IS" BASIS,
+--    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+--    See the License for the specific language governing permissions and
+--    limitations under the License.
+--
+
+-- // Create DROPITEMBYMF stored procedure
+-- Migration SQL that makes the change goes here.
+
+CREATE PROCEDURE DROPITEMBYMF(vRELATIVEPATH VARCHAR(1000))
+LANGUAGE SQL
+BEGIN
+       DECLARE vMEDIAFILEID BIGINT;
+       DECLARE vMEDIAID BIGINT;
+       DECLARE vITEMID BIGINT;
+       SELECT ID, MEDIAID INTO vMEDIAFILEID, vMEDIAID FROM MEDIAFILE WHERE RELATIVEPATH = vRELATIVEPATH;
+       SELECT ITEMID INTO vITEMID FROM MEDIA WHERE ID = vMEDIAID;
+
+       delete from mediafile where id = vMEDIAFILEID;
+       delete from media where id = vMEDIAID;
+       delete from item where id = vITEMID;
+END
+@
+
+-- //@UNDO
+-- SQL to undo the change goes here.
+
+DROP PROCEDURE DROPITEMBYMF
+@
index d29d6ef07313cebcd838e26324f7a977135df1f2..e478d064b465d2bc1142b8e21f29f0501c0e31fa 100644 (file)
@@ -1,4 +1,4 @@
-version=2.6.4\r
+version=2.6.5.1\r
 footer=2016-2020 © Copyright User Rendszerház Kft.\r
 \r
 login_info=Információ\r