From dfcd8afb9b6f440b2d900189452e40bd1aff33ef Mon Sep 17 00:00:00 2001 From: =?utf8?q?V=C3=A1s=C3=A1ry=20D=C3=A1niel?= Date: Thu, 12 Nov 2020 10:36:37 +0000 Subject: [PATCH] git-tfs-id: [http://tfs.userrendszerhaz.hu:8080/tfs/DefaultCollection]$/MediaCube;C32050 --- server/-modules/pom.xml | 1 - .../mediacube/executors/tests/SmallTests.java | 234 ++++++++++++++++++ .../jobtemplates/register-vod-restore.xml | 17 ++ .../server/steps/CreateMissingLowresStep.java | 2 +- .../server/steps/ForkUploadStep.java | 1 + .../server/steps/HSMMigrateStep.java | 10 +- .../server/steps/RegisterVODRestoreStep.java | 76 ++++++ .../jobengine/server/steps/TransferStep.java | 31 ++- .../server/steps/VODTransferToStep.java | 21 ++ .../src/user/commons/StoreUri.java | 36 ++- .../user/commons/remotestore/TSMLister.java | 31 ++- .../scripts/021_create_dropitem_sp.sql | 1 + .../025_create_dropitem_bymediafile_sp.sql | 39 +++ .../resources/i3-label_hu.properties | 2 +- 14 files changed, 469 insertions(+), 33 deletions(-) create mode 100644 server/user.jobengine.executors/jobtemplates/register-vod-restore.xml create mode 100644 server/user.jobengine.executors/src/user/jobengine/server/steps/RegisterVODRestoreStep.java create mode 100644 server/user.jobengine.executors/src/user/jobengine/server/steps/VODTransferToStep.java create mode 100644 server/user.jobengine.osgi.db/migrations/scripts/025_create_dropitem_bymediafile_sp.sql diff --git a/server/-modules/pom.xml b/server/-modules/pom.xml index 15af2c72..ebf9f9bb 100644 --- a/server/-modules/pom.xml +++ b/server/-modules/pom.xml @@ -14,7 +14,6 @@ ../user.jobengine.osgi.server ../user.jobengine.osgi.services ../user.commons.log4j2 - ../user.peablebeach.api ../user.tsm.client ../-product diff --git a/server/hu.user.mediacube.executors.tests/src/hu/user/mediacube/executors/tests/SmallTests.java b/server/hu.user.mediacube.executors.tests/src/hu/user/mediacube/executors/tests/SmallTests.java index 10a0267f..9127677a 100644 --- a/server/hu.user.mediacube.executors.tests/src/hu/user/mediacube/executors/tests/SmallTests.java +++ b/server/hu.user.mediacube.executors.tests/src/hu/user/mediacube/executors/tests/SmallTests.java @@ -8,6 +8,7 @@ import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.Paths; import java.sql.Timestamp; +import java.text.DecimalFormat; import java.time.Duration; import java.time.Instant; import java.time.LocalDateTime; @@ -51,6 +52,14 @@ public class SmallTests { class PojoRoot { } + public static String readableFileSize(long size) { + if (size <= 0) + return "0"; + final String[] units = new String[] { "B", "kB", "MB", "GB", "TB" }; + int digitGroups = (int) (Math.log10(size) / Math.log10(1024)); + return new DecimalFormat("#,##0.#").format(size / Math.pow(1024, digitGroups)) + " " + units[digitGroups]; + } + @Test public void test0() throws Exception { File f = new File("/opt/test.txt"); @@ -573,4 +582,229 @@ public class SmallTests { } } + + //mv migracio report + @Test + public void test995() throws Exception { + String tsm = new String(Files.readAllBytes(Paths.get("/opt/mv-tsm.txt"))); + List om = Files.readAllLines(Paths.get("/opt/mv-omarchive.txt")); + List exclude = Files.readAllLines(Paths.get("/opt/mv-migrate-exclude.txt")); + + int omCount = 0; + int archivedCount = 0; + int errorCount = 0; + for (String omFilePath : om) { + if (omFilePath.indexOf("/") == omFilePath.lastIndexOf("/")) { + System.out.println("Skipping dir " + omFilePath); + continue; + } + + omCount++; + + String fileName = omFilePath.trim().substring(omFilePath.lastIndexOf("/") + 1); + String fileId = fileName.substring(0, fileName.lastIndexOf(".")); + + boolean excludeContains = exclude.contains(fileId); + if (excludeContains) { + System.out.println("Skipping exclude " + omFilePath); + continue; + } + + boolean tsmContains = tsm.contains(fileName); + if (tsmContains) { + archivedCount++; + //System.out.println("Skipping archived " + omFilePath); + continue; + } + + System.out.println("Not archived " + omFilePath); + errorCount++; + } + float archivePercent = archivedCount * 100 / omCount; + System.out.println(String.format("Archiválandó %d, archivált %d, nem archivált %d, százalék %f", omCount, archivedCount, errorCount, archivePercent)); + // Files.write(Paths.get("/opt/AMC/mcmissing.txt"), mcmissing); + } + + //mv mc-tsm kulonbseg report + //select relativepath from mediafile; + // dsmadmc -id=support -password=userkft1q2 -TABdelimited "select LL_NAME from backups where FILESPACE_NAME LIKE '%JOBENGINE%'" > /opt/mv-tsm.txt + @Test + public void test996() throws Exception { + String tsm = new String(Files.readAllBytes(Paths.get("/opt/mv-tsm.txt"))); + List mc = Files.readAllLines(Paths.get("/opt/mv-mc.txt")); + List exclude = Files.readAllLines(Paths.get("/opt/PTV_all_ID.txt")); + List sb = new ArrayList<>(); + int errorCount = 0; + for (String fileName : mc) { + + //TODO idezojelek eltavolitasa + int lastIndexOf = fileName.lastIndexOf("-"); + if (lastIndexOf > -1) { + String fileId = fileName.substring(0, lastIndexOf); + if (exclude.contains(fileId)) { + //System.out.println("Excluded " + fileName); + continue; + } + } + + boolean tsmContains = tsm.contains(fileName); + if (!tsmContains) { + errorCount++; + sb.add(String.format("delete from mediafile where relativepath = ('%s')@", fileName)); + System.out.println("Missing " + fileName); + continue; + } + } + System.out.println(String.format("All %d, missing %d", mc.size(), errorCount)); + Files.write(Paths.get("/opt/mc-drop.sql"), sb); + } + + //mv exclude size by ID + @Test + public void test997() throws Exception { + List om = Files.readAllLines(Paths.get("/opt/mv-omarchive-size.txt")); + List exclude = Files.readAllLines(Paths.get("/opt/PTV_all_ID.txt")); + + int omExcludeCount = 0; + long omExcludeSizeSum = 0; + List out = new ArrayList<>(); + for (String omToken : om) { + int firstSpace = omToken.indexOf(" "); + String omPath = omToken.substring(firstSpace, omToken.length()); + int lastSlash = omPath.lastIndexOf("/"); + if (lastSlash < 0) + System.out.println(omPath); + if (omPath.indexOf("/") == lastSlash) { + System.out.println("Skipping dir " + omPath); + continue; + } + long omSize = Long.parseLong(omToken.substring(0, firstSpace)); + String fileName = omPath.substring(lastSlash + 1, omPath.length()); + int lastHyphen = fileName.lastIndexOf("-"); + String fileId = (lastHyphen < 0) ? fileName : fileName.substring(0, lastHyphen); + if (exclude.contains(fileId)) { + out.add(omPath); + //System.out.println(omPath); + omExcludeCount++; + omExcludeSizeSum += omSize; + } + + } + String excludedReport = String.format("Kihagyva %d db, összméret %s", omExcludeCount, readableFileSize(omExcludeSizeSum)); + System.out.println(excludedReport); + + Files.write(Paths.get("/opt/mv-excluded-parlament-byid.txt"), out); + } + + //mv exclude size by name + @Test + public void test998() throws Exception { + List om = Files.readAllLines(Paths.get("/opt/mv-omarchive-size.txt")); + + int omExcludeCount = 0; + long omExcludeSizeSum = 0; + List out = new ArrayList<>(); + for (String omToken : om) { + int firstSpace = omToken.indexOf(" "); + String omPath = omToken.substring(firstSpace, omToken.length()); + int lastSlash = omPath.lastIndexOf("/"); + if (lastSlash < 0) + System.out.println(omPath); + if (omPath.indexOf("/") == lastSlash) { + System.out.println("Skipping dir " + omPath); + continue; + } + long omSize = Long.parseLong(omToken.substring(0, firstSpace)); + String fileName = omPath.substring(lastSlash + 1, omPath.length()); + if (fileName.toLowerCase().contains("parlament")) { + out.add(omPath); + System.out.println(omPath); + omExcludeCount++; + omExcludeSizeSum += omSize; + } + + } + String excludedReport = String.format("%d db, összméret %s", omExcludeCount, readableFileSize(omExcludeSizeSum)); + System.out.println(excludedReport); + + Files.write(Paths.get("/opt/mv-excluded-parlament-name.txt"), out); + } + + //lastminute + @Test + public void test999() throws Exception { + List lastMinute = Files.readAllLines(Paths.get("/opt/mv-lastminute.txt")); + List exclude = Files.readAllLines(Paths.get("/opt/PTV_all_ID.txt")); + String tsm = new String(Files.readAllBytes(Paths.get("/opt/mv-tsm.txt"))); + String om = new String(Files.readAllBytes(Paths.get("/opt/mv-omarchive.txt"))); + + List out = new ArrayList<>(); + int count = 0; + for (String lastMinuteToken : lastMinute) { + String fileName = lastMinuteToken.substring(lastMinuteToken.lastIndexOf("/") + 1); + String fileId = fileName.substring(0, fileName.lastIndexOf("-")); + if (exclude.contains(fileId)) { + System.out.println("Excluded " + fileName); + continue; + } + if (!om.contains(fileName)) { + System.out.println("Not OM " + fileName); + continue; + } + + if (!tsm.contains(fileName)) { + //System.out.println("Missing " + fileName); + out.add(lastMinuteToken); + count++; + } + + } + System.out.println("All " + count); + // String excludedReport = String.format("Kihagyva %d db, összméret %s", omExcludeCount, readableFileSize(omExcludeSizeSum)); + // System.out.println(excludedReport); + // + Files.write(Paths.get("/opt/mv-lastminute-missing.txt"), out); + } + + //migration report + @Test + public void test9990() throws Exception { + List mc = Files.readAllLines(Paths.get("/opt/mv-mc-houseid.txt")); + List exclude = Files.readAllLines(Paths.get("/opt/PTV_all_ID.txt")); + List om = Files.readAllLines(Paths.get("/opt/mv-omarchive.txt")); + + List out = new ArrayList<>(); + int count = 0; + int omcount = 0; + for (String omToken : om) { + if (omToken.indexOf("/") == omToken.lastIndexOf("/")) { + //System.out.println("Skipping " + omToken); + continue; + } + + String fileName = omToken.substring(omToken.lastIndexOf("/") + 1); + int lastIndexOf = fileName.lastIndexOf("-"); + if (lastIndexOf > -1) { + String fileId = fileName.substring(0, lastIndexOf); + if (exclude.contains(fileId)) { + //System.out.println("Excluded " + fileName); + continue; + } + } + + omcount++; + if (!mc.contains(fileName)) { + //System.out.println("Missing " + fileName); + out.add(omToken); + count++; + continue; + } + } + System.out.println("All (no PTV) " + omcount + " missing " + count + " completed " + ((count * 100) / omcount) + "%"); + // String excludedReport = String.format("Kihagyva %d db, összméret %s", omExcludeCount, readableFileSize(omExcludeSizeSum)); + // System.out.println(excludedReport); + // + Files.write(Paths.get("/opt/mv-missing-noptv.txt"), out); + } + } diff --git a/server/user.jobengine.executors/jobtemplates/register-vod-restore.xml b/server/user.jobengine.executors/jobtemplates/register-vod-restore.xml new file mode 100644 index 00000000..6097d321 --- /dev/null +++ b/server/user.jobengine.executors/jobtemplates/register-vod-restore.xml @@ -0,0 +1,17 @@ + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/server/user.jobengine.executors/src/user/jobengine/server/steps/CreateMissingLowresStep.java b/server/user.jobengine.executors/src/user/jobengine/server/steps/CreateMissingLowresStep.java index dadb7f72..832b4df8 100644 --- a/server/user.jobengine.executors/src/user/jobengine/server/steps/CreateMissingLowresStep.java +++ b/server/user.jobengine.executors/src/user/jobengine/server/steps/CreateMissingLowresStep.java @@ -58,7 +58,7 @@ public class CreateMissingLowresStep extends JobStep { private Media getFirstUntranscodedMedia(IItemManager manager, DBCollection collection) { Media[] result = { null }; - String query = "SELECT mediaid FROM VW_MISSING_PROXY_IDS"; + String query = "SELECT mediaid FROM VW_MISSING_PROXY_IDS WHERE LCASE(HOUSEID) LIKE '%.mxf' OR LCASE(HOUSEID) LIKE '%.mov' ORDER BY modified DESC FETCH FIRST ROW ONLY"; manager.executeQuery(query, rs -> { try { long mediaId = rs.getLong(1); diff --git a/server/user.jobengine.executors/src/user/jobengine/server/steps/ForkUploadStep.java b/server/user.jobengine.executors/src/user/jobengine/server/steps/ForkUploadStep.java index d580e05f..67010d4c 100644 --- a/server/user.jobengine.executors/src/user/jobengine/server/steps/ForkUploadStep.java +++ b/server/user.jobengine.executors/src/user/jobengine/server/steps/ForkUploadStep.java @@ -162,6 +162,7 @@ public class ForkUploadStep extends JobStep { parameters.put("fileName", targetFileName); parameters.put("tempStoreUri", tempStoreUri); parameters.put("targetStoreUri", targetStoreUri); + parameters.put("relativeTargetPath", downloadable.getString("relativeTargetPath")); parameters.put("expectedFrameNumber", downloadable.getLong("frames")); parameters.put("expectedSize", downloadable.getLong("size")); parameters.put("expectedColorSpace", downloadable.getString("colorSpace")); diff --git a/server/user.jobengine.executors/src/user/jobengine/server/steps/HSMMigrateStep.java b/server/user.jobengine.executors/src/user/jobengine/server/steps/HSMMigrateStep.java index 507a804e..6b85f1d7 100644 --- a/server/user.jobengine.executors/src/user/jobengine/server/steps/HSMMigrateStep.java +++ b/server/user.jobengine.executors/src/user/jobengine/server/steps/HSMMigrateStep.java @@ -74,8 +74,10 @@ public class HSMMigrateStep extends JobStep { try { result = getPlanAirMetadata(mediaHouseId); } catch (Exception e) { + logger.error("PlanAir metadata error", e); - throw e; + //nem latja a drivert pl. + //throw e; } if (result == null) { @@ -283,7 +285,11 @@ public class HSMMigrateStep extends JobStep { } } } - if (successCopy) { + String metadataFileName = sourceFilePath.getFileName() + EscortFiles.DOT_JSON; + Path metadataPath = Paths.get(targetFilePath.getParent().toString(), EscortFiles.STATUSFOLDER, metadataFileName); + boolean createMetadata = Files.exists(targetFilePath) && !Files.exists(metadataPath); + + if (successCopy || createMetadata) { String metadata = null; try { metadata = createMetadata(volumeName, hsmFileName).toPrettyString(""); diff --git a/server/user.jobengine.executors/src/user/jobengine/server/steps/RegisterVODRestoreStep.java b/server/user.jobengine.executors/src/user/jobengine/server/steps/RegisterVODRestoreStep.java new file mode 100644 index 00000000..45ee400b --- /dev/null +++ b/server/user.jobengine.executors/src/user/jobengine/server/steps/RegisterVODRestoreStep.java @@ -0,0 +1,76 @@ +package user.jobengine.server.steps; + +import java.nio.file.Paths; + +import org.apache.commons.lang.StringUtils; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; + +import com.ibm.nosql.json.api.BasicDBList; +import com.ibm.nosql.json.api.BasicDBObject; + +import user.commons.DownloadableMedia; +import user.commons.StoreUri; +import user.commons.remotestore.RemoteStoreProtocol; +import user.jobengine.db.Media; + +public class RegisterVODRestoreStep extends JobStep { + private static final Logger logger = LogManager.getLogger(); + private static final String appendExtension = System.getProperty("missingmaterial.appendextension", ".mxf"); + + @StepEntry + public Object[] execute(BasicDBList basket) throws Exception { + try { + StoreUri sourceStoreUri = getManager().getStoreUri("TSM", RemoteStoreProtocol.TSM); + + if (true) + throw new Exception("Teszt success"); + + StoreUri targetStoreUri = getManager().getStore("FILEZILLA_PASARESTORE").getTargetStoreUri(); + StoreUri escortStoreUri = getManager().getStoreUri("MEDIACUBE_UPLOADS", RemoteStoreProtocol.LOCAL); + String targetStoreName = getManager().getStore(targetStoreUri.getStoreId()).getName(); + int processed = 0; + for (int i = 0; i < basket.size(); i++) { + BasicDBObject item = (BasicDBObject) basket.get(i); + processRecord(targetStoreName, sourceStoreUri, targetStoreUri, escortStoreUri, item); + processed++; + int progress = processed * 100 / basket.size(); + setProgress(progress); + } + } catch (Exception e) { + logger.error(getSessionMarker(), e.getMessage()); + throw e; + } finally { + } + + return null; + } + + private void processRecord(String targetStoreName, StoreUri sourceStoreUri, StoreUri targetStoreUri, StoreUri escortStoreUri, BasicDBObject item) { + String mediaTitle = item.getString("title"); + String relativeTargetPath = item.getString("path"); + + Media media = getManager().getMedia(mediaTitle); + String fileName = media.getTitle(); + if (StringUtils.isNotBlank(appendExtension)) + fileName += appendExtension; + + DownloadableMedia downloadable = DownloadableMedia.create(media.getTitle(), fileName, media.getModified(), media.getCreated(), media.getLength(), 0L, + sourceStoreUri.getId(), targetStoreUri.getId(), media.getId()); + downloadable.put("skipValidation", true); + downloadable.put("relativeTargetPath", relativeTargetPath); + String escortFileName = targetStoreName + "." + downloadable.getString("fileName"); + String outputPath = null; + try { + outputPath = Paths.get(escortStoreUri.toString(true)).toString(); + if (EscortFiles.createMetadataIfNotExists(outputPath, escortFileName, downloadable.toPrettyString(""))) + logger.info(getSessionMarker(), "Status file created {}", escortFileName); + else + logger.info(getSessionMarker(), "Status file already exists {}", escortFileName); + } catch (Exception e) { + logger.error("Can't create escort file {}", Paths.get(outputPath.toString(), escortFileName)); + } + + } + +} diff --git a/server/user.jobengine.executors/src/user/jobengine/server/steps/TransferStep.java b/server/user.jobengine.executors/src/user/jobengine/server/steps/TransferStep.java index 10270005..332b2c65 100644 --- a/server/user.jobengine.executors/src/user/jobengine/server/steps/TransferStep.java +++ b/server/user.jobengine.executors/src/user/jobengine/server/steps/TransferStep.java @@ -15,6 +15,10 @@ public class TransferStep extends JobStep { private static final Logger logger = LogManager.getLogger(); private static final boolean simulateTransferToTSM = Boolean.parseBoolean(System.getProperty("test.simulate.transfer.tsm", "false")); + protected StoreUri configureTargetUri(StoreUri targetStoreUri) { + return targetStoreUri; + } + @StepEntry public Object[] execute(StoreUri sourceStoreUri, String sourceFileName, StoreUri targetStoreUri, String targetFileName) throws Exception { try { @@ -33,19 +37,23 @@ public class TransferStep extends JobStep { } sourceStoreUri.addProgressListener(e -> setProgress(e.getProgress())); + + configureTargetUri(targetStoreUri); tryCopy(sourceStoreUri, sourceFileName, targetStoreUri, targetFileName); } catch (Exception e) { - logger.error(getMarker(), "Error in transfer of {} when copying from {} to {}. Retrying after 3 seconds.", sourceFileName, sourceStoreUri, - targetStoreUri); - - try { - Thread.sleep(3000); - tryCopy(sourceStoreUri, sourceFileName, targetStoreUri, targetFileName); - } catch (Exception e1) { - logger.error(getSessionMarker(), "Error in transfer of {} when copying from {} to {}. System message is: {}", sourceFileName, sourceStoreUri, - targetStoreUri, e1.getMessage()); - throw e1; - } + logger.error(getMarker(), "Error in transfer of {} when copying from {} to {}.", sourceFileName, sourceStoreUri, targetStoreUri); + throw e; + // logger.error(getMarker(), "Error in transfer of {} when copying from {} to {}. Retrying after 3 seconds.", sourceFileName, sourceStoreUri, + // targetStoreUri); + + // try { + // Thread.sleep(3000); + // tryCopy(sourceStoreUri, sourceFileName, targetStoreUri, targetFileName); + // } catch (Exception e1) { + // logger.error(getSessionMarker(), "Error in transfer of {} when copying from {} to {}. System message is: {}", sourceFileName, sourceStoreUri, + // targetStoreUri, e1.getMessage()); + // throw e1; + // } } finally { if (sourceStoreUri != null) sourceStoreUri.cleanUp(); @@ -66,6 +74,7 @@ public class TransferStep extends JobStep { private void tryCopy(StoreUri sourceStoreUri, String sourceFileName, StoreUri targetStoreUri, String targetFileName) throws Exception { String currentTargetFileName = targetFileName; + //TODO a .tmp masolas most csak local lehet, kellene ftp-re is, kiveve NEXIO boolean renameAfterCopy = false; if (getTmpExtension() != null && RemoteStoreProtocol.LOCAL.equals(targetStoreUri.getProtocol())) { currentTargetFileName += getTmpExtension(); diff --git a/server/user.jobengine.executors/src/user/jobengine/server/steps/VODTransferToStep.java b/server/user.jobengine.executors/src/user/jobengine/server/steps/VODTransferToStep.java new file mode 100644 index 00000000..7bf240fd --- /dev/null +++ b/server/user.jobengine.executors/src/user/jobengine/server/steps/VODTransferToStep.java @@ -0,0 +1,21 @@ +package user.jobengine.server.steps; + +import user.commons.StoreUri; + +public class VODTransferToStep extends TransferStep { + private String relativeTargetPath; + + @Override + protected StoreUri configureTargetUri(StoreUri targetStoreUri) { + targetStoreUri.setRootPath(relativeTargetPath); + return targetStoreUri; + } + + @StepEntry + public Object[] execute(StoreUri sourceStoreUri, String sourceFileName, StoreUri targetStoreUri, String relativeTargetPath, String targetFileName) + throws Exception { + this.relativeTargetPath = relativeTargetPath; + return super.execute(sourceStoreUri, sourceFileName, targetStoreUri, targetFileName); + } + +} diff --git a/server/user.jobengine.osgi.commons/src/user/commons/StoreUri.java b/server/user.jobengine.osgi.commons/src/user/commons/StoreUri.java index e7c0f8ad..3dd2019f 100644 --- a/server/user.jobengine.osgi.commons/src/user/commons/StoreUri.java +++ b/server/user.jobengine.osgi.commons/src/user/commons/StoreUri.java @@ -2,6 +2,7 @@ package user.commons; import java.io.BufferedInputStream; import java.io.BufferedOutputStream; +import java.io.FileNotFoundException; import java.io.InputStream; import java.io.OutputStream; import java.io.Serializable; @@ -26,6 +27,7 @@ import user.commons.remotestore.StatusEvent; @SuppressWarnings("serial") public class StoreUri extends EntityBase implements Serializable { private final static Logger logger = LogManager.getLogger(); + private static final String validateTransfers = System.getProperty("jobengine.validateTransfers", "false"); private final int BUFFER_SIZE = 32768; private String uri = null; @@ -45,6 +47,7 @@ public class StoreUri extends EntityBase implements Serializable { private EventListenerList progressListenerList; private EventListenerList statusListenerList; private int percent; + //TODO do not serialize, set transient private IDirectoryLister lister; private boolean showDirectories; //1-magas, n-alacsony @@ -100,14 +103,6 @@ public class StoreUri extends EntityBase implements Serializable { if (StringUtils.isEmpty(targetFileName)) targetFileName = source.getName(); - // Nexio feltöltésnél nem jó ha automatikusan másoljuk a kiterjesztést - // else { - // String name = source.getName(); - // int sep1 = name.lastIndexOf("."); - // int sep2 = targetFileName.lastIndexOf("."); - // if (sep2 == -1) - // targetFileName = targetFileName + name.substring(sep1); - // } result.setName(targetFileName); result.setSize(source.getSize()); @@ -146,6 +141,7 @@ public class StoreUri extends EntityBase implements Serializable { progressEvent.setProgress(100); fireProgressEvent(progressEvent); out.flush(); + } finally { try { if (out != null) @@ -162,6 +158,30 @@ public class StoreUri extends EntityBase implements Serializable { } } result.setSize(uploadedBytes); + + if ("true".equals(validateTransfers)) { + //validate + try { + RemoteFile remoteFile = outputLister.get(targetFileName); + if (remoteFile == null) + throw new FileNotFoundException("Remote file " + targetFileName + " not exists!"); + else + logger.info("File {} succesfully validated", targetName); + } catch (Exception e) { + logger.error("Error in copy validation! System message is: {}", e.getMessage()); + throw e; + } finally { + try { + if (outputLister != null) + outputLister.cleanUp(); + } catch (Exception e) { + logger.error("Error closing validation: {}", targetName); + throw e; + } + + } + } + return result; } diff --git a/server/user.jobengine.osgi.commons/src/user/commons/remotestore/TSMLister.java b/server/user.jobengine.osgi.commons/src/user/commons/remotestore/TSMLister.java index 266ace44..60513a3f 100644 --- a/server/user.jobengine.osgi.commons/src/user/commons/remotestore/TSMLister.java +++ b/server/user.jobengine.osgi.commons/src/user/commons/remotestore/TSMLister.java @@ -24,13 +24,10 @@ public class TSMLister implements IDirectoryLister { private static final String HLNAME = System.getProperty("tsm.hlname", "/JOBENGINE"); private static final Logger logger = LogManager.getLogger(); private TSMBufferedClient client; + private StoreUri storeUri; public TSMLister(StoreUri storeUri) throws Exception { - client = new TSMBufferedClient(NODENAME); - logger.info("TSMBufferedClient created"); - //client.connect(storeUri.getUserName(), storeUri.getPassword(), "\\"); - client.connect(storeUri.getUserName(), storeUri.getPassword()); - logger.info("TSMBufferedClient connected, parameters: {} {}/{} {}, separator: {}", NODENAME, FSNAME, ALTERNATE_FSNAME, HLNAME, File.separator); + this.storeUri = storeUri; } @Override @@ -41,7 +38,19 @@ public class TSMLister implements IDirectoryLister { @Override public void cleanUp() throws Exception { - client.disconnect(); + if (client != null) { + client.disconnect(); + client = null; + } + } + + private void connect() throws Exception { + if (client != null) + return; + client = new TSMBufferedClient(NODENAME); + //client.connect(storeUri.getUserName(), storeUri.getPassword(), "\\"); + client.connect(storeUri.getUserName(), storeUri.getPassword()); + logger.info("TSMBufferedClient connected, parameters: {} {}/{} {}, separator: {}", NODENAME, FSNAME, ALTERNATE_FSNAME, HLNAME, File.separator); } @Override @@ -64,15 +73,16 @@ public class TSMLister implements IDirectoryLister { @Override public RemoteFile get(String fileName) throws Exception { - RemoteFile result = null; + RemoteFile result = null; + connect(); String currentFileName = fileName; - logger.info("Getting {}, {}, {}", FSNAME, HLNAME, currentFileName); + logger.info("Getting {}, {}, {}", FSNAME, HLNAME, DELIMITER + currentFileName); TSMBackupFileObject backupFileObject = client.getActiveBackupFileObject(FSNAME, HLNAME, DELIMITER + currentFileName); //probaljuk meg a masik tarbol if (backupFileObject == null) { - logger.info("Getting {}, {}, {}", ALTERNATE_FSNAME, HLNAME, currentFileName); + logger.info("Getting {}, {}, {}", ALTERNATE_FSNAME, HLNAME, DELIMITER + currentFileName); backupFileObject = client.getActiveBackupFileObject(ALTERNATE_FSNAME, HLNAME, DELIMITER + currentFileName); } @@ -106,6 +116,7 @@ public class TSMLister implements IDirectoryLister { @Override public InputStream getInputStream(RemoteFile remoteFile) throws Exception { + connect(); TSMInputStream inputStream = new TSMInputStream(client, remoteFile); logger.info("TSM InputStream created"); inputStream.open(); @@ -120,6 +131,8 @@ public class TSMLister implements IDirectoryLister { // } catch (Exception e) { // logger.info(e.getMessage()); // } + + connect(); TSMOutputStream outputStream = new TSMOutputStream(client, FSNAME, HLNAME, DELIMITER, remoteFile); outputStream.open(); return outputStream; diff --git a/server/user.jobengine.osgi.db/migrations/scripts/021_create_dropitem_sp.sql b/server/user.jobengine.osgi.db/migrations/scripts/021_create_dropitem_sp.sql index 19ef07df..a5cbd976 100644 --- a/server/user.jobengine.osgi.db/migrations/scripts/021_create_dropitem_sp.sql +++ b/server/user.jobengine.osgi.db/migrations/scripts/021_create_dropitem_sp.sql @@ -1,3 +1,4 @@ + -- -- Copyright 2010-2016 the original author or authors. -- diff --git a/server/user.jobengine.osgi.db/migrations/scripts/025_create_dropitem_bymediafile_sp.sql b/server/user.jobengine.osgi.db/migrations/scripts/025_create_dropitem_bymediafile_sp.sql new file mode 100644 index 00000000..d88b2fe0 --- /dev/null +++ b/server/user.jobengine.osgi.db/migrations/scripts/025_create_dropitem_bymediafile_sp.sql @@ -0,0 +1,39 @@ +-- +-- Copyright 2010-2016 the original author or authors. +-- +-- Licensed under the Apache License, Version 2.0 (the "License"); +-- you may not use this file except in compliance with the License. +-- You may obtain a copy of the License at +-- +-- http://www.apache.org/licenses/LICENSE-2.0 +-- +-- Unless required by applicable law or agreed to in writing, software +-- distributed under the License is distributed on an "AS IS" BASIS, +-- WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +-- See the License for the specific language governing permissions and +-- limitations under the License. +-- + +-- // Create DROPITEMBYMF stored procedure +-- Migration SQL that makes the change goes here. + +CREATE PROCEDURE DROPITEMBYMF(vRELATIVEPATH VARCHAR(1000)) +LANGUAGE SQL +BEGIN + DECLARE vMEDIAFILEID BIGINT; + DECLARE vMEDIAID BIGINT; + DECLARE vITEMID BIGINT; + SELECT ID, MEDIAID INTO vMEDIAFILEID, vMEDIAID FROM MEDIAFILE WHERE RELATIVEPATH = vRELATIVEPATH; + SELECT ITEMID INTO vITEMID FROM MEDIA WHERE ID = vMEDIAID; + + delete from mediafile where id = vMEDIAFILEID; + delete from media where id = vMEDIAID; + delete from item where id = vITEMID; +END +@ + +-- //@UNDO +-- SQL to undo the change goes here. + +DROP PROCEDURE DROPITEMBYMF +@ diff --git a/server/user.jobengine.osgi.server/resources/i3-label_hu.properties b/server/user.jobengine.osgi.server/resources/i3-label_hu.properties index d29d6ef0..e478d064 100644 --- a/server/user.jobengine.osgi.server/resources/i3-label_hu.properties +++ b/server/user.jobengine.osgi.server/resources/i3-label_hu.properties @@ -1,4 +1,4 @@ -version=2.6.4 +version=2.6.5.1 footer=2016-2020 © Copyright User Rendszerház Kft. login_info=Információ -- 2.54.0