From: Vásáry Dániel Date: Fri, 17 Jul 2020 09:11:25 +0000 (+0000) Subject: git-tfs-id: [http://tfs.userrendszerhaz.hu:8080/tfs/DefaultCollection]$/MediaCube... X-Git-Url: http://git.useribm.hu/?a=commitdiff_plain;h=0a3cad9750b98726d2d9212cfbbeee269340ca7c;p=mediacube.git git-tfs-id: [tfs.userrendszerhaz.hu:8080/tfs/DefaultCollection]$/MediaCube;C31901 --- diff --git a/server/-modules/pom.xml b/server/-modules/pom.xml index ebf9f9bb..15af2c72 100644 --- a/server/-modules/pom.xml +++ b/server/-modules/pom.xml @@ -14,6 +14,7 @@ ../user.jobengine.osgi.server ../user.jobengine.osgi.services ../user.commons.log4j2 + ../user.peablebeach.api ../user.tsm.client ../-product diff --git a/server/-product/mediacube.product b/server/-product/mediacube.product index 6abffb7b..992a0bf5 100644 --- a/server/-product/mediacube.product +++ b/server/-product/mediacube.product @@ -160,7 +160,6 @@ - diff --git a/server/user.jobengine.executors/META-INF/MANIFEST.MF b/server/user.jobengine.executors/META-INF/MANIFEST.MF index f2c88b3d..f3bff09b 100644 --- a/server/user.jobengine.executors/META-INF/MANIFEST.MF +++ b/server/user.jobengine.executors/META-INF/MANIFEST.MF @@ -26,10 +26,22 @@ Require-Bundle: user.jobengine.osgi.commons;bundle-version="1.0.0", Bundle-ClassPath: . Bundle-RequiredExecutionEnvironment: JavaSE-1.8 Import-Package: groovy.lang;version="3.0.3", + hu.user.peablebeach.api, javax.ws.rs.client, javax.ws.rs.core;version="2.0.1", org.apache.commons.io;version="2.2.0", org.apache.commons.net.ftp;version="3.6.0", + org.apache.http, + org.apache.http.auth, + org.apache.http.client, + org.apache.http.client.methods, + org.apache.http.client.utils, + org.apache.http.conn, + org.apache.http.entity, + org.apache.http.impl.client, + org.apache.http.message, + org.apache.http.params, + org.apache.http.protocol, org.apache.logging.log4j;version="2.8.2", org.apache.logging.log4j.message;version="2.8.2", org.joda.time;version="2.2.0", diff --git a/server/user.jobengine.executors/src/user/jobengine/server/steps/PeableBeachMissingMaterialCheckerStep.java b/server/user.jobengine.executors/src/user/jobengine/server/steps/PeableBeachMissingMaterialCheckerStep.java index 3c53af0f..f66aa8a2 100644 --- a/server/user.jobengine.executors/src/user/jobengine/server/steps/PeableBeachMissingMaterialCheckerStep.java +++ b/server/user.jobengine.executors/src/user/jobengine/server/steps/PeableBeachMissingMaterialCheckerStep.java @@ -12,7 +12,6 @@ import org.apache.logging.log4j.Logger; import user.commons.DownloadableMedia; import user.commons.RemoteFile; import user.commons.StoreUri; -import user.commons.peablebeach.PBMissingMaterialSrc; import user.commons.remotestore.RemoteStoreProtocol; import user.jobengine.db.Media; @@ -26,26 +25,40 @@ public class PeableBeachMissingMaterialCheckerStep extends JobStep { StoreUri escortStoreUri = null; try { escortStoreUri = getManager().getStoreUri(escortStoreName, RemoteStoreProtocol.LOCAL); + StoreUri sourceStoreUri = getManager().getStoreUri("TSM", RemoteStoreProtocol.TSM); StoreUri targetStoreUri = getManager().getStoreUri(targetStoreName, Enum.valueOf(RemoteStoreProtocol.class, targetProtocol)); - PBMissingMaterialSrc source = new PBMissingMaterialSrc(); - source.init(primaryEndPoint, primaryUserName, primaryPassword, secondaryEndPoint, secondaryUserName, secondaryPassword, rangeForwardHours); + List poolContent = new ArrayList<>(); - List remoteFiles = targetStoreUri.getRemoteFiles(); + PBQuery query = new PBQuery(); + query.init(primaryEndPoint, primaryUserName, primaryPassword, secondaryEndPoint, secondaryUserName, secondaryPassword, rangeForwardHours); + LinkedHashSet titles = query.getPossibelMissingMaterialNames(poolContent); + logger.info(getMarker(), "API returned {} items", titles == null ? 0 : titles.size()); + + if (titles != null && titles.size() > 0) { + + List remoteFiles = targetStoreUri.getRemoteFiles(); + if (remoteFiles != null) + remoteFiles.forEach(i -> { + String name = i.getName(); + poolContent.add(name.substring(0, name.lastIndexOf("."))); + }); + + logger.info(getMarker(), "Pool contains {} items", poolContent == null ? 0 : poolContent.size()); + + if (!getJobRuntime().isWaitingCancel()) { + int count = titles.size(); + int processed = 0; + for (String title : titles) { + processRecord(title, sourceStoreUri, targetStoreName, targetStoreUri, escortStoreUri); + processed++; + int progress = processed * 100 / count; + setProgress(progress); + } + } - List poolContent = new ArrayList<>(); - remoteFiles.forEach(i -> { - String fileName = i.getName(); - String title = fileName.substring(0, fileName.lastIndexOf(".")); - poolContent.add(title); - }); - logger.info("Pool contains {} items", poolContent.size()); - LinkedHashSet fileNames = source.getPossibelMissingMaterialNames(poolContent); - logger.info("API returns {} items", fileNames.size()); - - for (String fileName : fileNames) { - processRecord(fileName, targetStoreName, targetStoreUri, escortStoreUri); } + } catch (Exception e) { logger.error(getSessionMarker(), e.getMessage()); throw e; @@ -57,30 +70,39 @@ public class PeableBeachMissingMaterialCheckerStep extends JobStep { return null; } - private void processRecord(String fileName, String targetStoreName, StoreUri targetStoreUri, StoreUri escortStoreUri) { - String title = fileName.substring(0, fileName.lastIndexOf(".")); - + private void processRecord(String title, StoreUri sourceStoreUri, String targetStoreName, StoreUri targetStoreUri, StoreUri escortStoreUri) { + //TODO kisbetu/nagybetu problema kezelese Media media = getManager().getMedia(title); - if (StringUtils.isNotBlank(appendExtension)) - fileName += appendExtension; + String fileName = normalizeName(title); if (media == null) { - logger.error(getSessionMarker(), "File {} not archived yet", fileName); + logger.error(getMarker(), "File {} not archived yet", fileName); return; } DownloadableMedia downloadable = DownloadableMedia.create(title, fileName, media.getModified(), media.getCreated(), media.getLength(), 0L, - targetStoreUri.getId(), media.getId()); + sourceStoreUri.getId(), targetStoreUri.getId(), media.getId()); String escortFileName = targetStoreName + "." + downloadable.getString("fileName"); String outputPath = null; try { outputPath = Paths.get(escortStoreUri.toString(true)).toString(); EscortFiles.createMetadataIfNotExists(outputPath, escortFileName, downloadable.toPrettyString("")); } catch (Exception e) { - logger.error("Can't create escort file {}", Paths.get(outputPath.toString(), escortFileName)); + logger.error(getSessionMarker(), "Can't create escort file {}", Paths.get(outputPath.toString(), escortFileName)); } } + //csak visszaterunk + private String normalizeName(String mediaName) { + String result = mediaName; + if (StringUtils.isNotBlank(appendExtension)) + result += appendExtension; + // if (StringUtils.isNotBlank(mediaName)) { + // result = mediaName.trim().toLowerCase(); + // } + return result; + } + }