From 5a849f000d82a73786c007e9b35bf36fa63b8461 Mon Sep 17 00:00:00 2001 From: "vasary.daniel" Date: Mon, 9 Aug 2021 11:43:46 +0000 Subject: [PATCH] git-tfs-id: [http://tfs.userrendszerhaz.hu:8080/tfs/DefaultCollection]$/MediaCube;C32294 --- .../debug-mediacube-mv-8889.launch | 18 ++ .../server/steps/SafeDeleteRecursive.java | 287 ++++++++++++++++++ .../src/user/commons/StoreUri.java | 9 +- .../planair/PlanAirMetadataProvider.java | 11 +- 4 files changed, 319 insertions(+), 6 deletions(-) create mode 100644 server/-configuration/debug-mediacube-mv-8889.launch create mode 100644 server/user.jobengine.executors/src/user/jobengine/server/steps/SafeDeleteRecursive.java diff --git a/server/-configuration/debug-mediacube-mv-8889.launch b/server/-configuration/debug-mediacube-mv-8889.launch new file mode 100644 index 00000000..4e90fa57 --- /dev/null +++ b/server/-configuration/debug-mediacube-mv-8889.launch @@ -0,0 +1,18 @@ + + + + + + + + + + + + + + + + + + diff --git a/server/user.jobengine.executors/src/user/jobengine/server/steps/SafeDeleteRecursive.java b/server/user.jobengine.executors/src/user/jobengine/server/steps/SafeDeleteRecursive.java new file mode 100644 index 00000000..c3d603c4 --- /dev/null +++ b/server/user.jobengine.executors/src/user/jobengine/server/steps/SafeDeleteRecursive.java @@ -0,0 +1,287 @@ +package user.jobengine.server.steps; + +import java.io.File; +import java.io.IOException; +import java.nio.file.FileVisitResult; +import java.nio.file.FileVisitor; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.nio.file.attribute.BasicFileAttributes; +import java.text.SimpleDateFormat; +import java.util.Arrays; +import java.util.List; + +import org.apache.commons.io.FilenameUtils; +import org.apache.commons.lang.StringUtils; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.apache.logging.log4j.message.Message; +import org.apache.logging.log4j.message.ParameterizedMessage; + +import com.ibm.nosql.json.api.BasicDBObject; + +import user.commons.MediaCubeMarker; +import user.commons.mediatool.MediaInfo; +import user.jobengine.server.steps.shared.EscortFiles; +import user.mediacube.metadata.interfaces.IMetadata; +import user.mediacube.metadata.interfaces.IMetadataProvider; +import user.mediacube.metadata.interfaces.IMetadataProviderFactory; +import user.mediacube.metadata.interfaces.MetadataProviderType; +import user.mediacube.metadata.interfaces.MetadataType; +import user.mediacube.metadata.interfaces.PlanAirMetadataListOptions; + +public class SafeDeleteRecursive extends JobStep implements FileVisitor { + private static final Logger logger = LogManager.getLogger(); + private static final String ITEM_TITLE = "itemTitle"; + private static final String ITEM_HOUSEID = "itemHouseId"; + private static final String MEDIA_HOUSEID = "mediaHouseId"; + private static final String MEDIA_TITLE = "mediaTitle"; + private static final String MEDIA_DESCRIPTION = "mediaDescription"; + private static final String MEDIA_TYPE = "mediaType"; + private static final String ARCHIVE = "Archiválás"; + private static final String ARCHIVE_ITEM = "archiveItem"; + private static final String KILL_DATE_DAYS = "killDateDays"; + private SimpleDateFormat df = new SimpleDateFormat("yyyy.MM.dd"); + + private List skipPathNames = Arrays.asList("!ARCHIVALAS_ALATT", EscortFiles.STATUSFOLDER, + EscortFiles.CONFLICTFOLDER); + + private boolean canReadMediaInfo(Path mediaFilePath) { + boolean result = false; + try { + MediaInfo mi = new MediaInfo(mediaFilePath); + mi.process(); + result = true; + } catch (Exception e) { + logger.warn(getSessionMarker(), e.getMessage()); + } + return result; + } + + private void checkArchiveItem(ArchiveItem archiveItem) throws Exception { + if (archiveItem == null) + throw new Exception("No metadata specified."); + + if (StringUtils.isBlank(archiveItem.getItemHouseId())) + throw new Exception("No Item HouseID specified in metadata."); + + if (StringUtils.isBlank(archiveItem.getItemTitle())) + throw new Exception("No Item Title specified in metadata."); + + if (StringUtils.isBlank(archiveItem.getMediaHouseId())) + throw new Exception("No Media HouseID specified in metadata."); + + if (StringUtils.isBlank(archiveItem.getMediaTitle())) + throw new Exception("No Media Title specified in metadata."); + } + + private String archiveItemJSON(ArchiveItem result) { + BasicDBObject obj = new BasicDBObject(); + obj.put("itemHouseId", result.getItemHouseId()); + obj.put("itemTitle", result.getItemTitle()); + obj.put("mediaHouseId", result.getMediaHouseId()); + obj.put("mediaTitle", result.getMediaTitle()); + obj.put("mediaDescription", result.getMediaDescription()); + obj.put("mediaType", result.getMediaType()); + return obj.toPrettyString(""); + } + + private ArchiveItem createArchiveItem(Path filePath) throws Exception { + ArchiveItem result = null; + + String fileName = filePath.getFileName().toString(); + String mediaHouseId = FilenameUtils.removeExtension(fileName); + try { + result = getPlanAirMetadata(mediaHouseId); + + if (result != null) + result.setMediaFile(filePath.toString()); + + logger.info(getSessionMarker(), "PlanAir query done for {}", filePath); + } catch (Exception e) { + logger.error(getSessionMarker(), "PlanAir metadata error", e); + // nem latja a drivert pl. + throw e; + } + + //210616 Ha nincs metaadat, nem archivalunk + // if (result == null) { + // result = new ArchiveItem(); + // BasicFileAttributes attr = Files.readAttributes(filePath, BasicFileAttributes.class); + // result.setItemHouseId(df.format(attr.lastModifiedTime().toMillis())); + // result.setItemTitle(filePath.getParent().toString()); + // result.setMediaHouseId(mediaHouseId); + // result.setMediaTitle(fileName); + // // TODO kesobb kivenni + // // result.setMediaDescription("/OMARCHIVE"); + // result.setMediaType("Generic"); + // result.setMediaFile(filePath.toString()); + // } + + return result; + } + + @StepEntry + public Object[] execute(String sourcePath) throws Exception { + logger.info(getSessionMarker(), "Starting in {}", sourcePath); + try { + if (getJobRuntime().forkPrepare()) { + Files.walkFileTree(Paths.get(sourcePath), this); + } + } catch (Exception e) { + logger.error(getSessionMarker(), "Az '{}' mappa elérése sikertelen. A rendszer hibaüzenete: {}", sourcePath, + e.getMessage()); + } finally { + getJobRuntime().forkWaitComplete(); + } + return null; + } + + private ArchiveItem getPlanAirMetadata(String mediaHouseId) throws Exception { + PlanAirMetadataListOptions opt = new PlanAirMetadataListOptions(); + opt.setSearch(mediaHouseId); + opt.setType(MetadataType.Material); + + BasicDBObject json = null; + List data = null; + + IMetadataProviderFactory factory = getService(IMetadataProviderFactory.class); + if (factory == null) + logger.info(getSessionMarker(), "IMetadataProviderFactory is null"); + + IMetadataProvider planairProvider = factory.getProvider(MetadataProviderType.PLANAIR); + if (planairProvider == null) + logger.info(getSessionMarker(), "IMetadataProvider is null"); + + ArchiveItem result = null; + data = planairProvider.list(opt); + if (data.size() != 0) + json = data.get(0).asJSON(); + else { + opt.setType(MetadataType.Promo); + data = planairProvider.list(opt); + if (data.size() != 0) + json = data.get(0).asJSON(); + else { + opt.setType(MetadataType.AD); + data = planairProvider.list(opt); + if (data.size() != 0) + json = data.get(0).asJSON(); + } + } + if (json != null) { + result = new ArchiveItem(); + result.setItemHouseId(json.getString(ITEM_HOUSEID)); + result.setItemTitle(json.getString(ITEM_TITLE)); + result.setMediaHouseId(json.getString(MEDIA_HOUSEID)); + result.setMediaTitle(json.getString(MEDIA_TITLE)); + result.setMediaDescription(json.getString(MEDIA_DESCRIPTION)); + result.setMediaType(json.getString(MEDIA_TYPE)); + } + return result; + } + + private boolean handleArchiveConflict(Path mediaPath) throws Exception { + boolean result = false; + String sourceFileName = mediaPath.getFileName().toString(); + if (getManager().isMediaFileExists(sourceFileName)) { + EscortFiles.createMediaCatch(mediaPath); + result = true; + } + return result; + } + + @Override + public FileVisitResult postVisitDirectory(Path paramT, IOException paramIOException) throws IOException { + return FileVisitResult.CONTINUE; + } + + @Override + public FileVisitResult preVisitDirectory(Path dir, BasicFileAttributes paramBasicFileAttributes) + throws IOException { + Path dirName = dir.getFileName(); + + if (skipPathNames.contains(dirName.toString())) { + logger.info(getSessionMarker(), "PreVisit skip {}", dir); + return FileVisitResult.SKIP_SUBTREE; + } else + logger.info(getSessionMarker(), "PreVisit {}", dir); + + return FileVisitResult.CONTINUE; + } + + private boolean processPathItem(Path mediaPath) throws Exception { + String fileName = mediaPath.getFileName().toString(); + if (fileName.startsWith(".") || fileName.endsWith(".nomd")) + return false; + + logger.info(getSessionMarker(), "Processing {}", mediaPath); + + File mediaFile = mediaPath.toFile(); + if (mediaFile.isDirectory()) { + //logger.info(getSessionMarker(), "Skipping directory {}", mediaPath); + return false; + } + + if (EscortFiles.isMediaCatched(mediaPath)) { + //logger.info(getSessionMarker(), "Skipping already catched {}", mediaPath); + return false; + } + + Path nomdFile = Paths.get(mediaPath.toString() + ".nomd"); + + if (nomdFile.toFile().exists()) { + //logger.info(getSessionMarker(), "Skipping nomd file exists {}", nomdFile); + return false; + } + + if (handleArchiveConflict(mediaPath)) { + logger.info(getSessionMarker(), "Skipping archive db already contains {}", mediaPath); + return false; + } + + if (!canReadMediaInfo(mediaPath)) { + logger.info(getSessionMarker(), "Skipping cant read mediainfo {}", mediaPath); + return false; + } + + ArchiveItem archiveItem = createArchiveItem(mediaPath); + + if (archiveItem == null) { + Message msg = new ParameterizedMessage("Nincs metaadat!"); + logger.info(new MediaCubeMarker("vasary@elgekko.net,muszak@mediavivantis.hu", + "Értesítés problémás " + mediaPath.getFileName().toString() + " archiválásról"), msg); + Files.createFile(nomdFile); + return false; + } + + try { + checkArchiveItem(archiveItem); + String metadata = archiveItemJSON(archiveItem); + } catch (Exception e) { + logger.error(getSessionMarker(), + "Az '{}' állomány archiválási kísérlete sikertelen. A rendszer üzenete: {}", mediaPath, + e.getMessage()); + } + + return true; + } + + @Override + public FileVisitResult visitFile(Path filePath, BasicFileAttributes paramBasicFileAttributes) throws IOException { + // logger.info(getSessionMarker(), "Will checked {}", filePath); + try { + processPathItem(filePath); + } catch (Exception e) { + logger.catching(e); + } + return FileVisitResult.CONTINUE; + } + + @Override + public FileVisitResult visitFileFailed(Path filePath, IOException paramIOException) throws IOException { + logger.info("Error archive {}", filePath); + return FileVisitResult.CONTINUE; + } +} diff --git a/server/user.jobengine.osgi.commons/src/user/commons/StoreUri.java b/server/user.jobengine.osgi.commons/src/user/commons/StoreUri.java index 6770706e..ede5844e 100644 --- a/server/user.jobengine.osgi.commons/src/user/commons/StoreUri.java +++ b/server/user.jobengine.osgi.commons/src/user/commons/StoreUri.java @@ -28,7 +28,8 @@ import user.commons.remotestore.StatusEvent; @SuppressWarnings("serial") public class StoreUri extends EntityBase implements Serializable { private final static Logger logger = LogManager.getLogger(); - private static final boolean VALIDATETRANSFERS = SystemConfiguration.getInstance().value("jobs.validate-transfers", false); + private static final boolean VALIDATETRANSFERS = SystemConfiguration.getInstance().value("jobs.validate-transfers", + false); private final int BUFFER_SIZE = SystemConfiguration.getInstance().value("jobs.copy-buffer-size", 32768); private String uri = null; @@ -82,7 +83,8 @@ public class StoreUri extends EntityBase implements Serializable { this.statusListenerList = new EventListenerList(); } - public RemoteFile copy(IDirectoryLister inputLister, IDirectoryLister outputLister, RemoteFile source, String targetName) throws Exception { + public RemoteFile copy(IDirectoryLister inputLister, IDirectoryLister outputLister, RemoteFile source, + String targetName) throws Exception { RemoteFile result = null; long uploadedBytes = 0; int byteCount = 0; @@ -94,7 +96,10 @@ public class StoreUri extends EntityBase implements Serializable { targetFileName = source.getName(); result.setName(targetFileName); + + //TSM teszthez egy kamu ertek bellitasaval! result.setSize(source.getSize()); + result.setSize(1001); ProgressEvent progressEvent = new ProgressEvent(this, 0); StatusEvent statusEvent = new StatusEvent(this); diff --git a/server/user.mediacube.metadata/src/user/mediacube/metadata/planair/PlanAirMetadataProvider.java b/server/user.mediacube.metadata/src/user/mediacube/metadata/planair/PlanAirMetadataProvider.java index de6a8198..97d0ea2c 100644 --- a/server/user.mediacube.metadata/src/user/mediacube/metadata/planair/PlanAirMetadataProvider.java +++ b/server/user.mediacube.metadata/src/user/mediacube/metadata/planair/PlanAirMetadataProvider.java @@ -18,7 +18,7 @@ import user.mediacube.metadata.interfaces.IMetadataListOptions; import user.mediacube.metadata.interfaces.PlanAirMetadataListOptions; public class PlanAirMetadataProvider extends MyBatisMetadataProvider { - String JDBC_DRIVER = "com.microsoft.sqlserver.jdbc.SQLServerDriver"; + private String JDBC_DRIVER = "com.microsoft.sqlserver.jdbc.SQLServerDriver"; public PlanAirMetadataProvider() throws Exception { super(); @@ -58,15 +58,18 @@ public class PlanAirMetadataProvider extends MyBatisMetadataProvider { switch (opt.getType()) { case AD: IPlanAirADMapper adMapper = session.getMapper(IPlanAirADMapper.class); - result = (List) (List) adMapper.getMaterials(problematic, opt.getSearch(), opt.getBroadcastDay()); + result = (List) (List) adMapper.getMaterials(problematic, opt.getSearch(), + opt.getBroadcastDay()); break; case Material: IPlanAirMaterialMapper materialMapper = session.getMapper(IPlanAirMaterialMapper.class); - result = (List) (List) materialMapper.getMaterials(problematic, opt.getSearch(), opt.getBroadcastDay()); + result = (List) (List) materialMapper.getMaterials(problematic, opt.getSearch(), + opt.getBroadcastDay()); break; case Promo: IPlanAirPromoMapper promoMapper = session.getMapper(IPlanAirPromoMapper.class); - result = (List) (List) promoMapper.getMaterials(problematic, opt.getSearch(), opt.getBroadcastDay()); + result = (List) (List) promoMapper.getMaterials(problematic, opt.getSearch(), + opt.getBroadcastDay()); break; } return result; -- 2.54.0