From 7bdb863ac5ca03e95d6e06b80c3407e9e9f653f5 Mon Sep 17 00:00:00 2001 From: "vasary.daniel" Date: Mon, 9 Aug 2021 14:27:32 +0000 Subject: [PATCH] git-tfs-id: [http://tfs.userrendszerhaz.hu:8080/tfs/DefaultCollection]$/MediaCube;C32295 --- .../server/steps/SafeDeleteRecursive.java | 222 ++++++++++-------- .../src/user/commons/StoreUri.java | 2 - .../metadata/MetadataProviderFactory.java | 19 +- .../metadata/MyBatisMetadataProvider.java | 10 +- .../planair/PlanAirMetadataProvider.java | 2 +- 5 files changed, 146 insertions(+), 109 deletions(-) diff --git a/server/user.jobengine.executors/src/user/jobengine/server/steps/SafeDeleteRecursive.java b/server/user.jobengine.executors/src/user/jobengine/server/steps/SafeDeleteRecursive.java index c3d603c4..491e5a38 100644 --- a/server/user.jobengine.executors/src/user/jobengine/server/steps/SafeDeleteRecursive.java +++ b/server/user.jobengine.executors/src/user/jobengine/server/steps/SafeDeleteRecursive.java @@ -2,6 +2,7 @@ package user.jobengine.server.steps; import java.io.File; import java.io.IOException; +import java.nio.file.DirectoryStream; import java.nio.file.FileVisitResult; import java.nio.file.FileVisitor; import java.nio.file.Files; @@ -9,20 +10,28 @@ import java.nio.file.Path; import java.nio.file.Paths; import java.nio.file.attribute.BasicFileAttributes; import java.text.SimpleDateFormat; +import java.util.ArrayList; import java.util.Arrays; +import java.util.Collections; +import java.util.Date; import java.util.List; import org.apache.commons.io.FilenameUtils; -import org.apache.commons.lang.StringUtils; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; -import org.apache.logging.log4j.message.Message; -import org.apache.logging.log4j.message.ParameterizedMessage; import com.ibm.nosql.json.api.BasicDBObject; -import user.commons.MediaCubeMarker; +import user.commons.IEntityBase; +import user.commons.RemoteFile; +import user.commons.StoreUri; import user.commons.mediatool.MediaInfo; +import user.commons.remotestore.RemoteStoreProtocol; +import user.jobengine.db.Item; +import user.jobengine.db.Media; +import user.jobengine.db.MediaFile; +import user.jobengine.db.MediaFileDAO; +import user.jobengine.db.Store; import user.jobengine.server.steps.shared.EscortFiles; import user.mediacube.metadata.interfaces.IMetadata; import user.mediacube.metadata.interfaces.IMetadataProvider; @@ -39,10 +48,8 @@ public class SafeDeleteRecursive extends JobStep implements FileVisitor { private static final String MEDIA_TITLE = "mediaTitle"; private static final String MEDIA_DESCRIPTION = "mediaDescription"; private static final String MEDIA_TYPE = "mediaType"; - private static final String ARCHIVE = "Archiválás"; - private static final String ARCHIVE_ITEM = "archiveItem"; - private static final String KILL_DATE_DAYS = "killDateDays"; - private SimpleDateFormat df = new SimpleDateFormat("yyyy.MM.dd"); + private static final String KILLDATEEXT = ".killdate"; + private SimpleDateFormat df = new SimpleDateFormat("yyyy.MM.dd HH:mm:ss"); private List skipPathNames = Arrays.asList("!ARCHIVALAS_ALATT", EscortFiles.STATUSFOLDER, EscortFiles.CONFLICTFOLDER); @@ -59,32 +66,31 @@ public class SafeDeleteRecursive extends JobStep implements FileVisitor { return result; } - private void checkArchiveItem(ArchiveItem archiveItem) throws Exception { + private boolean checkArchiveItem(ArchiveItem archiveItem, Item item, Media media) { if (archiveItem == null) - throw new Exception("No metadata specified."); + return false; + + String itemHouseId = archiveItem.getItemHouseId() == null ? "" : archiveItem.getItemHouseId(); + if (!itemHouseId.equals(item.getHouseId())) + return false; - if (StringUtils.isBlank(archiveItem.getItemHouseId())) - throw new Exception("No Item HouseID specified in metadata."); + String itemTitle = archiveItem.getItemTitle() == null ? "" : archiveItem.getItemTitle(); + if (!itemTitle.equals(item.getTitle())) + return false; - if (StringUtils.isBlank(archiveItem.getItemTitle())) - throw new Exception("No Item Title specified in metadata."); + String mediaHouseId = archiveItem.getMediaHouseId() == null ? "" : archiveItem.getMediaHouseId(); + if (!mediaHouseId.equals(media.getHouseId())) + return false; - if (StringUtils.isBlank(archiveItem.getMediaHouseId())) - throw new Exception("No Media HouseID specified in metadata."); + String mediaTitle = archiveItem.getMediaTitle() == null ? "" : archiveItem.getMediaTitle(); + if (!mediaTitle.equals(media.getTitle())) + return false; - if (StringUtils.isBlank(archiveItem.getMediaTitle())) - throw new Exception("No Media Title specified in metadata."); - } + String mediaDescription = archiveItem.getMediaDescription() == null ? "" : archiveItem.getMediaDescription(); + if (!mediaDescription.equals(media.getDescription())) + return false; - private String archiveItemJSON(ArchiveItem result) { - BasicDBObject obj = new BasicDBObject(); - obj.put("itemHouseId", result.getItemHouseId()); - obj.put("itemTitle", result.getItemTitle()); - obj.put("mediaHouseId", result.getMediaHouseId()); - obj.put("mediaTitle", result.getMediaTitle()); - obj.put("mediaDescription", result.getMediaDescription()); - obj.put("mediaType", result.getMediaType()); - return obj.toPrettyString(""); + return true; } private ArchiveItem createArchiveItem(Path filePath) throws Exception { @@ -98,33 +104,20 @@ public class SafeDeleteRecursive extends JobStep implements FileVisitor { if (result != null) result.setMediaFile(filePath.toString()); - logger.info(getSessionMarker(), "PlanAir query done for {}", filePath); } catch (Exception e) { logger.error(getSessionMarker(), "PlanAir metadata error", e); - // nem latja a drivert pl. - throw e; } - //210616 Ha nincs metaadat, nem archivalunk - // if (result == null) { - // result = new ArchiveItem(); - // BasicFileAttributes attr = Files.readAttributes(filePath, BasicFileAttributes.class); - // result.setItemHouseId(df.format(attr.lastModifiedTime().toMillis())); - // result.setItemTitle(filePath.getParent().toString()); - // result.setMediaHouseId(mediaHouseId); - // result.setMediaTitle(fileName); - // // TODO kesobb kivenni - // // result.setMediaDescription("/OMARCHIVE"); - // result.setMediaType("Generic"); - // result.setMediaFile(filePath.toString()); - // } - return result; } @StepEntry public Object[] execute(String sourcePath) throws Exception { logger.info(getSessionMarker(), "Starting in {}", sourcePath); + logger.info(getSessionMarker(), "{};{};{};{};{};{};{};{};{};{};{};{};{};", "Napló időbélyeg", "Törlés sikeres", + "Neve", "Elérése", "Mérete", "Létrehozva", "Módosítva", "Catched", "Killdate", "Archivált (MC)", + "Archiválva (MC)", "Archivált (TSM)", "Archiválva (TSM)", "Méret (TSM)", "Metaadat egyezés"); + try { if (getJobRuntime().forkPrepare()) { Files.walkFileTree(Paths.get(sourcePath), this); @@ -182,16 +175,6 @@ public class SafeDeleteRecursive extends JobStep implements FileVisitor { return result; } - private boolean handleArchiveConflict(Path mediaPath) throws Exception { - boolean result = false; - String sourceFileName = mediaPath.getFileName().toString(); - if (getManager().isMediaFileExists(sourceFileName)) { - EscortFiles.createMediaCatch(mediaPath); - result = true; - } - return result; - } - @Override public FileVisitResult postVisitDirectory(Path paramT, IOException paramIOException) throws IOException { return FileVisitResult.CONTINUE; @@ -212,62 +195,57 @@ public class SafeDeleteRecursive extends JobStep implements FileVisitor { } private boolean processPathItem(Path mediaPath) throws Exception { - String fileName = mediaPath.getFileName().toString(); - if (fileName.startsWith(".") || fileName.endsWith(".nomd")) - return false; - - logger.info(getSessionMarker(), "Processing {}", mediaPath); + BasicFileAttributes attr = Files.readAttributes(mediaPath, BasicFileAttributes.class); + File mediaFSFile = mediaPath.toFile(); - File mediaFile = mediaPath.toFile(); - if (mediaFile.isDirectory()) { - //logger.info(getSessionMarker(), "Skipping directory {}", mediaPath); + if (mediaFSFile.isDirectory()) return false; - } - - if (EscortFiles.isMediaCatched(mediaPath)) { - //logger.info(getSessionMarker(), "Skipping already catched {}", mediaPath); - return false; - } + String fileName = mediaPath.getFileName().toString(); Path nomdFile = Paths.get(mediaPath.toString() + ".nomd"); - - if (nomdFile.toFile().exists()) { - //logger.info(getSessionMarker(), "Skipping nomd file exists {}", nomdFile); - return false; - } - - if (handleArchiveConflict(mediaPath)) { - logger.info(getSessionMarker(), "Skipping archive db already contains {}", mediaPath); - return false; - } - - if (!canReadMediaInfo(mediaPath)) { - logger.info(getSessionMarker(), "Skipping cant read mediainfo {}", mediaPath); - return false; - } - ArchiveItem archiveItem = createArchiveItem(mediaPath); - - if (archiveItem == null) { - Message msg = new ParameterizedMessage("Nincs metaadat!"); - logger.info(new MediaCubeMarker("vasary@elgekko.net,muszak@mediavivantis.hu", - "Értesítés problémás " + mediaPath.getFileName().toString() + " archiválásról"), msg); - Files.createFile(nomdFile); - return false; + List killDateFiles = getKillDateFiles(mediaPath); + MediaFile mediaFile = getMediaFile(fileName); + + long size = mediaFSFile.length(); + Date lastModifiedDate = new Date(attr.lastModifiedTime().toMillis()); + Date lastAccesDate = new Date(attr.lastAccessTime().toMillis()); + Date createDate = new Date(attr.creationTime().toMillis()); + boolean catchedExists = EscortFiles.isMediaCatched(mediaPath); + boolean noMDExists = nomdFile.toFile().exists(); + boolean killdateExists = killDateFiles.size() > 0; + //boolean mediaInfoAvailable = canReadMediaInfo(mediaPath); + boolean mcArchived = mediaFile != null; + boolean metadataEquals = false; + Date mcArchivedDate = null; + if (mcArchived) { + Media media = getManager().getMedia(mediaFile.getMediaId()); + mcArchivedDate = media.getCreated(); + Item item = getManager().getItem(media.getItemId()); + metadataEquals = checkArchiveItem(archiveItem, item, media); } - try { - checkArchiveItem(archiveItem); - String metadata = archiveItemJSON(archiveItem); - } catch (Exception e) { - logger.error(getSessionMarker(), - "Az '{}' állomány archiválási kísérlete sikertelen. A rendszer üzenete: {}", mediaPath, - e.getMessage()); + RemoteFile tsmFile = getTSMFile(mediaFile.getRelativePath()); + boolean tsmArchived = tsmFile != null; + long tsmSize = 0; + if (tsmArchived) { + tsmSize = tsmFile.getSize(); } + boolean canDelete = !noMDExists && catchedExists && killdateExists && mcArchived && tsmArchived; + Date now = new Date(System.currentTimeMillis()); + logger.info(getSessionMarker(), "{};{};{};{};{};{};{};{};{};{};{};{};{};", df.format(now), YN(canDelete), + fileName, mediaPath.getParent(), size, df.format(createDate), df.format(lastModifiedDate), + YN(catchedExists), YN(killdateExists), YN(mcArchived), df.format(mcArchivedDate), YN(tsmArchived), + "Archiválva (TSM)", tsmSize, YN(metadataEquals)); + return true; } + private char YN(boolean value) { + return value ? 'Y' : 'N'; + } + @Override public FileVisitResult visitFile(Path filePath, BasicFileAttributes paramBasicFileAttributes) throws IOException { // logger.info(getSessionMarker(), "Will checked {}", filePath); @@ -284,4 +262,50 @@ public class SafeDeleteRecursive extends JobStep implements FileVisitor { logger.info("Error archive {}", filePath); return FileVisitResult.CONTINUE; } + + private List getKillDateFiles(Path filePath) { + String killDateFilePattern = String.format("%s.*%s", filePath.getFileName().toString(), KILLDATEEXT); + List result = new ArrayList<>(); + Path statusPath = null; + try { + statusPath = Paths.get(filePath.getParent().toString(), EscortFiles.STATUSFOLDER); + } catch (Exception e) { + logger.catching(e); + return null; + } + File statusPathFile = statusPath.toFile(); + if (statusPathFile.exists() && statusPathFile.isDirectory()) { + try (DirectoryStream stream = Files.newDirectoryStream(statusPath, killDateFilePattern)) { + stream.forEach(p -> result.add(p)); + } catch (Exception e) { + logger.catching(e); + } + } + Collections.sort(result); + return result; + } + + MediaFile getMediaFile(String fileName) { + MediaFile result = null; + MediaFileDAO mfDAO = (MediaFileDAO) getManager().getBaseDAO(MediaFile.class); + List mfList = mfDAO.getByHouseId(fileName); + if (mfList != null && mfList.size() == 1) + result = (MediaFile) mfList.get(0); + return result; + } + + RemoteFile getTSMFile(String mcFileName) { + RemoteFile result = null; + Store tsmStore = getManager().getSystemStore(false); + StoreUri tsmStoreUri = tsmStore.getSourceStoreUri(RemoteStoreProtocol.TSM); + try { + result = tsmStoreUri.getRemoteFile(mcFileName); + } catch (Exception e) { + logger.error(e); + } finally { + tsmStoreUri.cleanUp(); + } + return result; + } + } diff --git a/server/user.jobengine.osgi.commons/src/user/commons/StoreUri.java b/server/user.jobengine.osgi.commons/src/user/commons/StoreUri.java index ede5844e..37315c6a 100644 --- a/server/user.jobengine.osgi.commons/src/user/commons/StoreUri.java +++ b/server/user.jobengine.osgi.commons/src/user/commons/StoreUri.java @@ -97,9 +97,7 @@ public class StoreUri extends EntityBase implements Serializable { result.setName(targetFileName); - //TSM teszthez egy kamu ertek bellitasaval! result.setSize(source.getSize()); - result.setSize(1001); ProgressEvent progressEvent = new ProgressEvent(this, 0); StatusEvent statusEvent = new StatusEvent(this); diff --git a/server/user.mediacube.metadata/src/user/mediacube/metadata/MetadataProviderFactory.java b/server/user.mediacube.metadata/src/user/mediacube/metadata/MetadataProviderFactory.java index e2f9629b..273fa35b 100644 --- a/server/user.mediacube.metadata/src/user/mediacube/metadata/MetadataProviderFactory.java +++ b/server/user.mediacube.metadata/src/user/mediacube/metadata/MetadataProviderFactory.java @@ -7,7 +7,6 @@ import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import user.commons.configuration.IConfiguration; -import user.mediacube.metadata.hsm.HSMMetadataProvider; import user.mediacube.metadata.interfaces.IMetadataProvider; import user.mediacube.metadata.interfaces.IMetadataProviderFactory; import user.mediacube.metadata.interfaces.MetadataProviderType; @@ -31,11 +30,19 @@ public class MetadataProviderFactory implements IMetadataProviderFactory { if (service instanceof IConfiguration) { systemConfig = (IConfiguration) service; logger.info("IConfiguration service binded"); - try { - providers.put(MetadataProviderType.PLANAIR, new PlanAirMetadataProvider()); - providers.put(MetadataProviderType.HSM, new HSMMetadataProvider()); - } catch (Exception e) { - } + + createProvider(MetadataProviderType.PLANAIR); + createProvider(MetadataProviderType.HSM); + + } + } + + private void createProvider(MetadataProviderType type) { + try { + logger.info("Creating metadata provider for {}", type); + providers.put(type, new PlanAirMetadataProvider()); + } catch (Exception e) { + logger.error("Metadata provider not created for {}. Error: {}", type, e.getMessage()); } } diff --git a/server/user.mediacube.metadata/src/user/mediacube/metadata/MyBatisMetadataProvider.java b/server/user.mediacube.metadata/src/user/mediacube/metadata/MyBatisMetadataProvider.java index 2b0c807b..dbb90e54 100644 --- a/server/user.mediacube.metadata/src/user/mediacube/metadata/MyBatisMetadataProvider.java +++ b/server/user.mediacube.metadata/src/user/mediacube/metadata/MyBatisMetadataProvider.java @@ -10,6 +10,8 @@ import org.apache.ibatis.session.SqlSession; import org.apache.ibatis.session.SqlSessionFactory; import org.apache.ibatis.session.SqlSessionFactoryBuilder; import org.apache.ibatis.transaction.jdbc.JdbcTransactionFactory; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; import com.ibm.nosql.json.api.BasicDBObject; @@ -18,13 +20,19 @@ import user.mediacube.metadata.interfaces.IMetadataListOptions; import user.mediacube.metadata.interfaces.IMetadataProvider; public abstract class MyBatisMetadataProvider implements IMetadataProvider { + private final static Logger logger = LogManager.getLogger(); private final SqlSessionFactory sqlSessionFactory; public MyBatisMetadataProvider() throws Exception { + String driverClassName = null; try { - Class.forName(getDriverClassName()); + driverClassName = getDriverClassName(); + Class.forName(driverClassName); sqlSessionFactory = buildqlSessionFactory(); + + logger.info("Metadata driver {} loaded", driverClassName); } catch (Exception e) { + logger.error("Can't load metadata driver {}. Error: {}", driverClassName, e.getMessage()); throw e; } } diff --git a/server/user.mediacube.metadata/src/user/mediacube/metadata/planair/PlanAirMetadataProvider.java b/server/user.mediacube.metadata/src/user/mediacube/metadata/planair/PlanAirMetadataProvider.java index 97d0ea2c..50c42fc8 100644 --- a/server/user.mediacube.metadata/src/user/mediacube/metadata/planair/PlanAirMetadataProvider.java +++ b/server/user.mediacube.metadata/src/user/mediacube/metadata/planair/PlanAirMetadataProvider.java @@ -18,7 +18,7 @@ import user.mediacube.metadata.interfaces.IMetadataListOptions; import user.mediacube.metadata.interfaces.PlanAirMetadataListOptions; public class PlanAirMetadataProvider extends MyBatisMetadataProvider { - private String JDBC_DRIVER = "com.microsoft.sqlserver.jdbc.SQLServerDriver"; + private static final String JDBC_DRIVER = "com.microsoft.sqlserver.jdbc.SQLServerDriver"; public PlanAirMetadataProvider() throws Exception { super(); -- 2.54.0