From: Vásáry Dániel Date: Tue, 21 May 2019 11:26:08 +0000 (+0000) Subject: git-tfs-id: [http://tfs.userrendszerhaz.hu:8080/tfs/DefaultCollection]$/MediaCube... X-Git-Url: http://git.useribm.hu/?a=commitdiff_plain;h=68f74f7866979c7b897662529ba1bcaa9e057a7c;p=mediacube.git git-tfs-id: [tfs.userrendszerhaz.hu:8080/tfs/DefaultCollection]$/MediaCube;C31435 --- diff --git a/server/-configuration/run-mediacube-server-bsh.launch b/server/-configuration/run-mediacube-server-bsh.launch index b5abf0ff..60593dc9 100644 --- a/server/-configuration/run-mediacube-server-bsh.launch +++ b/server/-configuration/run-mediacube-server-bsh.launch @@ -19,7 +19,7 @@ - + diff --git a/server/hu.user.mediacube.executors.tests/src/hu/user/mediacube/executors/tests/Support.java b/server/hu.user.mediacube.executors.tests/src/hu/user/mediacube/executors/tests/Support.java index 3a2adc70..c164fd7f 100644 --- a/server/hu.user.mediacube.executors.tests/src/hu/user/mediacube/executors/tests/Support.java +++ b/server/hu.user.mediacube.executors.tests/src/hu/user/mediacube/executors/tests/Support.java @@ -32,15 +32,14 @@ import user.commons.ListUtils; import user.commons.logging.LogUtils; import user.commons.nosql.NoSQLUtils; import user.jobengine.db.IItemManager; -import user.jobengine.db.IResultSetConsumer; import user.jobengine.db.Item; import user.jobengine.db.ItemManager; import user.jobengine.db.Media; import user.jobengine.db.MediaFile; +import user.jobengine.server.JobRuntime; import user.jobengine.server.steps.ItemManagerExtensions; -import user.jobengine.server.steps.MetadataTypeDetector; -import user.jobengine.server.steps.MetadataTypeDetector.MetadataType; import user.jobengine.server.steps.PlanAirExtensions; +import user.jobengine.server.steps.ProjectCleanupMountedLocationStep; import user.jobengine.server.steps.RundownArchive; public class Support { @@ -97,7 +96,7 @@ public class Support { } stream.close(); - check_rd_process(allFiles, allStatus); + //check_rd_process(allFiles, allStatus); } catch (Exception e) { System.err.println(e.getMessage()); } @@ -136,37 +135,37 @@ public class Support { return null; } - private void check_rd_process(List allFiles, List allStatus) { - File mediaFile = mediaFilePath.toFile(); - if (mediaFile.isDirectory() || !mediaFile.getName().toUpperCase().endsWith(".MXF")) - return; - Path status = Paths.get(mediaFilePath.getParent().toString(), ".STATUS", mediaFile.getName() + ".json"); - if (!status.toFile().exists()) { - System.out.println("Nincs JSON: " + mediaFile.getAbsolutePath()); - } - - if (mediaFile.length() == 0) { - String houseid = mediaFile.getName().substring(0, mediaFile.getName().lastIndexOf(".")); - final String idToCheck = houseid.substring(0, houseid.lastIndexOf("-")); - MetadataType metadataType = MetadataTypeDetector.GuessMetadataType(idToCheck); - if (metadataType == MetadataType.OctopusPlaceholder) { - String query = String.format("select count(*) from mediafile where houseid like '%s%%'", idToCheck); - IResultSetConsumer consumer = rs -> { - if (rs.getInt(1) == 0) { - - File f = check_rd_nonzero_fellow(idToCheck, allFiles); - if (f == null) - System.out.println("0 még sincs korábbi: " + mediaFile.getAbsolutePath()); - - } - return false; - }; - manager.executeQuery(query, consumer, null); - } - - } - - } + // private void check_rd_process(List allFiles, List allStatus) { + // File mediaFile = mediaFilePath.toFile(); + // if (mediaFile.isDirectory() || !mediaFile.getName().toUpperCase().endsWith(".MXF")) + // return; + // Path status = Paths.get(mediaFilePath.getParent().toString(), ".STATUS", mediaFile.getName() + ".json"); + // if (!status.toFile().exists()) { + // System.out.println("Nincs JSON: " + mediaFile.getAbsolutePath()); + // } + // + // if (mediaFile.length() == 0) { + // String houseid = mediaFile.getName().substring(0, mediaFile.getName().lastIndexOf(".")); + // final String idToCheck = houseid.substring(0, houseid.lastIndexOf("-")); + // MetadataType metadataType = MetadataTypeDetector.GuessMetadataType(idToCheck); + // if (metadataType == MetadataType.OctopusPlaceholder) { + // String query = String.format("select count(*) from mediafile where houseid like '%s%%'", idToCheck); + // IResultSetConsumer consumer = rs -> { + // if (rs.getInt(1) == 0) { + // + // File f = check_rd_nonzero_fellow(idToCheck, allFiles); + // if (f == null) + // System.out.println("0 még sincs korábbi: " + mediaFile.getAbsolutePath()); + // + // } + // return false; + // }; + // manager.executeQuery(query, consumer, null); + // } + // + // } + // + // } private void rescue_archive(RescueNEXIOMaterials processor, String clipName, long duration) throws Exception { /* @@ -220,6 +219,13 @@ public class Support { assertTrue(actual.length() > 0); } + @Test + public void test_ProjectCleanupMountedLocationStep() throws Exception { + //TODO + ProjectCleanupMountedLocationStep sut = new ProjectCleanupMountedLocationStep(); + sut.execute("c:\\_video\\projects", null, new JobRuntime()); + } + @Test public void test_Solr() throws Exception { //http://lucene.apache.org/solr/guide/7_5/using-solrj.html#using-solrj @@ -260,5 +266,4 @@ public class Support { System.out.println("itemid=" + i.getId() + " AND mediaid=" + m.getId() + " AND mediafileid=" + mf.getId()); } - } diff --git a/server/user.jobengine.executors/config/config.xml b/server/user.jobengine.executors/config/config.xml index 744df7f4..a4cfd3ba 100644 --- a/server/user.jobengine.executors/config/config.xml +++ b/server/user.jobengine.executors/config/config.xml @@ -7,6 +7,7 @@ + diff --git a/server/user.jobengine.executors/jobtemplates/delete-promo-materials.xml b/server/user.jobengine.executors/jobtemplates/delete-promo-materials.xml new file mode 100644 index 00000000..3dc9a7da --- /dev/null +++ b/server/user.jobengine.executors/jobtemplates/delete-promo-materials.xml @@ -0,0 +1,17 @@ + + + + + + + + + + + + + + + + + \ No newline at end of file diff --git a/server/user.jobengine.executors/src/user/jobengine/server/steps/CreateMissingLowresStep.java b/server/user.jobengine.executors/src/user/jobengine/server/steps/CreateMissingLowresStep.java index e623850e..9c4d021d 100644 --- a/server/user.jobengine.executors/src/user/jobengine/server/steps/CreateMissingLowresStep.java +++ b/server/user.jobengine.executors/src/user/jobengine/server/steps/CreateMissingLowresStep.java @@ -46,7 +46,8 @@ public class CreateMissingLowresStep extends JobStep { } catch (Exception e) { logger.catching(e); - logger.error(marker, e.getMessage()); + logger.info(marker, e.getMessage()); + throw e; } finally { setProgress(100); } diff --git a/server/user.jobengine.executors/src/user/jobengine/server/steps/ProjectCleanupMountedLocationStep.java b/server/user.jobengine.executors/src/user/jobengine/server/steps/ProjectCleanupMountedLocationStep.java new file mode 100644 index 00000000..27f1b5e1 --- /dev/null +++ b/server/user.jobengine.executors/src/user/jobengine/server/steps/ProjectCleanupMountedLocationStep.java @@ -0,0 +1,210 @@ +package user.jobengine.server.steps; + +import java.io.File; +import java.io.IOException; +import java.nio.file.DirectoryStream; +import java.nio.file.FileVisitResult; +import java.nio.file.FileVisitor; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.nio.file.SimpleFileVisitor; +import java.nio.file.attribute.BasicFileAttributes; +import java.text.ParseException; +import java.text.SimpleDateFormat; +import java.util.ArrayList; +import java.util.Collections; +import java.util.Date; +import java.util.List; + +import org.apache.commons.lang.StringUtils; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.apache.logging.log4j.Marker; + +import user.jobengine.server.IJobEngine; +import user.jobengine.server.IJobRuntime; +import user.jobengine.server.steps.MetadataTypeDetector.MetadataType; + +public class ProjectCleanupMountedLocationStep extends JobStep implements FileVisitor { + + private static final Logger logger = LogManager.getLogger(); + private static final String DATEFORMAT = "yyyyMMdd"; + private static final String DOT = "."; + private static final String STATUSFOLDER = ".STATUS"; + private static final String KILLDATEEXT = ".killdate"; + private Marker marker; + final int[] allCount = { 0 }; + final int[] currentCount = { 0 }; + private Path sourcePath; + private SimpleDateFormat dateFormat; + + private Date checkExpiration(List killDateFiles) { + Date killDate = null; + for (Path killDateFile : killDateFiles) { + Date currentKillDate = getKillDate(killDateFile); + if (currentKillDate == null) + continue; + if ((killDate != null && currentKillDate.after(killDate)) || killDate == null) + killDate = currentKillDate; + } + return new Date().after(killDate) ? killDate : null; + } + + @StepEntry + public Object[] execute(String sourceFolder, IJobEngine jobEngine, IJobRuntime jobRuntime) throws Exception { + marker = jobRuntime.getMarker(); + sourcePath = Paths.get(sourceFolder); + DirectoryStream directoryStream = null; + if (StringUtils.isBlank(sourcePath.toString())) { + logger.error(marker, "A folyamat 'sourcePath' bemeneti paramétere üres."); + throw new NullPointerException("System is not configured properly, 'sourceFolder' input parameter missing."); + } + + if (!sourcePath.toFile().exists() || !sourcePath.toFile().isDirectory()) { + logger.error(marker, "A {} mappa nem létezik.", sourceFolder); + throw new NullPointerException(String.format("Directory %s not exists.", sourceFolder)); + } + try { + setProgress(1); + dateFormat = new SimpleDateFormat(DATEFORMAT); + Files.walkFileTree(sourcePath, new SimpleFileVisitor() { + @Override + public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) throws IOException { + allCount[0]++; + return super.visitFile(file, attrs); + } + }); + Files.walkFileTree(sourcePath, this); + } catch (Exception e) { + logger.catching(e); + logger.error(marker, "Az '{}' mappa elérése sikertelen. A rendszer hibaüzenete: {}", sourcePath, e.getMessage()); + throw e; + } finally { + if (directoryStream != null) { + try { + directoryStream.close(); + } catch (IOException e) { + } + } + } + return null; + } + + private Date getKillDate(Path killDateFile) { + String fileName = killDateFile.getFileName().toString(); + int end = fileName.lastIndexOf(DOT); + if (end < 1) + return null; + int start = fileName.lastIndexOf(DOT, end - 1); + if (start < 0) + return null; + String strKillDate = fileName.substring(start + 1, end); + Date result = null; + if (StringUtils.isNumeric(strKillDate)) { + try { + result = dateFormat.parse(strKillDate); + } catch (ParseException e) { + logger.error(marker, "A {} fájl 'killdate' állománya hibás formátumú, a {} karaktersorozat nem konvertálható dátummá.", fileName, strKillDate); + return null; + } + } else + logger.error(marker, "A {} fájl 'killdate' állománya hibás formátumú, az dátum helyett ez áll: '{}'.", fileName, strKillDate); + return result; + } + + private List getKillDateFiles(Path filePath) { + String killDateFilePattern = String.format("%s.*%s", filePath.getFileName().toString(), KILLDATEEXT); + List result = new ArrayList<>(); + Path statusPath = null; + try { + statusPath = Paths.get(filePath.getParent().toString(), STATUSFOLDER); + } catch (Exception e) { + logger.catching(e); + return null; + } + File statusPathFile = statusPath.toFile(); + if (statusPathFile.exists() && statusPathFile.isDirectory()) { + try (DirectoryStream stream = Files.newDirectoryStream(statusPath, killDateFilePattern)) { + stream.forEach(p -> result.add(p)); + } catch (Exception e) { + logger.catching(e); + } + } + Collections.sort(result); + return result; + } + + private Path getProjectRootPath(Path filePath) { + Path result = null; + if (filePath.getNameCount() > sourcePath.getNameCount()) { + String dir = filePath.getName(sourcePath.getNameCount()).toString(); + String[] tokens = dir.split("-"); + if (tokens.length != 0 && MetadataTypeDetector.GuessMetadataType(tokens[0]) == MetadataType.TrafficPromo) + result = Paths.get(sourcePath.toAbsolutePath().toString(), dir); + } + return result; + } + + @Override + public FileVisitResult postVisitDirectory(Path dir, IOException exc) throws IOException { + return FileVisitResult.CONTINUE; + } + + @Override + public FileVisitResult preVisitDirectory(Path dir, BasicFileAttributes attrs) throws IOException { + if (dir.equals(sourcePath)) + return FileVisitResult.CONTINUE; + + Path parent = getProjectRootPath(dir); + if (parent == null) { + System.out.println("Skipping " + dir); + return FileVisitResult.SKIP_SUBTREE; + } + + return FileVisitResult.CONTINUE; + } + + private boolean processPathItem(Path filePath) { + currentCount[0]++; + setProgress(currentCount[0] * 100 / allCount[0]); + + if (filePath.toFile().isDirectory() || !filePath.toString().toUpperCase().endsWith(".EZP")) + return false; + + List killDateFiles = getKillDateFiles(filePath); + if (killDateFiles == null || killDateFiles.size() == 0) { + logger.warn(marker, "A {} fájlhoz nem található 'killdate' állomány.", filePath); + return false; + } + + if (killDateFiles.size() != 1) + logger.warn(marker, "A {} fájlhoz több 'killdate' állomány található, a legújabb dátum határozza meg a törlés időpontját.", filePath); + + Date killDate = checkExpiration(killDateFiles); + if (killDate == null) + return false; + + Path parent = getProjectRootPath(filePath); + if (parent != null) { + Path pathToDelete = Paths.get(sourcePath.toAbsolutePath().toString(), filePath.getName(sourcePath.getNameCount()).toString()); + //FileUtils.deleteDirectory(pathToDelete); + logger.info(marker, "A {} mappa törölhető", pathToDelete); + System.out.println(pathToDelete); + } else + logger.warn(marker, "A {} fájlhoz nem határozható meg a törlendő szülőkönyvtár.", filePath); + return true; + } + + @Override + public FileVisitResult visitFile(Path file, BasicFileAttributes attrs) throws IOException { + processPathItem(file); + return FileVisitResult.CONTINUE; + } + + @Override + public FileVisitResult visitFileFailed(Path file, IOException e) throws IOException { + logger.error(marker, "A {} fájl nem érhető el. A rendszer hibaüzenete: {}", file.toString(), e.getMessage()); + return FileVisitResult.CONTINUE; + } +} diff --git a/server/user.jobengine.osgi.server/pages/searchitems.zul b/server/user.jobengine.osgi.server/pages/searchitems.zul index 3a671d1a..2c6beaed 100644 --- a/server/user.jobengine.osgi.server/pages/searchitems.zul +++ b/server/user.jobengine.osgi.server/pages/searchitems.zul @@ -80,7 +80,7 @@
- +
diff --git a/server/user.jobengine.osgi.server/src/user/jobengine/server/steps/JobStep.java b/server/user.jobengine.osgi.server/src/user/jobengine/server/steps/JobStep.java index 79b8b9e7..fee97cd3 100644 --- a/server/user.jobengine.osgi.server/src/user/jobengine/server/steps/JobStep.java +++ b/server/user.jobengine.osgi.server/src/user/jobengine/server/steps/JobStep.java @@ -19,7 +19,7 @@ import user.jobengine.server.IJobRuntime; */ public class JobStep implements IJobStep { private static final Logger logger = LogManager.getLogger(); - private IJobRuntime jobRuntime = null; + protected IJobRuntime jobRuntime = null; private Object[] addParameter(Method stepEntry, Object[] inputs, Class type, Object parameter) { Object[] result = inputs; @@ -93,6 +93,7 @@ public class JobStep implements IJobStep { } protected void setProgress(int progress) { - jobRuntime.incrementProgress(progress); + if (jobRuntime != null) + jobRuntime.incrementProgress(progress); } } diff --git a/server/user.jobengine.osgi.server/src/user/jobengine/zk/model/SearchModel.java b/server/user.jobengine.osgi.server/src/user/jobengine/zk/model/SearchModel.java index 171ab5eb..238d533d 100644 --- a/server/user.jobengine.osgi.server/src/user/jobengine/zk/model/SearchModel.java +++ b/server/user.jobengine.osgi.server/src/user/jobengine/zk/model/SearchModel.java @@ -45,6 +45,7 @@ import org.zkoss.zul.Messagebox; import org.zkoss.zul.Paging; import org.zkoss.zul.Row; import org.zkoss.zul.Timer; +import org.zkoss.zul.West; import org.zkoss.zul.Window; import user.commons.ListUtils; @@ -63,12 +64,16 @@ public class SearchModel extends AsyncBaseModel { private static final String ARG = "arg"; private static final String MEDIA_ID = "showMediaID"; private static final String GENERIC = "Egyéb"; + private static final String ECHO_RECORDING = "ECHO visszarögzített"; private static final String RECORDING = "Visszarögzített"; private static final String RAW_AD = "Reklám nyers"; + private static final String ECHO_AD = "ECHO reklám"; private static final String AD = "Reklám"; private static final String RAW_PROMO = "Promo nyers"; + private static final String ECHO_PROMO = "ECHO promo"; private static final String PROMO = "Promo"; private static final String RAW_MATERIAL = "Műsor nyers"; + private static final String ECHO_MATERIAL = "ECHO műsor"; private static final String MATERIAL = "Műsor"; private static final String RAW_NEWS_MATERIAL = "Hír nyers"; private static final String NEWS_MATERIAL = "Hír bejátszó"; @@ -79,7 +84,7 @@ public class SearchModel extends AsyncBaseModel { private Map typeFilters; private Map columnHeaders; private String searchValue = null; - private CachedListModel searchResult = null; + private ListModelList searchResult = null; private ArchivedMedia selectedObject = null; private Validator allFilterCheckedValidator; private Validator itemSelectedValidator; @@ -121,16 +126,20 @@ public class SearchModel extends AsyncBaseModel { addTypeFilter(TypeFilter.as(NEWS_MATERIAL, "RED200", "ic_language_black_18dp.png")); addTypeFilter(TypeFilter.as(RAW_NEWS_MATERIAL, "RED100", "ic_perm_camera_mic_black_18dp.png")); addTypeFilter(TypeFilter.as(MATERIAL, "TEAL200", "ic_theaters_black_18dp.png")); + addTypeFilter(TypeFilter.as(ECHO_MATERIAL, "TEAL200", "ic_theaters_black_18dp.png")); addTypeFilter(TypeFilter.as(RAW_MATERIAL, "TEAL100", "ic_invert_colors_black_18dp.png")); addTypeFilter(TypeFilter.as(PROMO, "INDIGO200", "ic_picture_in_picture_black_18dp.png")); + addTypeFilter(TypeFilter.as(ECHO_PROMO, "INDIGO200", "ic_picture_in_picture_black_18dp.png")); addTypeFilter(TypeFilter.as(RAW_PROMO, "INDIGO100", "ic_bug_report_black_18dp.png")); addTypeFilter(TypeFilter.as(AD, "BLUE200", "ic_picture_in_picture_alt_black_18dp.png")); + addTypeFilter(TypeFilter.as(ECHO_AD, "BLUE200", "ic_picture_in_picture_alt_black_18dp.png")); addTypeFilter(TypeFilter.as(RAW_AD, "BLUE100", "ic_settings_brightness_black_18dp.png")); addTypeFilter(TypeFilter.as(RECORDING, "BROWN100", "ic_group_work_black_18dp.png")); + addTypeFilter(TypeFilter.as(ECHO_RECORDING, "BROWN100", "ic_group_work_black_18dp.png")); addTypeFilter(TypeFilter.as(GENERIC, "YELLOW200", "ic_view_quilt_black_18dp.png")); addTypeFilter(TypeFilter.as(GENERIC_ORIGINAL, "YELLOW200", "ic_view_quilt_black_18dp.png")); - topTypeFilterNames = new String[] { NEWS_MATERIAL, RAW_NEWS_MATERIAL, RECORDING, GENERIC }; - bottomTypeFilterNames = new String[] { MATERIAL, RAW_MATERIAL, PROMO, RAW_PROMO, AD, RAW_AD }; + topTypeFilterNames = new String[] { NEWS_MATERIAL, RAW_NEWS_MATERIAL, ECHO_RECORDING, RECORDING, GENERIC }; + bottomTypeFilterNames = new String[] { ECHO_MATERIAL, MATERIAL, RAW_MATERIAL, ECHO_PROMO, PROMO, RAW_PROMO, ECHO_AD, AD, RAW_AD }; columnHeaders = new LinkedHashMap<>(); columnHeaders.put("ID", "mediahouseid"); @@ -377,7 +386,7 @@ public class SearchModel extends AsyncBaseModel { return searchHistory; } - public CachedListModel getSearchResult() { + public ListModelList getSearchResult() { return searchResult; } @@ -497,6 +506,11 @@ public class SearchModel extends AsyncBaseModel { logger.info("onPaging"); } + @Command + public void onWestResize(@BindingParam("west") West _west) { + _west.invalidate(); + } + @Command public void openMetaDataDialog() { String template = "/pages/metadatadetails.zul"; diff --git a/server/user.jobengine.osgi.server/test/user/jobengine/server/IT/Support.java b/server/user.jobengine.osgi.server/test/user/jobengine/server/IT/Support.java index ceebf214..54619b60 100644 --- a/server/user.jobengine.osgi.server/test/user/jobengine/server/IT/Support.java +++ b/server/user.jobengine.osgi.server/test/user/jobengine/server/IT/Support.java @@ -30,6 +30,7 @@ import org.junit.AfterClass; import org.junit.BeforeClass; import org.junit.Test; +import com.ibm.nosql.json.api.BasicDBList; import com.ibm.nosql.json.api.BasicDBObject; import com.ibm.nosql.json.api.DB; import com.ibm.nosql.json.api.DBCollection; @@ -624,4 +625,32 @@ public class Support { result = result.substring(0, result.lastIndexOf(".")); System.out.println(result); } + + @Test + public void updateClipRecordDate() { + + DB db = NoSQLUtils.getNoSQLDB(); + DBCollection collection = db.getCollection("nexioclips"); + BasicDBList agencies = new BasicDBList(new Object[] { "schedule-rec" }); + + Calendar now = CalendarUtils.createZeroCalendar(); + QueryBuilder dateQueryBuilder = QueryBuilder.start("recorddate").lessThan(now.getTime()); + QueryBuilder queryBuilder = QueryBuilder.start("extagency").in(agencies).and(dateQueryBuilder.get()); + DBCursor cursor = collection.find(queryBuilder.get()).sort("recorddate", -1); + if (!cursor.hasNext()) + return; + List clips = ListUtils.cast(cursor.toArray()); + for (BasicDBObject c : clips) { + String name = c.getString("longnameid"); + Date recorddate = c.getDate("recorddate"); + if (name != null && name.toLowerCase().contains("_nodelete")) { + // recorddate.setMonth(2); + // recorddate.setDate(31); + System.out.println(name + " " + recorddate); + // collection.save(c); + } + + } + + } }