From: Vásáry Dániel Date: Thu, 4 Oct 2018 14:46:37 +0000 (+0000) Subject: git-tfs-id: [http://tfs.userrendszerhaz.hu:8080/tfs/DefaultCollection]$/MediaCube... X-Git-Url: http://git.useribm.hu/?a=commitdiff_plain;h=e85d76024cd07e4d1db3867b5fd1fd060cba7782;p=mediacube.git git-tfs-id: [tfs.userrendszerhaz.hu:8080/tfs/DefaultCollection]$/MediaCube;C31310 --- diff --git a/client/Maestro/Configuration/configuration-nle.json b/client/Maestro/Configuration/configuration-nle.json index cacd30fe..8baca8e5 100644 --- a/client/Maestro/Configuration/configuration-nle.json +++ b/client/Maestro/Configuration/configuration-nle.json @@ -1,5 +1,5 @@ { - "title": "NLE", + "title": "UNC2FTP_NEXIO_UNC", "active": true, "startInTray": false, "enableCustomMetadataId": true, @@ -111,64 +111,6 @@ "timeout": 1000 } }, - { - "label": "Octopus mappába küldés", - "processor": "FTPTargetProcessor", - "tag": "Betöltés", - "outputFormat": "%ID%", - "subFolderFormat": "%IDROOT%-%TEXT%", - "killDateDays": 7, - "saveArchiveMetadata": false, - "remote": { - "address": "ftp://10.10.1.100/OCTOPUS", - "userName": "mediacube", - "password": "Dn8t4gfHcK98o8hyPgLDhr5SgSji4JCxsfpMJsODikUp3nXgrM0UNCi45lLAK8ZOnmEneO44P9qpJ4QDqhctN6MxZodjJgdZTyoZKmSa+ECzEzLr/wPYNgxVaXrVotEy", - "timeout": 1000 - } - }, - { - "label": "Megtekintőbe küldés", - "processor": "FTPTargetProcessor", - "outputFormat": "%ID%-%SOURCENAME%", - "tag": "Betöltés", - "subFolderFormat": "%IDROOT%", - "killDateDays": 7, - "saveArchiveMetadata": false, - "remote": { - "address": "ftp://10.10.1.100/OCTOPUS", - "userName": "mediacube", - "password": "Dn8t4gfHcK98o8hyPgLDhr5SgSji4JCxsfpMJsODikUp3nXgrM0UNCi45lLAK8ZOnmEneO44P9qpJ4QDqhctN6MxZodjJgdZTyoZKmSa+ECzEzLr/wPYNgxVaXrVotEy", - "timeout": 1000 - } - }, - { - "label": "Traffic mappába küldés", - "processor": "FTPTargetProcessor", - "outputFormat": "%ID%-%SOURCENAME%", - "tag": "Betöltés", - "subFolderFormat": "%IDROOT%", - "killDateDays": 7, - "saveArchiveMetadata": false, - "remote": { - "address": "ftp://10.10.1.100/PLAYOUT_NLE", - "userName": "mediacube", - "password": "Dn8t4gfHcK98o8hyPgLDhr5SgSji4JCxsfpMJsODikUp3nXgrM0UNCi45lLAK8ZOnmEneO44P9qpJ4QDqhctN6MxZodjJgdZTyoZKmSa+ECzEzLr/wPYNgxVaXrVotEy", - "timeout": 1000 - } - }, - { - "label": "Lebonyba küldés", - "processor": "FTPTargetProcessor", - "outputFormat": "%ID%", - "tag": "Betöltés", - "killDateDays": 7, - "remote": { - "address": "ftp://10.10.1.100/PLAYOUT_CHECK", - "userName": "mediacube", - "password": "Dn8t4gfHcK98o8hyPgLDhr5SgSji4JCxsfpMJsODikUp3nXgrM0UNCi45lLAK8ZOnmEneO44P9qpJ4QDqhctN6MxZodjJgdZTyoZKmSa+ECzEzLr/wPYNgxVaXrVotEy", - "timeout": 1000 - } - }, { "label": "Helyi küldés", "processor": "UNCTargetProcessor", @@ -190,7 +132,7 @@ "deleteAfterCopy": true, "temporaryCopy": true, "remote": { - "address": "ftp://10.10.1.100/ARCHIVE/ONE", + "address": "ftp://10.10.1.100/ARCHIVE/TESZT", "userName": "mediacube", "password": "Dn8t4gfHcK98o8hyPgLDhr5SgSji4JCxsfpMJsODikUp3nXgrM0UNCi45lLAK8ZOnmEneO44P9qpJ4QDqhctN6MxZodjJgdZTyoZKmSa+ECzEzLr/wPYNgxVaXrVotEy", "timeout": 1000 diff --git a/client/Maestro/Configuration/configuration-studio.json b/client/Maestro/Configuration/configuration-studio.json index 74e6e227..c33de940 100644 --- a/client/Maestro/Configuration/configuration-studio.json +++ b/client/Maestro/Configuration/configuration-studio.json @@ -1,5 +1,5 @@ { - "title": "Stúdió", + "title": "NEXIO2UNC_FXP", "active": true, "startInTray": false, "enableCustomMetadataId": true, @@ -45,29 +45,15 @@ } ], "targets": [ - { - "label": "Octopus", - "processor": "FXPTargetProcessor", - "outputFormat": "%ID%", - "tag": "Adáskész", - "subFolderFormat": "%IDROOT%-%TEXT%", - "killDateDays": 15, - "saveArchiveMetadata": false, - "remote": { - "address": "ftp://10.10.1.100:21/OCTOPUS", - "userName": "mediacube", - "password": "Dn8t4gfHcK98o8hyPgLDhr5SgSji4JCxsfpMJsODikUp3nXgrM0UNCi45lLAK8ZOnmEneO44P9qpJ4QDqhctN6MxZodjJgdZTyoZKmSa+ECzEzLr/wPYNgxVaXrVotEy", - "timeout": 1000 - } - }, { "label": "Archiválás", "processor": "FXPTargetProcessor", "outputFormat": "%ID%", "tag": "Archiválás", "saveArchiveMetadata": true, + "deleteAfterCopy": false, "remote": { - "address": "ftp://10.10.1.100:21/ARCHIVE/ONE", + "address": "ftp://10.10.1.100:21/ARCHIVE/TESZT", "userName": "mediacube", "password": "Dn8t4gfHcK98o8hyPgLDhr5SgSji4JCxsfpMJsODikUp3nXgrM0UNCi45lLAK8ZOnmEneO44P9qpJ4QDqhctN6MxZodjJgdZTyoZKmSa+ECzEzLr/wPYNgxVaXrVotEy", "timeout": 1000 diff --git a/client/Maestro/Sources/NexioRESTSource.cs b/client/Maestro/Sources/NexioRESTSource.cs index c13bee02..c3b463ef 100644 --- a/client/Maestro/Sources/NexioRESTSource.cs +++ b/client/Maestro/Sources/NexioRESTSource.cs @@ -285,7 +285,7 @@ namespace Maestro.Sources { }); }); } - Debug.WriteLine(string.Format("Changed {0}", item.Name)); + //Debug.WriteLine(string.Format("Changed {0}", item.Name)); } private void OnCreated(JToken token) { @@ -301,7 +301,7 @@ namespace Maestro.Sources { Source = parent }); }); - Debug.WriteLine(string.Format("Created {0}", item.Name)); + //Debug.WriteLine(string.Format("Created {0}", item.Name)); } private void OnDeleted(JToken d) { diff --git a/client/MaestroShared/Targets/FTPTargetProcessor.cs b/client/MaestroShared/Targets/FTPTargetProcessor.cs index 7d7fbd58..a3d909ff 100644 --- a/client/MaestroShared/Targets/FTPTargetProcessor.cs +++ b/client/MaestroShared/Targets/FTPTargetProcessor.cs @@ -12,9 +12,9 @@ namespace MaestroShared.Targets { public class FTPTargetProcessor : UNCTargetProcessor { private const string PATTERN_CONCAT = "{0}{1}"; - private const string _226 = "226"; + protected const string _226 = "226"; protected const string UPLOAD_ERROR = "Feltöltési hiba."; - private const string _200 = "200"; + protected const string _200 = "200"; private const string SITE0011 = "SITE 0011"; private const string _1211 = "1211"; private readonly Logger logger = LogManager.GetCurrentClassLogger(); @@ -168,7 +168,7 @@ namespace MaestroShared.Targets { InternetProtocolVersions = FtpIpVersion.IPv4, UploadDataType = FtpDataType.Binary, DownloadDataType = FtpDataType.Binary, - Credentials = new NetworkCredential(connection.UserName, connection.Password) + Credentials = new NetworkCredential(connection.UserName, connection.Password), }; result.Connect(); return result; diff --git a/client/MaestroShared/Targets/FXPTargetProcessor.cs b/client/MaestroShared/Targets/FXPTargetProcessor.cs index e1ee7d3e..8882da73 100644 --- a/client/MaestroShared/Targets/FXPTargetProcessor.cs +++ b/client/MaestroShared/Targets/FXPTargetProcessor.cs @@ -4,13 +4,18 @@ using MaestroShared.Target; using NLog; using System; using System.IO; +using System.Linq; +using System.Text; using System.Text.RegularExpressions; using System.Threading; using System.Windows.Forms; +using System.Xml; +using System.Xml.Linq; namespace MaestroShared.Targets { public class FXPTargetProcessor : FTPTargetProcessor { + protected const string DOWNLOAD_ERROR = "Letöltési hiba."; private const string LITERAL_SPACE = "%20"; private const string SPACE = " "; private Logger logger = LogManager.GetCurrentClassLogger(); @@ -48,98 +53,117 @@ namespace MaestroShared.Targets { try { sourceFTP = CreateClient(sourceConfig.Remote); - string input = Uri.UnescapeDataString(inputUri.AbsolutePath); - sourceFileLength = sourceFTP.GetFileSize(input); - bool nexioTarget = IsNexioServer(targetFTP); - if (nexioTarget) - sourceFileLength = sourceFileLength / 2; + Input = Uri.UnescapeDataString(inputUri.AbsolutePath); + if (Input.LastIndexOf("/") == 0) + Input = Input.Replace("/", ""); + sourceFileLength = sourceFTP.GetFileSize(Input); - FtpReply replyPASV = targetFTP.Execute("PASV"); - if (!replyPASV.Success) - throw new Exception(replyPASV.ErrorMessage); + bool isNexioSource = sourceFTP.IsNexio(); + //Nexio eseten frame-eket kerdunk le + if (isNexioSource) + sourceFileLength = GetNexioLength(); - Regex regex = new Regex(Regex.Escape("(") + "(.*)" + Regex.Escape(")")); - var v = regex.Match(replyPASV.Message); - string port = v.Groups[1].ToString(); + string port = targetFTP.Pasv(); + sourceFTP.Command("PORT " + port); - FtpReply replyPORT = sourceFTP.Execute("PORT " + port); - if (!replyPORT.Success) - throw new Exception(replyPORT.ErrorMessage); - - FtpReply sourceReplyTYPE = sourceFTP.Execute("TYPE I"); - if (!sourceReplyTYPE.Success) - throw new Exception(sourceReplyTYPE.ErrorMessage); - - if (input.LastIndexOf("/") == 0) - input = input.Replace("/", ""); - FtpReply replyRETR = sourceFTP.Execute("RETR " + input); + sourceFTP.Command("TYPE I"); + FtpReply replyRETR = sourceFTP.Execute("RETR " + Input); + //TODO nem lehet jobb? if (!replyRETR.Success) { - input = inputUri.Segments[inputUri.Segments.Length - 1]; - input = input.Replace("%20", " "); - replyRETR = sourceFTP.Execute("RETR " + input); - + Input = inputUri.Segments[inputUri.Segments.Length - 1]; + Input = Input.Replace("%20", " "); + replyRETR = sourceFTP.Execute("RETR " + Input); if (!replyRETR.Success) throw new Exception(replyRETR.ErrorMessage); } - FtpReply targetReplyTYPE = targetFTP.Execute("TYPE I"); - if (!targetReplyTYPE.Success) - throw new Exception(targetReplyTYPE.ErrorMessage); + targetFTP.Command("TYPE I"); string currentWorkingDir = targetFTP.GetWorkingDirectory(); logger.Debug("Current target working directory is {0}", currentWorkingDir); - bool useTemporaryName = !nexioTarget && Parameters.TargetConfig.TemporaryCopy; string targetFilePath = OutputName; + bool useTemporaryName = !targetFTP.IsNexio() && Parameters.TargetConfig.TemporaryCopy; if (useTemporaryName) targetFilePath = OutputName + ".tmp"; - FtpReply replySTOR = targetFTP.Execute("STOR " + targetFilePath); - if (!replySTOR.Success) - throw new Exception(replySTOR.ErrorMessage); - - monitorFTP = CreateClient(Parameters.TargetConfig.Remote); - monitorFTP.SetWorkingDirectory(workingDir); - FtpReply monitorReplyTYPE = monitorFTP.Execute("TYPE I"); - if (!monitorReplyTYPE.Success) - throw new Exception(targetReplyTYPE.ErrorMessage); - //if (!FileExists(OutputName)) - // throw new Exception("A cél állomány nem jött létre."); - long lastSize = 0; + targetFTP.Command("STOR " + targetFilePath); + + if (!isNexioSource) { + monitorFTP = CreateClient(Parameters.TargetConfig.Remote); + monitorFTP.SetWorkingDirectory(workingDir); + monitorFTP.Command("TYPE I"); + } + while (uploadedFileLength != sourceFileLength) { - uploadedFileLength = monitorFTP.GetFileSize(OutputName); - /* - FtpReply statRETR = sourceFTP.Execute("STAT"); - if (!statRETR.Success) - Debug.WriteLine($"Overall {overall}, last {lastSize}"); - else - Debug.WriteLine($"Overall {overall}, last {lastSize} site {statRETR.Message}"); -*/ - if (uploadedFileLength == lastSize && uploadedFileLength > 0) { - Progress = 100; - break; + if (isNexioSource) { + string response = sourceFTP.Command("STAT"); + logger.Info("Monitor: " + response); + if (response.EndsWith("No System Status") || response.EndsWith("Transfer Complete(TRANSACTION_SUCCESS)")) { + Progress = 100; + break; + } + if (!response.Contains("Fields Transferred")) + throw new Exception($"Unsopported STAT response: {response}"); + long.TryParse(response.Replace("STAT Fields Transferred", "").Trim(), out uploadedFileLength); + + logger.Info($"Progress {uploadedFileLength} / {sourceFileLength}"); + } else { - int currentProgress = (int)((double)uploadedFileLength / sourceFileLength * 100); - Progress = currentProgress > 100 ? 100 : currentProgress; - lastSize = uploadedFileLength; + uploadedFileLength = monitorFTP.GetFileSize(targetFilePath); } - Thread.Sleep(1000); + Progress = (int)((double)uploadedFileLength / sourceFileLength * 100); + Thread.Sleep(500); } if (useTemporaryName) { if (!targetFTP.MoveFile(targetFilePath, OutputName)) throw new Exception(UPLOAD_ERROR); } - } - catch (Exception e) { + } catch (Exception e) { throw e; } //logger.Debug("Done"); } + public static String ReadFully(Stream input) { + byte[] buffer = new byte[1024]; + using (MemoryStream ms = new MemoryStream()) { + int read; + while ((read = input.Read(buffer, 0, buffer.Length)) > 0) { + ms.Write(buffer, 0, read); + } + return ms.ToString(); + } + } + + private long GetNexioLength() { + long result = 0; + string xml = Input.Replace(".MXF", ".XML"); + using (MemoryStream ms = new MemoryStream()) { + try { + //Binary modban lekerdezi a meretet, ez nem tamogatott + sourceFTP.DownloadDataType = FtpDataType.ASCII; + if (!sourceFTP.Download(ms, xml)) + throw new Exception(DOWNLOAD_ERROR); + //a Unicode BOM 2 byte + ms.Seek(2, SeekOrigin.Begin); + XDocument xdoc = XDocument.Load(ms); + long.TryParse(xdoc.Descendants("Duration").Single().Value, out result); + result *= 2; + } catch (Exception e) { + logger.Error(e.Message); + throw e; + } finally { + sourceFTP.DownloadDataType = FtpDataType.Binary; + } + } + return result; + } + protected override void ValidateTransfer() { - if (!IsNexioServer(sourceFTP) && !IsNexioServer(monitorFTP)) { + if (!targetFTP.IsNexio() && !sourceFTP.IsNexio()) { if (sourceFileLength != uploadedFileLength) throw new Exception(COPY_SIZE_ERROR); } @@ -151,6 +175,51 @@ namespace MaestroShared.Targets { TerminateClient(sourceFTP); } + protected bool FileExists(FtpClient client, string path) { + long x = -1; + try { + x = client.GetFileSize(path); + } catch (Exception e) { + logger.Warn(e.Message); + } + return x > -1; + } + + protected override void DeleteAfterCopy() { + logger.Debug("Deleting source: " + Input); + try { + if (FileExists(sourceFTP, Input)) + sourceFTP.DeleteFile(Input); + } catch (Exception e) { + logger.Error(e.Message); + } + } + + } + + public static class FtpClientExtensions { + public static String Command(this FtpClient client, String command) { + FtpReply response = client.Execute(command); + if (!response.Success) + throw new Exception($"Error executing {command}. Error message: {response.ErrorMessage}"); + return response.Message; + } + + public static String Pasv(this FtpClient client) { + String pasv = client.Command("PASV"); + Regex regex = new Regex(Regex.Escape("(") + "(.*)" + Regex.Escape(")")); + return regex.Match(pasv).Groups[1].ToString(); + } + + public static bool IsNexio(this FtpClient client) { + bool result = false; + try { + //Device Type lekerdezese + string response = client.Command("SITE 0011"); + result = response.Contains("1211"); + } catch { } + return result; + } } } diff --git a/client/MaestroShared/Targets/UNCTargetProcessor.cs b/client/MaestroShared/Targets/UNCTargetProcessor.cs index 77615dfc..3a111c55 100644 --- a/client/MaestroShared/Targets/UNCTargetProcessor.cs +++ b/client/MaestroShared/Targets/UNCTargetProcessor.cs @@ -93,6 +93,8 @@ namespace MaestroShared.Targets { Status = READY; Message = SUCCESS; } + if (Parameters.TargetConfig.DeleteAfterCopy) + DeleteAfterCopy(); ExecuteCompleted(); } @@ -263,17 +265,6 @@ namespace MaestroShared.Targets { } protected virtual void AfterExecute() { - - if (Parameters.SourceConfig is UNCSource && Parameters.TargetConfig.DeleteAfterCopy) { - logger.Debug("Deleting source: " + Input); - try { - if (FileExists(inputFile.FullName)) - DeleteFile(inputFile.FullName); - } catch (Exception e) { - logger.Error(e.Message); - } - } - logger.Trace(Strings.ENTRY); Finished = DateTime.Now; workFlowAction.finished = Finished; @@ -282,6 +273,16 @@ namespace MaestroShared.Targets { logger.Trace(Strings.EXIT); } + protected virtual void DeleteAfterCopy() { + logger.Debug("Deleting source: " + Input); + try { + if (Parameters.SourceConfig is UNCSource && FileExists(inputFile.FullName)) + DeleteFile(inputFile.FullName); + } catch (Exception e) { + logger.Error(e.Message); + } + } + protected virtual bool FileExists(string path) { return File.Exists(path); } diff --git a/server/-configuration/scheduledjobs.json b/server/-configuration/scheduledjobs.json index b39af791..2a8407b7 100644 --- a/server/-configuration/scheduledjobs.json +++ b/server/-configuration/scheduledjobs.json @@ -17,8 +17,8 @@ "name" : "SYS: retrieve-ondemand", "template": "retrieve-ondemand.xml", "parameters": [ - {"name": "globalRetrievePath", "value": "file://isilon.intra.echotv.hu", "type": "java.lang.String"}, - {"name": "localRetrievePath", "value": "/mnt/ISILON", "type": "java.lang.String"}, + {"name": "globalRetrievePath", "value": "c:\\temp", "type": "java.lang.String"}, + {"name": "localRetrievePath", "value": "c:\\temp", "type": "java.lang.String"}, {"name": "materialOutputFolder", "value": "PLAYOUT_NLE", "type": "java.lang.String"}, {"name": "promoOutputFolder", "value": "PROMO_NLE", "type": "java.lang.String"}, {"name": "advertisementOutputFolder", "value": "REKLAM_NLE", "type": "java.lang.String"}, @@ -64,7 +64,7 @@ "template": "import-statistics.xml", "cronexpression": "0 0 0/1 1/1 * ? *", "parameters": [ - {"name": "daysBeforeNow", "value": 5, "type": "java.lang.Integer"} + {"name": "daysBeforeNow", "value": 7, "type": "java.lang.Integer"} ] }, { diff --git a/server/user.jobengine.executors/src/user/jobengine/server/steps/BatchRetrieveForkStep.java b/server/user.jobengine.executors/src/user/jobengine/server/steps/BatchRetrieveForkStep.java index cf2bc4df..101e9047 100644 --- a/server/user.jobengine.executors/src/user/jobengine/server/steps/BatchRetrieveForkStep.java +++ b/server/user.jobengine.executors/src/user/jobengine/server/steps/BatchRetrieveForkStep.java @@ -19,23 +19,28 @@ public class BatchRetrieveForkStep extends JobStep { private static final String MEDIACUBEMEDIA = "mediaCubeMedia"; private static final String RECIPIENT = "successRecipient"; private static final String HOUSEID = "houseId"; - private MediaCubeMarker marker; + private MediaCubeMarker systemMarker; @StepEntry public Object[] execute(List basket, String houseId, String recipient, String targetPathType, IJobEngine jobEngine, IJobRuntime jobRuntime) throws Exception { - marker = (MediaCubeMarker) jobRuntime.getMarker(); - marker.setTo(recipient); + systemMarker = (MediaCubeMarker) jobRuntime.getMarker(); + systemMarker.setTo(recipient); if (basket == null || basket.size() == 0) return null; setProgress(10); + + // StringBuilder sb = new StringBuilder(); + // sb.append("A visszatöltések elindultak az alábbi állományokra:
"); if (jobRuntime.forkPrepare()) { for (Media mediaCubeMedia : basket) { + // sb.append(mediaCubeMedia.getMediaFilesName() + "
"); submit(mediaCubeMedia, recipient, houseId, targetPathType, jobEngine, jobRuntime); } } setProgress(50); + // logger.info(new MediaCubeMarker(recipient), sb.toString()); jobRuntime.forkWaitComplete(); setProgress(100); return null; @@ -54,7 +59,7 @@ public class BatchRetrieveForkStep extends JobStep { ((MediaCubeMarker) child.getMarker()).setTo(recipient); } catch (Exception e) { logger.catching(e); - logger.error(marker, "Hiba a kötegelt visszatöltésben. A rendszer üzenete: {}", e.getMessage()); + logger.error(systemMarker, "Hiba a kötegelt visszatöltésben. A rendszer üzenete: {}", e.getMessage()); } } diff --git a/server/user.jobengine.executors/src/user/jobengine/server/steps/ImportStatisticsStep.java b/server/user.jobengine.executors/src/user/jobengine/server/steps/ImportStatisticsStep.java index ad00be9a..0ac75423 100644 --- a/server/user.jobengine.executors/src/user/jobengine/server/steps/ImportStatisticsStep.java +++ b/server/user.jobengine.executors/src/user/jobengine/server/steps/ImportStatisticsStep.java @@ -198,7 +198,8 @@ public class ImportStatisticsStep extends JobStep { BasicDBObject clip = (BasicDBObject) clipCollection.findOne(new BasicDBObject(LONGNAMEID, clipName)); if (clip != null) { long duration = NoSQLUtils.asLong(clip, DURATION); - if (duration > 0) { + //az ures klipp 1 kocka hosszu + if (duration > 1) { mosObject.put("duration", duration); foundAnyClip = true; } @@ -271,6 +272,8 @@ public class ImportStatisticsStep extends JobStep { continue; long parentStoryId = NoSQLUtils.asLong(story, IOctopusAPI.PARENT_STORY_ID); + logger.info("Processing parentStoryId {}", parentStoryId); + String key = String.format("%d-%s", parentStoryId, objId); if (processed.contains(key)) continue; @@ -316,7 +319,7 @@ public class ImportStatisticsStep extends JobStep { NoSQLUtils.addLong(perReporterStat, "ingest_duration", ingestDuration); } - BasicDBObject archiveInfo = manager.getArchiveInfo(scheduledDate, String.valueOf(parentStoryId)); + BasicDBObject archiveInfo = ItemManagerExtensions.getArchiveInfo(manager, parentStoryId); if (archiveInfo != null) { long archiveCount = archiveInfo.getLong("count"); long archiveDuration = archiveInfo.getLong("duration"); diff --git a/server/user.jobengine.executors/src/user/jobengine/server/steps/ItemManagerExtensions.java b/server/user.jobengine.executors/src/user/jobengine/server/steps/ItemManagerExtensions.java index 6cc8a316..2e6852d2 100644 --- a/server/user.jobengine.executors/src/user/jobengine/server/steps/ItemManagerExtensions.java +++ b/server/user.jobengine.executors/src/user/jobengine/server/steps/ItemManagerExtensions.java @@ -1,10 +1,31 @@ package user.jobengine.server.steps; +import com.ibm.nosql.json.api.BasicDBObject; + import user.jobengine.db.IItemManager; import user.jobengine.db.IResultSetConsumer; public class ItemManagerExtensions { + public static BasicDBObject getArchiveInfo(IItemManager manager, long houseid) { + final BasicDBObject[] result = { null }; + StringBuilder query = new StringBuilder(); + query.append("select count(*) as count, sum(length) as duration FROM media"); + query.append(" "); + query.append(String.format("where houseid='%d' and itemtypeid = 82", houseid)); + query.append(" "); + query.append("group by houseid"); + IResultSetConsumer consumer = rs -> { + BasicDBObject o = new BasicDBObject(); + o.put("count", rs.getLong("count")); + o.put("duration", rs.getLong("duration")); + result[0] = o; + return false; + }; + manager.executeQuery(query.toString(), consumer, null); + return result[0]; + } + public static long getExistingRundownMedia(IItemManager manager, String houseid) { final long[] result = { 0 }; StringBuilder query = new StringBuilder(); diff --git a/server/user.jobengine.osgi.db/sql/2-CreateStructure.db2 b/server/user.jobengine.osgi.db/sql/2-CreateStructure.db2 index fc42b36d..21ff860c 100644 --- a/server/user.jobengine.osgi.db/sql/2-CreateStructure.db2 +++ b/server/user.jobengine.osgi.db/sql/2-CreateStructure.db2 @@ -385,3 +385,35 @@ CREATE TABLE WORKFLOWACTION CREATE INDEX IDX_WORKFLOWACTION_FINISHED ON WORKFLOWACTION ("FINISHED")@ CREATE INDEX IDX_WORKFLOWACTION_HOUSEID ON WORKFLOWACTION ("HOUSEID")@ + +CREATE TABLE MEDIADESCRIPTION ( + ID BIGINT GENERATED ALWAYS AS IDENTITY (START WITH 1, INCREMENT BY 1, CACHE 20) NOT NULL, + ITEMID BIGINT NOT NULL, + MEDIAID BIGINT NOT NULL, + MEDIAFILEID BIGINT NOT NULL, + DESCRIPTION CLOB(400000) NOT LOGGED COMPACT, + CONSTRAINT PK_MEDIADESCRIPTION PRIMARY KEY(ID) +) +@ + +ALTER TABLE DB2ADMIN.MEDIADESCRIPTION + ADD CONSTRAINT FK_MEDIADESCRIPTION_MEDIAID + FOREIGN KEY(MEDIAID) + REFERENCES DB2ADMIN.MEDIA(ID) + ON DELETE CASCADE + ON UPDATE NO ACTION +@ +ALTER TABLE DB2ADMIN.MEDIADESCRIPTION + ADD CONSTRAINT FK_MEDIADESCRIPTION_MEDIAFILEID + FOREIGN KEY(MEDIAFILEID) + REFERENCES DB2ADMIN.MEDIAFILE(ID) + ON DELETE CASCADE + ON UPDATE NO ACTION +@ +ALTER TABLE DB2ADMIN.MEDIADESCRIPTION + ADD CONSTRAINT FK_MEDIADESCRIPTION_ITEMID + FOREIGN KEY(ITEMID) + REFERENCES DB2ADMIN.ITEM(ID) + ON DELETE CASCADE + ON UPDATE NO ACTION +@ diff --git a/server/user.jobengine.osgi.db/sql/4-CreateTrigger.db2 b/server/user.jobengine.osgi.db/sql/4-CreateTrigger.db2 index e734f387..ce9888c0 100644 --- a/server/user.jobengine.osgi.db/sql/4-CreateTrigger.db2 +++ b/server/user.jobengine.osgi.db/sql/4-CreateTrigger.db2 @@ -1,16 +1,104 @@ -- execute with -- db2 -td@ -vmf 2-CreateStructure.db2 -DROP TRIGGER TRG_INSERT_ITEM@ +--DROP TRIGGER TRG_INSERT_ITEM@ -CREATE TRIGGER TRG_INSERT_ITEM - AFTER INSERT ON ITEM - REFERENCING NEW AS r - FOR EACH ROW -BEGIN ATOMIC - DECLARE x VARCHAR(255); - CALL SYSPROC.SYSTS_UPDATE(NULL, 'item_title', NULL, 'hu_HU', :x); - CALL SYSPROC.SYSTS_UPDATE(NULL, 'item_desc', NULL, 'hu_HU', :x); - CALL SYSPROC.SYSTS_UPDATE(NULL, 'item_houseid', NULL, 'hu_HU', :x); -END -@ +--CREATE TRIGGER TRG_INSERT_ITEM + --AFTER INSERT ON ITEM + --REFERENCING NEW AS r + --FOR EACH ROW +--BEGIN ATOMIC +-- DECLARE x VARCHAR(255); +-- CALL SYSPROC.SYSTS_UPDATE(NULL, 'item_title', NULL, 'hu_HU', :x); +-- CALL SYSPROC.SYSTS_UPDATE(NULL, 'item_desc', NULL, 'hu_HU', :x); +-- CALL SYSPROC.SYSTS_UPDATE(NULL, 'item_houseid', NULL, 'hu_HU', :x); +--END +--@ + + +CREATE TRIGGER DB2ADMIN.TRG_MEDIAFILE_AFTER_INSERT after insert +on mediafile +referencing new as n +for each row +when (n.storeid = 1) +begin atomic + insert into mediadescription (itemid, mediaid, mediafileid, description) + select + itemid, + mediaid, + mediafileid, + COALESCE(hundate(archived), '') concat ' ' + concat COALESCE(defdate(archived), '') concat ' ' + concat COALESCE(itemhouseid, '') concat ' ' + concat COALESCE(itemtitle, '') concat ' ' + concat COALESCE(itemdescription,'') concat ' ' + concat COALESCE(mediahouseid, '') concat ' ' + concat COALESCE(mediatitle, '') concat ' ' + concat COALESCE(mediadescription, '') concat ' ' + concat COALESCE(mediafilehouseid, '') + from vw_items_description + where mediafileid = n.id; +end +GO + +CREATE TRIGGER DB2ADMIN.TRG_MEDIAFILE_AFTER_UPDATE after update of HOUSEID +on mediafile +referencing new as n +for each row +when (n.storeid = 1) +begin atomic + update mediadescription set description = ( + select + COALESCE(hundate(archived), '') concat ' ' + concat COALESCE(defdate(archived), '') concat ' ' + concat COALESCE(itemhouseid, '') concat ' ' + concat COALESCE(itemtitle, '') concat ' ' + concat COALESCE(itemdescription,'') concat ' ' + concat COALESCE(mediahouseid, '') concat ' ' + concat COALESCE(mediatitle, '') concat ' ' + concat COALESCE(mediadescription, '') concat ' ' + concat COALESCE(mediafilehouseid, '') + from vw_items_description + where mediafileid = n.id); +end +GO + +CREATE TRIGGER DB2ADMIN.TRG_MEDIA_AFTER_UPDATE after update on media +referencing new as n +for each row +begin atomic + update mediadescription set description = ( + select + COALESCE(hundate(archived), '') concat ' ' + concat COALESCE(defdate(archived), '') concat ' ' + concat COALESCE(itemhouseid, '') concat ' ' + concat COALESCE(itemtitle, '') concat ' ' + concat COALESCE(itemdescription,'') concat ' ' + concat COALESCE(mediahouseid, '') concat ' ' + concat COALESCE(mediatitle, '') concat ' ' + concat COALESCE(mediadescription, '') concat ' ' + concat COALESCE(mediafilehouseid, '') + from vw_items_description + where mediaid = n.id); +end +GO + +CREATE TRIGGER DB2ADMIN.TRG_ITEM_AFTER_UPDATE after update on item +referencing new as n +for each row +begin atomic + update mediadescription set description = ( + select + COALESCE(hundate(archived), '') concat ' ' + concat COALESCE(defdate(archived), '') concat ' ' + concat COALESCE(itemhouseid, '') concat ' ' + concat COALESCE(itemtitle, '') concat ' ' + concat COALESCE(itemdescription,'') concat ' ' + concat COALESCE(mediahouseid, '') concat ' ' + concat COALESCE(mediatitle, '') concat ' ' + concat COALESCE(mediadescription, '') concat ' ' + concat COALESCE(mediafilehouseid, '') + from vw_items_description + where itemid = n.id); +end +GO diff --git a/server/user.jobengine.osgi.db/src/user/jobengine/db/IItemManager.java b/server/user.jobengine.osgi.db/src/user/jobengine/db/IItemManager.java index a3353537..9f4fa388 100644 --- a/server/user.jobengine.osgi.db/src/user/jobengine/db/IItemManager.java +++ b/server/user.jobengine.osgi.db/src/user/jobengine/db/IItemManager.java @@ -128,8 +128,6 @@ public interface IItemManager extends IEntityPersister { */ List getAllCached(Class baseClass); - BasicDBObject getArchiveInfo(Calendar scheduleDate, String houseId); - /** * Visszaadja az entitásnak megfelelő DAO objektumot. * diff --git a/server/user.jobengine.osgi.db/src/user/jobengine/db/ItemManager.java b/server/user.jobengine.osgi.db/src/user/jobengine/db/ItemManager.java index 2c1c95ab..2824fe2c 100644 --- a/server/user.jobengine.osgi.db/src/user/jobengine/db/ItemManager.java +++ b/server/user.jobengine.osgi.db/src/user/jobengine/db/ItemManager.java @@ -484,51 +484,6 @@ public class ItemManager extends MemoryCache implements IItemManager { return result; } - @Override - public BasicDBObject getArchiveInfo(Calendar scheduleDate, String houseId) { - BasicDBObject result = null; - ResultSet rs = null; - PreparedStatement st = null; - DefaultContext context = getDbContext(); - Connection connection = context.getConnection(); - SimpleDateFormat df = new SimpleDateFormat("yyyy-MM-dd"); - try { - String date = df.format(scheduleDate.getTime()); - String query = String.format( - "SELECT COUNT(*) as count, SUM(length) as duration FROM VW_ITEMS WHERE mediahouseid='%s' AND archived LIKE '%s%%' GROUP BY mediahouseid", - houseId, date); - st = connection.prepareStatement(query); - rs = st.executeQuery(); - if (rs.next()) { - result = new BasicDBObject(); - result.put("count", rs.getLong("count")); - result.put("duration", rs.getLong("duration")); - } - - connection.commit(); - } catch (Exception e) { - try { - connection.rollback(); - } catch (Exception e1) { - } - throwError(e); - } finally { - try { - if (rs != null) - rs.close(); - } catch (Exception e1) { - } - try { - if (st != null) - st.close(); - } catch (Exception e1) { - } - putDbContext(context); - } - - return result; - } - @Override public IEntityBaseDAO getBaseDAO(Class classInfo) { IEntityBaseDAO entityBaseDb = null; diff --git a/server/user.jobengine.osgi.server/pages/metadatadetails.zul b/server/user.jobengine.osgi.server/pages/metadatadetails.zul index 8a578d57..2ac11633 100644 --- a/server/user.jobengine.osgi.server/pages/metadatadetails.zul +++ b/server/user.jobengine.osgi.server/pages/metadatadetails.zul @@ -50,7 +50,7 @@ - + diff --git a/server/user.jobengine.osgi.server/src/user/jobengine/zk/model/MetadataDetailsModel.java b/server/user.jobengine.osgi.server/src/user/jobengine/zk/model/MetadataDetailsModel.java index 32ab33f0..1e413dbe 100644 --- a/server/user.jobengine.osgi.server/src/user/jobengine/zk/model/MetadataDetailsModel.java +++ b/server/user.jobengine.osgi.server/src/user/jobengine/zk/model/MetadataDetailsModel.java @@ -99,8 +99,9 @@ public class MetadataDetailsModel extends BaseModel { @Command public void save() { + manager.modify(archivedMedia.getItem()); manager.modify(archivedMedia.getMedia()); - logger.info("Metadat of media was updated by user, id of media: {}", archivedMedia.getMedia().getHouseId()); + logger.info("Metadata modified: {}", archivedMedia.getMedia().getId()); wnd.detach(); } diff --git a/server/user.jobengine.osgi.server/src/user/jobengine/zk/model/NewsHistoryModel.java b/server/user.jobengine.osgi.server/src/user/jobengine/zk/model/NewsHistoryModel.java index c52e3af2..ee5fc66b 100644 --- a/server/user.jobengine.osgi.server/src/user/jobengine/zk/model/NewsHistoryModel.java +++ b/server/user.jobengine.osgi.server/src/user/jobengine/zk/model/NewsHistoryModel.java @@ -83,16 +83,17 @@ public class NewsHistoryModel { SimpleDateFormat dateFormat = new SimpleDateFormat("yyyyMMdd"); StringBuilder data = new StringBuilder(); - boolean headeradded = false; + for (String column : columnList) { + data.append(column).append(";"); + } + data.append("\r\n"); + for (BasicDBObject o : dataList) { - if (!headeradded) { - for (String key : o.keySet()) - data.append(key).append(";"); - data.append("\r\n"); - headeradded = true; + for (String column : columnList) { + String columnData = NoSQLUtils.asString(o, column); + columnData = columnData == null ? "" : columnData; + data.append(columnData).append(";"); } - for (String key : o.keySet()) - data.append(o.getString(key)).append(";"); data.append("\r\n"); } diff --git a/server/user.jobengine.osgi.server/test/user/jobengine/server/IT/Support.java b/server/user.jobengine.osgi.server/test/user/jobengine/server/IT/Support.java index 262abe82..62074d69 100644 --- a/server/user.jobengine.osgi.server/test/user/jobengine/server/IT/Support.java +++ b/server/user.jobengine.osgi.server/test/user/jobengine/server/IT/Support.java @@ -362,7 +362,7 @@ public class Support { } private void processLowresDuplicateGroup(String fileName) { - System.out.println("*** Processing: " + fileName); + //System.out.println("*** Processing: " + fileName); String query = "select mediafileid, mediafilehouseid, relativepath from vw_items_rd_lh where filename = ?"; IStatementDecorator decorator = st -> { st.setString(1, fileName); @@ -376,7 +376,7 @@ public class Support { String mediaFileHouseId = rs.getString("mediafilehouseid"); if (masterMediaFile[0] == null) { masterMediaFile[0] = mediaFile; - System.out.println("Skipping: " + mediaFileHouseId); + //System.out.println("Skipping: " + mediaFileHouseId); return true; } @@ -385,7 +385,7 @@ public class Support { File file = Paths.get(LOWRES_ROOT, path).toFile(); duplicateLength[0] += file.length(); - System.out.println("Inspecting: " + mediaFileHouseId + " now available " + duplicateLength[0]); + //System.out.println("Inspecting: " + mediaFileHouseId + " now available " + duplicateLength[0]); return true; }; manager.executeQuery(query, consumer, decorator); @@ -393,14 +393,24 @@ public class Support { @Test public void processLowresDuplicates() { - String query = "select filename from vw_items_rd_dup order by filecount"; - IResultSetConsumer consumer = rs -> { + long count[] = { 0, 0 }; + manager.executeQuery("select count(filecount) from vw_items_rd_dup", rs -> { + count[0] = rs.getLong(1); + return false; + }, null); + + int progress[] = { 0, 0 }; + manager.executeQuery("select filename from vw_items_rd_dup order by filecount desc", rs -> { String fileName = rs.getString("filename"); processLowresDuplicateGroup(fileName); + count[1]++; + progress[1] = (int) ((double) count[1] * 100 / count[0]); + if (progress[1] > progress[0]) { + progress[0] = progress[1]; + } + System.out.println("Completed " + progress[0] + " " + count[1] + "/" + count[0] + " available " + duplicateLength[0]); return true; - }; - manager.executeQuery(query, consumer, null); - System.out.println("Available will " + duplicateLength[0]); + }, null); } @Test