From: Vásáry Dániel Date: Thu, 25 Jan 2018 13:48:45 +0000 (+0000) Subject: git-tfs-id: [http://tfs.userrendszerhaz.hu:8080/tfs/DefaultCollection]$/MediaCube... X-Git-Url: http://git.useribm.hu/?a=commitdiff_plain;h=6e8ab3e867b27e1f0c67dca0ac7757bf257f456c;p=mediacube.git git-tfs-id: [tfs.userrendszerhaz.hu:8080/tfs/DefaultCollection]$/MediaCube;C30850 --- diff --git a/client/Maestro/Configuration/-configuration-transcoder.json b/client/Maestro/Configuration/-configuration-transcoder.json new file mode 100644 index 00000000..12922727 --- /dev/null +++ b/client/Maestro/Configuration/-configuration-transcoder.json @@ -0,0 +1,118 @@ +{ + "title": "TRANSCODER", + "active": true, + "startInTray": true, + "enableCustomMetadataId": true, + "defaultWindowColor": "#E3F2FD", + "partialWindowColor": "#BBDEFB", + "player": { + "enabled": true, + "autoStart": false, + "segmentEditor": true + }, + "source": { + "$type": "UNCSource", + "filter": "avi,wav,mxf", + "local": { + "address": "file://10.10.1.100/braavos/TRANSCODER" + } + }, + "metadatas": [ + { + "$type": "OctopusMetadata", + "server": { + "address": "http://10.10.1.27/services/rest/octopus/", + "timeout": 1000 + } + }, + { + "$type": "TrafficMetadata", + "server": { + "address": "Data Source=10.10.1.45\\sql16;Initial Catalog=PA_Echo;Persist Security Info=True;", + "userName": "MAM", + "password": "7RKZYBzumKjL40SJwuwiFCvX57xuCN8zay6OttUm2wbrgImyYZBHyZTUUYrXX31Ge2Uwew07HYsqh2uzdJeDBDwcVntxaHg3nIpv9Dyq/odVoiC4tUF/K+lgvKWANcrZ", + "timeout": 1000 + } + }, + { + "$type": "MediaCubeMetadata", + "server": { + "address": "http://10.10.1.27/services/rest/jobengine/", + "timeout": 1000 + } + } + ], + "targets": [ + { + "label": "Octopus mappába küldés", + "processor": "FTPTargetProcessor", + "outputFormat": "%ID%-%SOURCENAME%", + "tag": "Betöltés", + "subFolderFormat": "%IDROOT%-%TEXT%", + "killDateDays": 7, + "saveArchiveMetadata": false, + "remote": { + "address": "ftp://10.10.1.100/OCTOPUS", + "userName": "mediacube", + "password": "Dn8t4gfHcK98o8hyPgLDhr5SgSji4JCxsfpMJsODikUp3nXgrM0UNCi45lLAK8ZOnmEneO44P9qpJ4QDqhctN6MxZodjJgdZTyoZKmSa+ECzEzLr/wPYNgxVaXrVotEy", + "timeout": 1000 + } + }, + { + "label": "Traffic mappába küldés", + "processor": "FTPTargetProcessor", + "outputFormat": "%ID%-%SOURCENAME%", + "tag": "Betöltés", + "subFolderFormat": "%IDROOT%-%TEXT%", + "killDateDays": 7, + "saveArchiveMetadata": false, + "remote": { + "address": "ftp://10.10.1.100/PLAYOUT_NLE", + "userName": "mediacube", + "password": "Dn8t4gfHcK98o8hyPgLDhr5SgSji4JCxsfpMJsODikUp3nXgrM0UNCi45lLAK8ZOnmEneO44P9qpJ4QDqhctN6MxZodjJgdZTyoZKmSa+ECzEzLr/wPYNgxVaXrVotEy", + "timeout": 1000 + } + }, + { + "label": "Reklám mappába küldés", + "processor": "FTPTargetProcessor", + "outputFormat": "%ID%-%SOURCENAME%", + "tag": "Betöltés", + "subFolderFormat": "%IDROOT%-%TEXT%/INGEST-%TIMESTAMP%", + "killDateDays": 14, + "saveArchiveMetadata": false, + "remote": { + "address": "ftp://10.10.1.100/REKLAM_NLE", + "userName": "mediacube", + "password": "Dn8t4gfHcK98o8hyPgLDhr5SgSji4JCxsfpMJsODikUp3nXgrM0UNCi45lLAK8ZOnmEneO44P9qpJ4QDqhctN6MxZodjJgdZTyoZKmSa+ECzEzLr/wPYNgxVaXrVotEy", + "timeout": 1000 + } + }, + { + "label": "Lebonyba küldés", + "processor": "FTPTargetProcessor", + "outputFormat": "%ID%", + "tag": "Lebony", + "killDateDays": 7, + "remote": { + "address": "ftp://10.10.1.100/TQC/CHECK", + "userName": "mediacube", + "password": "Dn8t4gfHcK98o8hyPgLDhr5SgSji4JCxsfpMJsODikUp3nXgrM0UNCi45lLAK8ZOnmEneO44P9qpJ4QDqhctN6MxZodjJgdZTyoZKmSa+ECzEzLr/wPYNgxVaXrVotEy", + "timeout": 1000 + } + }, + { + "label": "Archiválás", + "processor": "FTPTargetProcessor", + "outputFormat": "%ID%", + "tag": "Archiválás", + "saveArchiveMetadata": true, + "remote": { + "address": "ftp://10.10.1.100/ARCHIVE", + "userName": "mediacube", + "password": "Dn8t4gfHcK98o8hyPgLDhr5SgSji4JCxsfpMJsODikUp3nXgrM0UNCi45lLAK8ZOnmEneO44P9qpJ4QDqhctN6MxZodjJgdZTyoZKmSa+ECzEzLr/wPYNgxVaXrVotEy", + "timeout": 1000 + } + } + ] +} diff --git a/client/Maestro/Configuration/configuration-tqc-check.json b/client/Maestro/Configuration/configuration-tqc-check.json deleted file mode 100644 index 6989f703..00000000 --- a/client/Maestro/Configuration/configuration-tqc-check.json +++ /dev/null @@ -1,98 +0,0 @@ -{ - "title": "TQC check", - "active": true, - "startInTray": false, - "enableCustomMetadataId": true, - "player": { - "enabled": true, - "autoStart": false, - "segmentEditor": true - }, - "source": { - "$type": "UNCSource", - "filter": "avi,wav,mxf", - "local": { - "address": "file://10.10.1.100/BRAAVOS/TQC/CHECK", - "userName": "mediacube", - "password": "Dn8t4gfHcK98o8hyPgLDhr5SgSji4JCxsfpMJsODikUp3nXgrM0UNCi45lLAK8ZOnmEneO44P9qpJ4QDqhctN6MxZodjJgdZTyoZKmSa+ECzEzLr/wPYNgxVaXrVotEy", - "timeout": 1000 - }, - "remote": { - "address": "ftp://10.10.1.100/TQC/CHECK", - "userName": "mediacube", - "password": "Dn8t4gfHcK98o8hyPgLDhr5SgSji4JCxsfpMJsODikUp3nXgrM0UNCi45lLAK8ZOnmEneO44P9qpJ4QDqhctN6MxZodjJgdZTyoZKmSa+ECzEzLr/wPYNgxVaXrVotEy", - "timeout": 1000 - } - }, - "metadatas": [ - { - "$type": "TrafficMetadata", - "server": { - "address": "Data Source=10.10.1.45\\sql16;Initial Catalog=PA_Echo;Persist Security Info=True;", - "userName": "MAM", - "password": "7RKZYBzumKjL40SJwuwiFCvX57xuCN8zay6OttUm2wbrgImyYZBHyZTUUYrXX31Ge2Uwew07HYsqh2uzdJeDBDwcVntxaHg3nIpv9Dyq/odVoiC4tUF/K+lgvKWANcrZ", - "timeout": 1000 - } - }, - { - "$type": "MediaCubeMetadata", - "server": { - "address": "http://10.10.1.27/services/rest/jobengine/", - "timeout": 1000 - } - } - ], - "targets": [ - { - "label": "Adáskész", - "processor": "FXPTargetProcessor", - "outputFormat": "%ID%", - "saveSegments": true, - "tag": "Adáskész", - "disableFileVersioning": true, - "enableOverride": true, - "saveMorpheusMetadata": true, - "pathMorpheusMetadata": "/PLAYOUT/MorpheusGateway/Input", - "deviceIdMorpheus": "ISILON", - "remote": { - "address": "ftp://10.10.1.100/PLAYOUT/Video", - "userName": "mediacube", - "password": "Dn8t4gfHcK98o8hyPgLDhr5SgSji4JCxsfpMJsODikUp3nXgrM0UNCi45lLAK8ZOnmEneO44P9qpJ4QDqhctN6MxZodjJgdZTyoZKmSa+ECzEzLr/wPYNgxVaXrVotEy", - "timeout": 1000 - } - }, - { - "label": "Újravágás", - "processor": "FXPTargetProcessor", - "subFolderFormat": "%IDROOT%-%TEXT%", - "outputFormat": "%ID%-%TEXT%", - "tag": "Újravágás", - "disableFileVersioning": true, - "enableOverride": true, - "remote": { - "address": "ftp://10.10.1.100/PLAYOUT_NLE", - "userName": "mediacube", - "password": "Dn8t4gfHcK98o8hyPgLDhr5SgSji4JCxsfpMJsODikUp3nXgrM0UNCi45lLAK8ZOnmEneO44P9qpJ4QDqhctN6MxZodjJgdZTyoZKmSa+ECzEzLr/wPYNgxVaXrVotEy", - "timeout": 1000 - }, - "sendEmailOnSuccess": true, - "successEmailRecipient": "vasary@elgekko.net", - "successEmailPattern": "A %TARGETNAME% újravágása szükséges." - }, - { - "label": "Archiválás", - "processor": "FXPTargetProcessor", - "outputFormat": "%ID%", - "tag": "Archiválás", - "saveArchiveMetadata": true, - "disableFileVersioning": true, - "enableOverride": true, - "remote": { - "address": "ftp://10.10.1.100/ARCHIVE", - "userName": "mediacube", - "password": "Dn8t4gfHcK98o8hyPgLDhr5SgSji4JCxsfpMJsODikUp3nXgrM0UNCi45lLAK8ZOnmEneO44P9qpJ4QDqhctN6MxZodjJgdZTyoZKmSa+ECzEzLr/wPYNgxVaXrVotEy", - "timeout": 1000 - } - } - ] -} diff --git a/client/Maestro/Maestro.csproj b/client/Maestro/Maestro.csproj index 69d690b4..edf46e53 100644 --- a/client/Maestro/Maestro.csproj +++ b/client/Maestro/Maestro.csproj @@ -270,7 +270,7 @@ Always - + Always diff --git a/client/Maestro/Program.cs b/client/Maestro/Program.cs index ff4bb750..b0e97ce9 100644 --- a/client/Maestro/Program.cs +++ b/client/Maestro/Program.cs @@ -6,7 +6,6 @@ using NLog; using System; using System.Collections.Concurrent; using System.Collections.Generic; -using System.Diagnostics; using System.DirectoryServices.AccountManagement; using System.IO; using System.Linq; @@ -14,7 +13,6 @@ using System.Reflection; using System.Runtime.InteropServices; using System.Security.AccessControl; using System.Security.Principal; -using System.Text.RegularExpressions; using System.Threading; using System.Threading.Tasks; using System.Windows.Forms; @@ -22,6 +20,7 @@ using System.Windows.Forms; namespace Maestro { static class Program { + private static Logger logger = LogManager.GetCurrentClassLogger(); /// /// The main entry point for the application. @@ -61,7 +60,7 @@ namespace Maestro { Application.Run(context); } catch (Exception e) { - Debug.WriteLine(e.Message); + logger.Error(e); MessageBox.Show(new Form() { TopMost = true }, e.Message); } finally { @@ -77,18 +76,19 @@ namespace Maestro { } public class TrayApplicationContext : ApplicationContext { + private static Logger logger = LogManager.GetCurrentClassLogger(); private const string GLOBAL_JSON = "global.json"; private const string AUDIORECORDER_JSON = "audiorecorder.json"; private const string JSON_WILDCARD = "*.json"; private const string CONFIG_ERROR = "Nem található aktív konfiguráció."; private const string SYSCONFIG_ERROR = "Nem található a 'global.json' rendszer konfiguráció."; + private const string SYSCONFIG_DOMAIN_ERROR = "Nem található a 'global.json' rendszer konfigurációban a tartományhitelesítő beállítása."; private const string CONFIGURATION_FOLDER = "./Configuration"; private const string DOMAIN_QUERY_ERROR = "A bejelentkezett felhasználó domain információja nem elérhető."; private const string NOSPLASH = "nosplash"; private const string NOLOGIN = "nologin"; public const string NOSINGLE = "nosingle"; private const string HYPHEN = "-"; - private static Logger logger = LogManager.GetCurrentClassLogger(); private NotifyIcon trayIcon; private ConcurrentBag forms = new ConcurrentBag(); private IMessageBus messageBus = new MessageBus(); @@ -154,6 +154,9 @@ namespace Maestro { if (userInfo.Length < 2) throw new Exception(DOMAIN_QUERY_ERROR); string userDomainName = userInfo[0]; + + logger.Info("User is authenticated by domain: {0}", userDomainName); + if (userDomainName.Contains(".")) userDomainName = userDomainName.Split('.')[0]; if (domainName.ToLower().Equals(userDomainName.ToLower())) @@ -163,23 +166,28 @@ namespace Maestro { private void CheckUser() { string globalFile = Path.Combine(CONFIGURATION_FOLDER, GLOBAL_JSON); - string domainName = GlobalConfig.UserDomain.Address.ToString(); + string domainName = GlobalConfig?.UserDomain?.Address?.ToString(); if (String.IsNullOrEmpty(domainName)) - throw new Exception(SYSCONFIG_ERROR); + throw new Exception(SYSCONFIG_DOMAIN_ERROR); else { UserName = WindowsIdentity.GetCurrent().Name; - if (IsDomainUser(UserName, domainName)) + logger.Debug("Current user is: {0}", UserName); + if (IsDomainUser(UserName, domainName)) { + logger.Debug("User automatically accepted"); return; - UserName = GlobalConfig.UserDomain.UserName; + } + UserName = GlobalConfig.UserDomain?.UserName; if (HasCommandlineParameter(NOLOGIN)) return; - string password = GlobalConfig.UserDomain.Password; + string password = GlobalConfig.UserDomain?.Password; LoginForm loginForm = null; try { using (PrincipalContext pc = new PrincipalContext(ContextType.Domain, domainName)) { while (true) { + logger.Debug("Trying authenticate user {0}", UserName); bool isValid = pc.ValidateCredentials(UserName, password); if (isValid) { + logger.Debug("Authenticated: {0}", UserName); break; } diff --git a/server/-configuration/scheduledjobs.json b/server/-configuration/scheduledjobs.json index 403db8eb..bda37e10 100644 --- a/server/-configuration/scheduledjobs.json +++ b/server/-configuration/scheduledjobs.json @@ -58,13 +58,28 @@ {"name": "archiveFtp", "value": "ftp://10.10.1.100/ARCHIVE/TEST", "type": "java.lang.String"}, {"name": "archiveUserName", "value": "mediacube", "type": "java.lang.String"}, {"name": "archivePassword", "value": "Broadca5T", "type": "java.lang.String"}, - {"name": "agencies", "value": "schedule-rec", "type": "java.lang.String"}, + {"name": "filterAgencies", "value": "schedule-rec", "type": "java.lang.String"}, {"name": "limit", "value": 10, "type": "java.lang.Integer"}, {"name": "nexioKillDateDays", "value": 7, "type": "java.lang.Integer"}, {"name": "nexioAgency", "value": "ARCHIVED", "type": "java.lang.String"} ] }, { + "active": true, + "executeimmediate": false, + "name" : "Lejárt NEXIO anyagok törlése", + "template": "delete-nexio-materials.xml", + "cronexpression": "0 0 10 * * ?", + "parameters": [ + {"name": "port", "value": 2098, "type": "java.lang.Integer"}, + {"name": "userName", "value": "administrator", "type": "java.lang.String"}, + {"name": "password", "value": "system", "type": "java.lang.String"}, + {"name": "filterAgencies", "value": "ARCHIVED", "type": "java.lang.String"}, + {"name": "gracePeriodDays", "value": 1, "type": "java.lang.Integer"}, + {"name": "notificationOnly", "value": true, "type": "java.lang.Boolean"} + ] + }, + { "active": false, "name" : "TRAFFIC anyagok visszatöltése", "template": "retrieve-traffic-missing-materials.xml", @@ -102,19 +117,6 @@ ] }, { - "active": false, - "name" : "NEXIO anyagok törlése", - "template": "delete-nexio-materials.xml", - "executeimmediate": true, - "cronexpression": "0 0 0/1 1/1 * ? *", - "parameters": [ - {"name": "port", "value": 2098, "type": "java.lang.Integer"}, - {"name": "userName", "value": "administrator", "type": "java.lang.String"}, - {"name": "password", "value": "system", "type": "java.lang.String"}, - {"name": "gracePeriodDays", "value": 12, "type": "java.lang.Integer"} - ] - }, - { "active": false, "name" : "OCTOPUS adatok szinkronizálása", "template": "sync-octopus.xml", diff --git a/server/user.commons.log4j2/src/user/commons/log4j2/appender/SmtpManager.java b/server/user.commons.log4j2/src/user/commons/log4j2/appender/SmtpManager.java index aa4a792d..9e402f8e 100644 --- a/server/user.commons.log4j2/src/user/commons/log4j2/appender/SmtpManager.java +++ b/server/user.commons.log4j2/src/user/commons/log4j2/appender/SmtpManager.java @@ -355,7 +355,7 @@ public class SmtpManager extends AbstractManager { if (events == null) return; String to = mcm.getTo() == null ? defaultRecipient : mcm.getTo(); - String subject = mcm.getSessionName() == null ? defaultSubject : String.format("%s: %s", level, mcm.getSessionName()); + String subject = mcm.getSessionName() == null ? defaultSubject : String.format("%s: %s #%s", level, mcm.getSessionName(), mcm.getSessionID()); sendEvents(to, subject, layout, events); sessionEvents.remove(sessionID); } diff --git a/server/user.jobengine.executors/jobtemplates/copyforarchive-nexio-recordings.xml b/server/user.jobengine.executors/jobtemplates/copyforarchive-nexio-recordings.xml index b9feae22..5ad631a4 100644 --- a/server/user.jobengine.executors/jobtemplates/copyforarchive-nexio-recordings.xml +++ b/server/user.jobengine.executors/jobtemplates/copyforarchive-nexio-recordings.xml @@ -9,10 +9,10 @@ - + - + @@ -37,7 +37,7 @@ - + @@ -46,7 +46,7 @@ - + diff --git a/server/user.jobengine.executors/jobtemplates/delete-nexio-materials.xml b/server/user.jobengine.executors/jobtemplates/delete-nexio-materials.xml index d3354868..925290af 100644 --- a/server/user.jobengine.executors/jobtemplates/delete-nexio-materials.xml +++ b/server/user.jobengine.executors/jobtemplates/delete-nexio-materials.xml @@ -6,7 +6,9 @@ + + @@ -21,9 +23,15 @@ + + + + + + diff --git a/server/user.jobengine.executors/src/user/jobengine/server/steps/ArchiveMaterialSubmitStep.java b/server/user.jobengine.executors/src/user/jobengine/server/steps/ArchiveMaterialSubmitStep.java index 7f9cb2c4..31e22db3 100644 --- a/server/user.jobengine.executors/src/user/jobengine/server/steps/ArchiveMaterialSubmitStep.java +++ b/server/user.jobengine.executors/src/user/jobengine/server/steps/ArchiveMaterialSubmitStep.java @@ -30,8 +30,9 @@ public class ArchiveMaterialSubmitStep extends JobStep { try { for (ArchiveItem archiveItem : archiveList) { try { - IJobRuntime submit = jobEngine.submit(JOBTEMPLATE, ARCHIVE, ListUtils.asMap(ARCHIVE_ITEM, archiveItem, GLOBAL_SOURCE_PATH, globalSourcePath, - TRANSCODER_TARGET_PATH, transcoderTargetPath, WEB_PATH, webPath, KILL_DATE_DAYS, killDateDays)); + IJobRuntime runtime = jobEngine.submit(JOBTEMPLATE, ARCHIVE, ListUtils.asMap(ARCHIVE_ITEM, archiveItem, GLOBAL_SOURCE_PATH, + globalSourcePath, TRANSCODER_TARGET_PATH, transcoderTargetPath, WEB_PATH, webPath, KILL_DATE_DAYS, killDateDays)); + } catch (Exception e) { if (archiveItem != null) archiveItem.removeCatchedFile(); diff --git a/server/user.jobengine.executors/src/user/jobengine/server/steps/CopyForArchiveNEXIOMaterialsStep.java b/server/user.jobengine.executors/src/user/jobengine/server/steps/CopyForArchiveNEXIOMaterialsStep.java index ad38685f..5f7eedbb 100644 --- a/server/user.jobengine.executors/src/user/jobengine/server/steps/CopyForArchiveNEXIOMaterialsStep.java +++ b/server/user.jobengine.executors/src/user/jobengine/server/steps/CopyForArchiveNEXIOMaterialsStep.java @@ -45,6 +45,7 @@ public class CopyForArchiveNEXIOMaterialsStep extends JobStep { private static final String LONGNAMEID = "longnameid"; private static final String ARCHIVEDRUNDOWNS = "archivedrundowns"; private static final String ID = "id"; + private static final String MEDIATYPE = "Octopus bejátszó"; private OctopusAPI octopusAPI; private IItemManager manager; @@ -96,16 +97,6 @@ public class CopyForArchiveNEXIOMaterialsStep extends JobStep { } private BasicDBObject createMetadata(RundownArchive rundownArchive, StoryArchive storyArchive, FileArchive fileArchive) { - // { - // "itemHouseId": "43", - // "itemTitle": "Hazahúzó", - // "itemDescription": null, - // "mediaHouseId": "002570", - // "mediaTitle": "2017.12.13", - // "mediaDescription": null, - // "ok": false, - // "userName": "echotest" - // } BasicDBObject result = new BasicDBObject(); result.put("itemHouseId", rundownArchive.getItemHouseId()); result.put("itemTitle", rundownArchive.getItemTitle()); @@ -115,6 +106,7 @@ public class CopyForArchiveNEXIOMaterialsStep extends JobStep { result.put("mediaHouseId", storyArchive.getMediaHouseId()); result.put("mediaTitle", storyArchive.getMediaTitle()); result.put("mediaDescription", storyArchive.getMediaDesc()); + result.put("mediaType", MEDIATYPE); result.put("duration", fileArchive.getDuration()); return result; diff --git a/server/user.jobengine.executors/src/user/jobengine/server/steps/CopyForArchiveNEXIORecordingsStep.java b/server/user.jobengine.executors/src/user/jobengine/server/steps/CopyForArchiveNEXIORecordingsStep.java index 296c84c9..1e58316f 100644 --- a/server/user.jobengine.executors/src/user/jobengine/server/steps/CopyForArchiveNEXIORecordingsStep.java +++ b/server/user.jobengine.executors/src/user/jobengine/server/steps/CopyForArchiveNEXIORecordingsStep.java @@ -38,7 +38,7 @@ import user.jobengine.server.IJobEngine; import user.jobengine.server.IJobRuntime; public class CopyForArchiveNEXIORecordingsStep extends JobStep { - private static final String MEDIATYPE_RECORDING = "Visszarögzített"; + private static final String MEDIATYPE = "Visszarögzített"; private static final String SCHEDULED_FORMAT = "yyyy.MM.dd HH:mm"; private static final String STARTTIME_FORMAT = "HHmm"; private static final Logger logger = LogManager.getLogger(); @@ -118,7 +118,7 @@ public class CopyForArchiveNEXIORecordingsStep extends JobStep { result.put("mediaDescription", storyArchive.getMediaDesc()); result.put("duration", fileArchive.getDuration()); - result.put("mediaType", MEDIATYPE_RECORDING); + result.put("mediaType", MEDIATYPE); return result; } @@ -148,10 +148,10 @@ public class CopyForArchiveNEXIORecordingsStep extends JobStep { @StepEntry public Object[] execute(int nexioPort, String nexioUserName, String nexioPassword, String archiveFtp, String archiveUserName, String archivePassword, - String agencies, int limit, int nexioKillDateDays, String nexioAgency, IJobEngine jobEngine, IJobRuntime jobRuntime) throws Exception { + String filterAgencies, int limit, int nexioKillDateDays, String targetAgency, IJobEngine jobEngine, IJobRuntime jobRuntime) throws Exception { systemMarker = jobRuntime.getMarker(); - setAndCheck(nexioPort, nexioUserName, nexioPassword, archiveFtp, archiveUserName, archivePassword, agencies, limit, nexioKillDateDays, nexioAgency, - jobEngine); + setAndCheck(nexioPort, nexioUserName, nexioPassword, archiveFtp, archiveUserName, archivePassword, filterAgencies, limit, nexioKillDateDays, + targetAgency, jobEngine); octopusAPI = new OctopusAPI(); List clips = queryClips(); processClips(clips, limit); diff --git a/server/user.jobengine.executors/src/user/jobengine/server/steps/DeleteNEXIOMaterialsStep.java b/server/user.jobengine.executors/src/user/jobengine/server/steps/DeleteNEXIOMaterialsStep.java index ac3add9c..2318509c 100644 --- a/server/user.jobengine.executors/src/user/jobengine/server/steps/DeleteNEXIOMaterialsStep.java +++ b/server/user.jobengine.executors/src/user/jobengine/server/steps/DeleteNEXIOMaterialsStep.java @@ -1,5 +1,6 @@ package user.jobengine.server.steps; +import java.text.SimpleDateFormat; import java.util.Calendar; import java.util.Date; import java.util.List; @@ -8,13 +9,13 @@ import org.apache.commons.lang.StringUtils; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; +import com.ibm.nosql.json.api.BasicDBList; import com.ibm.nosql.json.api.BasicDBObject; import com.ibm.nosql.json.api.DB; import com.ibm.nosql.json.api.DBCollection; import com.ibm.nosql.json.api.DBCursor; import com.ibm.nosql.json.api.QueryBuilder; -import user.commons.CalendarUtils; import user.commons.ListUtils; import user.commons.RemoteFile; import user.commons.StoreUri; @@ -23,6 +24,7 @@ import user.commons.remotestore.RemoteStoreProtocol; import user.jobengine.db.IItemManager; import user.jobengine.server.IJobEngine; import user.jobengine.server.IJobRuntime; +import user.jobengine.server.JobRuntime; public class DeleteNEXIOMaterialsStep extends JobStep { private static final Logger logger = LogManager.getLogger(); @@ -31,42 +33,51 @@ public class DeleteNEXIOMaterialsStep extends JobStep { private static final String LONGNAMEID = "longnameid"; private static final String EXTAGENCY = "extagency"; private StoreUri sourceUri; + private JobRuntime jobRuntime; private void delete(String name) { RemoteFile remoteFile = null; try { remoteFile = sourceUri.getRemoteFile(name); - if (remoteFile == null) - throw new NullPointerException("File not exists: " + name); + if (remoteFile == null) { + logger.warn(jobRuntime.getMarker(), "A '{}' fájl már nem található a NEXIO szerveren", name); + return; + } sourceUri.delete(remoteFile); - logger.info(getMarker(), "A {} fájl törlése sikeres volt.", remoteFile.getName()); + logger.info(jobRuntime.getMarker(), "A '{}' fájl törlése sikeres volt.", remoteFile.getName()); } catch (Exception e) { - logger.error(getMarker(), "A {} fájl nem törölhető. A rendszer hibaüzenete: {}", remoteFile.getName(), e.getMessage()); + logger.error(jobRuntime.getMarker(), "A '{}' fájl nem törölhető. A rendszer hibaüzenete: {}", remoteFile.getName(), e.getMessage()); } } @StepEntry - public Object[] execute(int port, String userName, String password, int gracePeriodDays, IJobEngine jobEngine, IJobRuntime jobRuntime) throws Exception { + public Object[] execute(int port, String userName, String password, String filterAgencies, int gracePeriodDays, boolean notificationOnly, + IJobEngine jobEngine, IJobRuntime jobRuntime) throws Exception { String nexioHost = System.getProperty("nexio.host"); if (StringUtils.isBlank(nexioHost)) { - logger.error(getMarker(), "A 'nexio.host' rendszer paraméter nem található."); - throw new NullPointerException("System is not configured properly, 'jobengine.selenio.address' startup parameter missing."); + logger.error(jobRuntime.getMarker(), "A 'nexio.host' rendszer paraméter nem található."); + throw new NullPointerException("System is not configured properly, 'nexio.host' startup parameter missing."); } - DB db = NoSQLUtils.getNoSQLDB(); + if (StringUtils.isBlank(filterAgencies)) { + logger.error(jobRuntime.getMarker(), "A 'nexioAgencies' folyamat paraméter nem található."); + throw new NullPointerException("System is not configured properly, 'nexioAgencies' job parameter missing."); + } + DB db = NoSQLUtils.getNoSQLDB(); DBCollection collection = db.getCollection(NEXIOCLIPS); //https://docs.oracle.com/javase/7/docs/api/java/util/regex/Pattern.html //http://www.vogella.com/tutorials/JavaRegularExpressions/article.html // if (StringUtils.isNotBlank(fileNameRegex)) // queryBuilder.and(QueryBuilder.start(LONGNAMEID).regex(Pattern.compile(fileNameRegex)).get()); - // if (StringUtils.isNotBlank(agency)) - // queryBuilder.and(QueryBuilder.start(EXTAGENCY).all(agency).get()); Calendar now = Calendar.getInstance(); now.add(Calendar.DAY_OF_YEAR, gracePeriodDays * -1); + BasicDBList agencies = new BasicDBList((Object[]) filterAgencies.split(",")); + + QueryBuilder dateQueryBuilder = QueryBuilder.start(KILLDATE).lessThan(now.getTime()); + QueryBuilder queryBuilder = QueryBuilder.start(EXTAGENCY).in(agencies).and(dateQueryBuilder.get()); + DBCursor cursor = collection.find(queryBuilder.get()).sort(KILLDATE, -1); - QueryBuilder queryBuilder = QueryBuilder.start(KILLDATE).lessThan(now.getTime()); - DBCursor cursor = collection.find(queryBuilder.get()); if (!cursor.hasNext()) return null; @@ -77,11 +88,18 @@ public class DeleteNEXIOMaterialsStep extends JobStep { sourceUri.setPassword(password); List clips = ListUtils.cast(cursor.toArray()); + SimpleDateFormat df = new SimpleDateFormat("yyyy.MM.dd"); + + int i = 0; for (BasicDBObject clip : clips) { String name = String.valueOf(clip.get(LONGNAMEID)); Date killdate = clip.getDate(KILLDATE); - logger.info("{}, {}, {}", name, CalendarUtils.toDateString(CalendarUtils.createCalendar(killdate)), clip.getString(EXTAGENCY)); - delete(name); + if (notificationOnly) + logger.info(jobRuntime.getMarker(), "Az {} fájl törölhető. Lejárt: {} ", name, df.format(killdate)); + // else + // delete(name); + i++; + setProgress(i * 100 / clips.size()); } return null; diff --git a/server/user.jobengine.executors/src/user/jobengine/server/steps/FakeNoParamsStep.java b/server/user.jobengine.executors/src/user/jobengine/server/steps/FakeNoParamsStep.java index 5fb9225c..20f6f980 100644 --- a/server/user.jobengine.executors/src/user/jobengine/server/steps/FakeNoParamsStep.java +++ b/server/user.jobengine.executors/src/user/jobengine/server/steps/FakeNoParamsStep.java @@ -1,31 +1,47 @@ package user.jobengine.server.steps; +import java.util.concurrent.CountDownLatch; + import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; +import user.commons.JobStatus; +import user.commons.ListUtils; import user.jobengine.server.IJobEngine; import user.jobengine.server.IJobRuntime; +import user.jobengine.server.JobStatusChangedEvent; public class FakeNoParamsStep extends JobStep { + private static final String CHILD_TITLE = "Párhuzamosított alfolyamat"; + private static final String CHILD_TEMPLATE = "fake-concurrent.xml"; private static final Logger logger = LogManager.getLogger(); int count = 10; + CountDownLatch latch = new CountDownLatch(count); + + private void childStatusChanged(JobStatusChangedEvent event) { + if (JobStatus.SUSPENDED.equals(event.getStatus()) || JobStatus.FINISHED.equals(event.getStatus())) + latch.countDown(); + } @StepEntry public Object[] execute(IJobEngine jobEngine, IJobRuntime jobRuntime) throws Exception { - logger.warn(jobRuntime.getMarker(), "Egy WARN {}", "paraméterrel"); - logger.error(jobRuntime.getMarker(), "Egy ERROR {}", "paraméterrel"); + // logger.warn(jobRuntime.getMarker(), "Egy WARN {}", "paraméterrel"); + // logger.error(jobRuntime.getMarker(), "Egy ERROR {}", "paraméterrel"); + // for (int i = 0; i < count; i++) { + // if (!canContinue()) + // break; + // jobRuntime.incrementProgress((i + 1) * count); + // + // Thread.sleep(100); + // logger.info(jobRuntime.getMarker(), "Log markered" + i); + // + // if (i == 8) + // throw new Exception("stop"); + // } for (int i = 0; i < count; i++) { - if (!canContinue()) - break; - jobRuntime.incrementProgress((i + 1) * count); - - Thread.sleep(100); - logger.info(jobRuntime.getMarker(), "Log markered" + i); - - if (i == 8) - throw new Exception("stop"); + jobEngine.submit(jobRuntime, e -> childStatusChanged(e), CHILD_TEMPLATE, CHILD_TITLE, ListUtils.asMap("itemID", i)); } + latch.await(); return null; } - } diff --git a/server/user.jobengine.executors/src/user/jobengine/server/steps/FakeStep.java b/server/user.jobengine.executors/src/user/jobengine/server/steps/FakeStep.java index b259f0d6..60853110 100644 --- a/server/user.jobengine.executors/src/user/jobengine/server/steps/FakeStep.java +++ b/server/user.jobengine.executors/src/user/jobengine/server/steps/FakeStep.java @@ -13,20 +13,22 @@ public class FakeStep extends JobStep { @StepEntry public Object[] execute(long itemID, IJobEngine jobEngine, IJobRuntime jobRuntime) throws Exception { - Object[] result = { 1 }; + Object[] result = { itemID + 1 }; + + logger.info(jobRuntime.getMarker(), "Starting with {}", itemID); try { for (int i = 0; i < count; i++) { //sb.append("Sikeres iteráció : ") if (!canContinue()) break; jobRuntime.incrementProgress((i + 1) * count); - Thread.sleep(500); + Thread.sleep(100); //logger.info("Progress {}", jobRuntime.getProgress()); } } catch (Exception e) { logger.error(e.getMessage()); } - //logger.info(getMarker(), "Log markered"); + logger.info(jobRuntime.getMarker(), "Resulting {}", result[0]); return result; } diff --git a/server/user.jobengine.executors/src/user/jobengine/server/steps/MetadataTransformStep.java b/server/user.jobengine.executors/src/user/jobengine/server/steps/MetadataTransformStep.java index 6a7bf76f..28e7c192 100644 --- a/server/user.jobengine.executors/src/user/jobengine/server/steps/MetadataTransformStep.java +++ b/server/user.jobengine.executors/src/user/jobengine/server/steps/MetadataTransformStep.java @@ -21,7 +21,7 @@ import user.jobengine.server.IJobRuntime; public class MetadataTransformStep extends JobStep { private static final Logger logger = LogManager.getLogger(); private static final String ITEM_MANAGER_IS_NULL = "ItemManager is null"; - public static final String ITEMTYPENAME = "Generic"; + public static final String DEFAULT_MEDIATYPE = "Generic"; @StepEntry public Object[] execute(ArchiveItem archiveItem, IJobEngine jobEngine, IJobRuntime jobRuntime) { @@ -34,18 +34,18 @@ public class MetadataTransformStep extends JobStep { jobRuntime.incrementProgress(50); if (mediaCubeItem == null) - mediaCubeItem = itemManager.createItem(ITEMTYPENAME, archiveItem.getItemTitle(), archiveItem.getItemDescription(), + mediaCubeItem = itemManager.createItem(DEFAULT_MEDIATYPE, archiveItem.getItemTitle(), archiveItem.getItemDescription(), archiveItem.getItemHouseId()); - String mediaTypeName = archiveItem.getMediaType(); - if (StringUtils.isBlank(mediaTypeName)) { - mediaCubeMedia = itemManager.createMedia(ITEMTYPENAME, archiveItem.getMediaTitle(), archiveItem.getMediaDescription(), + String mediaType = archiveItem.getMediaType(); + if (StringUtils.isBlank(mediaType)) { + mediaCubeMedia = itemManager.createMedia(DEFAULT_MEDIATYPE, archiveItem.getMediaTitle(), archiveItem.getMediaDescription(), archiveItem.getMediaHouseId()); } else { - ItemType itemType = itemManager.getItemType(mediaTypeName); - if (itemType == null) - itemManager.createItemType(mediaTypeName, mediaTypeName).add(); - mediaCubeMedia = itemManager.createMedia(mediaTypeName, archiveItem.getMediaTitle(), archiveItem.getMediaDescription(), + ItemType mediaItemType = itemManager.getItemType(mediaType); + if (mediaItemType == null) + itemManager.createItemType(mediaType, mediaType).add(); + mediaCubeMedia = itemManager.createMedia(mediaType, archiveItem.getMediaTitle(), archiveItem.getMediaDescription(), archiveItem.getMediaHouseId()); } diff --git a/server/user.jobengine.osgi.server/src/user/jobengine/server/IJobEngine.java b/server/user.jobengine.osgi.server/src/user/jobengine/server/IJobEngine.java index b917e498..cfc5de21 100644 --- a/server/user.jobengine.osgi.server/src/user/jobengine/server/IJobEngine.java +++ b/server/user.jobengine.osgi.server/src/user/jobengine/server/IJobEngine.java @@ -106,6 +106,9 @@ public interface IJobEngine { void startup(); + IJobRuntime submit(IJobRuntime parent, IJobStatusChangedListener statusListener, String template, String name, Map parameters) + throws JobEngineException; + IJobRuntime submit(String template, String name, Map parameters) throws JobEngineException; void suspendJob(Throwable t, IJobRuntime jobRuntime); diff --git a/server/user.jobengine.osgi.server/src/user/jobengine/server/JobEngine.java b/server/user.jobengine.osgi.server/src/user/jobengine/server/JobEngine.java index aaaee406..7a0f8080 100644 --- a/server/user.jobengine.osgi.server/src/user/jobengine/server/JobEngine.java +++ b/server/user.jobengine.osgi.server/src/user/jobengine/server/JobEngine.java @@ -237,6 +237,8 @@ public class JobEngine implements IJobEngine { public void addToRunQueue(IJobRuntime jobRuntime) { try { logger.debug("{} adding to run queue", jobRuntime); + if (jobRuntime.getIp() == 0 && !jobRuntime.isService() && jobRuntime.getParentJobId() == 0) + logger.info(jobRuntime.getMarker(), "A '{}' folyamat elindult", jobRuntime.getName()); runQueue.put(jobRuntime); } catch (Exception e) { logger.error(e.getMessage(), e); @@ -679,7 +681,14 @@ public class JobEngine implements IJobEngine { IJobRuntime jobRuntime = submittedJobs.remove(id); fireJobChangedEvent(new JobChangedEvent(jobRuntime, SignalType.DELETE)); logger.info("--- {} removed from VM", jobRuntime); + if (!jobRuntime.isService() && jobRuntime.getParentJobId() == 0) { + if (JobStatus.FINISHED.equals(jobRuntime.getStatus())) + logger.info(jobRuntime.getFinishMarker(), "A '{}' folyamat futása sikeresen véget ért", jobRuntime.getName()); + else + logger.error(jobRuntime.getFinishMarker(), "A '{}' folyamat futása megszakadt", jobRuntime.getName()); + } } + } @Override @@ -813,6 +822,29 @@ public class JobEngine implements IJobEngine { logger.info("+++ {} added to VM ", runtime); } + @Override + public IJobRuntime submit(IJobRuntime parent, IJobStatusChangedListener statusListener, String template, String name, Map parameters) + throws JobEngineException { + IJobRuntime result = null; + IProgram program = getProgram(template); + if (program != null) { + isRunnable(program); + result = new JobRuntime(this, program); + result.setPersister(itemManager); + result.setTemplate(template); + result.setName(name); + result.setParameters(parameters); + result.setService(program.getTemplate().isService()); + result.setParentJobId(parent.getId()); + if (statusListener != null) + result.addEventListener(statusListener); + submit(result); + } else { + logger.error("Unknown template: " + template); + } + return result; + } + IJobRuntime submit(IProgram program, IJobStatusChangedListener listener) { IJobRuntime jobRuntime = new JobRuntime(this, program); jobRuntime.addEventListener(listener); diff --git a/server/user.jobengine.osgi.server/src/user/jobengine/server/JobRuntime.java b/server/user.jobengine.osgi.server/src/user/jobengine/server/JobRuntime.java index bf551c6f..1d55710b 100644 --- a/server/user.jobengine.osgi.server/src/user/jobengine/server/JobRuntime.java +++ b/server/user.jobengine.osgi.server/src/user/jobengine/server/JobRuntime.java @@ -131,7 +131,7 @@ public class JobRuntime extends Job implements IJobRuntime { children.add(c); Message m = logger.getMessageFactory().newMessage("{} {} alfolyamata", this.getId(), this.getName()); c.setName(m.getFormattedMessage()); - c.setParentJobId(this.getId()); + c.setParentJobId(getParentJobId() == 0 ? getId() : getParentJobId()); } @Override @@ -225,7 +225,7 @@ public class JobRuntime extends Job implements IJobRuntime { public Marker getMarker() { if (sessionMarker == null) { sessionMarker = new MediaCubeMarker(); - sessionMarker.setSessionID(String.valueOf(getId())); + sessionMarker.setSessionID(getParentJobId() == 0 ? String.valueOf(getId()) : String.valueOf(getParentJobId())); sessionMarker.setSessionName(name); } return sessionMarker; diff --git a/server/user.jobengine.osgi.server/src/user/jobengine/server/actions/DoneExecutingAction.java b/server/user.jobengine.osgi.server/src/user/jobengine/server/actions/DoneExecutingAction.java index 121c377c..6918b208 100644 --- a/server/user.jobengine.osgi.server/src/user/jobengine/server/actions/DoneExecutingAction.java +++ b/server/user.jobengine.osgi.server/src/user/jobengine/server/actions/DoneExecutingAction.java @@ -15,7 +15,5 @@ public class DoneExecutingAction implements IJobStatusAction { jobRuntime.setStatus(JobStatus.RUNABLE); jobRuntime.NotifyUpdate(); jobEngine.addToRunQueue(jobRuntime); - if (!jobRuntime.isService()) - logger.info(jobRuntime.getFinishMarker(), "A '{}' folyamat futása sikeresen véget ért", jobRuntime.getName()); } } diff --git a/server/user.jobengine.osgi.server/src/user/jobengine/server/actions/DoneSuspendAction.java b/server/user.jobengine.osgi.server/src/user/jobengine/server/actions/DoneSuspendAction.java index f9fa229b..b0f88c6a 100644 --- a/server/user.jobengine.osgi.server/src/user/jobengine/server/actions/DoneSuspendAction.java +++ b/server/user.jobengine.osgi.server/src/user/jobengine/server/actions/DoneSuspendAction.java @@ -2,16 +2,11 @@ package user.jobengine.server.actions; import java.sql.Timestamp; -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; - import user.commons.JobStatus; import user.jobengine.server.IJobEngine; import user.jobengine.server.IJobRuntime; public class DoneSuspendAction implements IJobStatusAction { - private static final Logger logger = LogManager.getLogger(); - @Override public void processAction(IJobEngine jobEngine, IJobRuntime jobRuntime) { jobRuntime.saveStatus(); @@ -20,7 +15,5 @@ public class DoneSuspendAction implements IJobStatusAction { jobRuntime.NotifyUpdate(); jobRuntime.restoreStack(); jobRuntime.decrementInstructionPointer(); - if (!jobRuntime.isService()) - logger.error(jobRuntime.getFinishMarker(), "A '{}' folyamat futása megszakadt", jobRuntime.getName()); } } diff --git a/server/user.jobengine.osgi.server/src/user/jobengine/server/actions/ExecuteRunableAction.java b/server/user.jobengine.osgi.server/src/user/jobengine/server/actions/ExecuteRunableAction.java index 2d6c15d4..d9a9b9fa 100644 --- a/server/user.jobengine.osgi.server/src/user/jobengine/server/actions/ExecuteRunableAction.java +++ b/server/user.jobengine.osgi.server/src/user/jobengine/server/actions/ExecuteRunableAction.java @@ -1,22 +1,15 @@ package user.jobengine.server.actions; -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; - import user.commons.JobStatus; import user.jobengine.server.IJobEngine; import user.jobengine.server.IJobRuntime; public class ExecuteRunableAction implements IJobStatusAction { - private static final Logger logger = LogManager.getLogger(); - @Override public void processAction(IJobEngine jobEngine, IJobRuntime jobRuntime) { jobRuntime.setStatus(JobStatus.WAIT_EXECUTOR); jobRuntime.NotifyUpdate(); jobRuntime.saveStack(); jobEngine.addToExecutorQueue(jobRuntime); - if (!jobRuntime.isService()) - logger.info(jobRuntime.getMarker(), "A '{}' folyamat elindult", jobRuntime.getName()); } } diff --git a/server/user.jobengine.osgi.server/src/user/jobengine/server/actions/StatusMachine.java b/server/user.jobengine.osgi.server/src/user/jobengine/server/actions/StatusMachine.java index a030bcdd..698e7af3 100644 --- a/server/user.jobengine.osgi.server/src/user/jobengine/server/actions/StatusMachine.java +++ b/server/user.jobengine.osgi.server/src/user/jobengine/server/actions/StatusMachine.java @@ -30,7 +30,7 @@ public class StatusMachine implements IStatusMachine { StatusMachineAction machineAction = new StatusMachineAction(jobAction, jobRuntime.getStatus()); if (actions.containsKey(machineAction)) { IJobStatusAction action = actions.get(machineAction); - logger.debug("{} status change {} -> {}", jobRuntime, jobRuntime.getStatus(), jobAction); + logger.info("{} status change {} -> {}", jobRuntime, jobRuntime.getStatus(), jobAction); action.processAction(jobEngine, jobRuntime); }