git.useribm.hu - mediacube.git/blob

   1 package user.jobengine.server.steps;\r
   2 \r
   3 import java.io.IOException;\r
   4 import java.net.InetAddress;\r
   5 import java.net.URI;\r
   6 import java.net.URISyntaxException;\r
   7 import java.util.Calendar;\r
   8 import java.util.Date;\r
   9 import java.util.List;\r
  10 \r
  11 import org.apache.commons.lang.StringUtils;\r
  12 import org.apache.commons.net.ftp.FTP;\r
  13 import org.apache.commons.net.ftp.FTPClient;\r
  14 import org.apache.commons.net.ftp.FTPReply;\r
  15 import org.apache.logging.log4j.LogManager;\r
  16 import org.apache.logging.log4j.Logger;\r
  17 \r
  18 import com.ibm.nosql.json.api.BasicDBObject;\r
  19 import com.ibm.nosql.json.api.DB;\r
  20 import com.ibm.nosql.json.api.DBCollection;\r
  21 import com.ibm.nosql.json.api.DBObject;\r
  22 \r
  23 import user.commons.CalendarUtils;\r
  24 import user.commons.StoreUri;\r
  25 import user.commons.nosql.NoSQLUtils;\r
  26 import user.commons.octopus.IOctopusAPI;\r
  27 import user.commons.octopus.OctopusAPI;\r
  28 import user.commons.remotestore.FtpDirectoryLister;\r
  29 import user.commons.remotestore.RemoteFileHandler;\r
  30 import user.commons.remotestore.RemoteStoreProtocol;\r
  31 import user.jobengine.db.IItemManager;\r
  32 import user.jobengine.server.IJobEngine;\r
  33 import user.jobengine.server.IJobRuntime;\r
  34 import user.jobengine.server.JobEngineException;\r
  35 import user.jobengine.server.steps.MetadataTypeDetector.MetadataType;\r
  36 \r
  37 public class CopyForArchiveNEXIOMaterialsStep extends JobStep {\r
  38         private static final String MXFEXT = ".MXF";\r
  39         private static final Logger logger = LogManager.getLogger();\r
  40         private static final String NEXIOCLIPS = "nexioclips";\r
  41         private static final String KILLDATE = "killdate";\r
  42         private static final String LONGNAMEID = "longnameid";\r
  43         private static final String ID = "id";\r
  44         private StoreUri sourceUri;\r
  45         private StoreUri targetUri;\r
  46         private OctopusAPI octopusAPI;\r
  47         private IItemManager manager;\r
  48 \r
  49         private DB db;\r
  50 \r
  51         private int check(int value, String name) {\r
  52                 if (value == 0) {\r
  53                         logger.error(getMarker(), "A folyamat '{}' bemeneti paramétere 0.", name);\r
  54                         throw new NullPointerException(String.format("System is not configured properly, missing '%s' input parameter.", name));\r
  55                 }\r
  56                 return value;\r
  57         }\r
  58 \r
  59         private String check(String value, String name) {\r
  60                 if (value == null) {\r
  61                         logger.error(getMarker(), "A folyamat '{}' bemeneti paramétere üres.", name);\r
  62                         throw new NullPointerException(String.format("System is not configured properly, missing '%s' input parameter.", name));\r
  63                 }\r
  64                 return value;\r
  65         }\r
  66 \r
  67         private void copy(RundownArchive rundownArchive) throws Exception {\r
  68                 for (StoryArchive storyArchive : rundownArchive.getStoryArchives()) {\r
  69                         for (String fileName : storyArchive.getFileNames()) {\r
  70                                 copyFile(fileName, rundownArchive, storyArchive);\r
  71                         }\r
  72                 }\r
  73         }\r
  74 \r
  75         private void copyFile(String fileName, RundownArchive rundownArchive, StoryArchive storyArchive) throws Exception {\r
  76                 FTPClient source = ((FtpDirectoryLister) RemoteFileHandler.createLister(sourceUri)).connect();\r
  77                 FTPClient target = ((FtpDirectoryLister) RemoteFileHandler.createLister(sourceUri)).connect();\r
  78                 transferFile(source, target, fileName);\r
  79                 BasicDBObject metadata = createMetadata(rundownArchive, storyArchive);\r
  80                 transferMetadata(target, metadata);\r
  81                 createTargetKillDateFile(target, fileName);\r
  82                 createSourceKillDateFile(source, fileName);\r
  83         }\r
  84 \r
  85         private BasicDBObject createMetadata(RundownArchive rundownArchive, StoryArchive storyArchive) {\r
  86                 //              {\r
  87                 //                        "itemHouseId": "43",\r
  88                 //                        "itemTitle": "Hazahúzó",\r
  89                 //                        "itemDescription": null,\r
  90                 //                        "mediaHouseId": "002570",\r
  91                 //                        "mediaTitle": "2017.12.13",\r
  92                 //                        "mediaDescription": null,\r
  93                 //                        "ok": false,\r
  94                 //                        "userName": "echotest"\r
  95                 //              }\r
  96                 BasicDBObject result = new BasicDBObject();\r
  97                 result.put("itemHouseID", rundownArchive.getItemHouseId());\r
  98                 result.put("itemTitle", rundownArchive.getItemTitle());\r
  99                 result.put("itemDescription", rundownArchive.getItemDesc());\r
 100                 result.put("userName", "mediacube");\r
 101 \r
 102                 result.put("mediaHouseId", storyArchive.getMediaHouseId());\r
 103                 result.put("mediaTitle", storyArchive.getMediaTitle());\r
 104                 result.put("mediaDescription", storyArchive.getMediaDesc());\r
 105                 return result;\r
 106         }\r
 107 \r
 108         private void createSourceKillDateFile(FTPClient source, String fileName) {\r
 109         }\r
 110 \r
 111         private void createTargetKillDateFile(FTPClient target, String fileName) {\r
 112         }\r
 113 \r
 114         @StepEntry\r
 115         public Object[] execute(int nexioPort, String nexioUserName, String nexioPassword, String archiveFtp, String archiveUserName, String archivePassword,\r
 116                         int daysBeforeNow, IJobEngine jobEngine, IJobRuntime jobRuntime) throws Exception {\r
 117                 setAndCheck(nexioPort, nexioUserName, nexioPassword, archiveFtp, archiveUserName, archivePassword, jobEngine);\r
 118                 octopusAPI = new OctopusAPI();\r
 119                 Calendar scheduledDate = Calendar.getInstance();\r
 120                 scheduledDate.add(Calendar.DAY_OF_YEAR, -1 * daysBeforeNow);\r
 121                 List<DBObject> rundowns = octopusAPI.getRundowns(scheduledDate.getTime());\r
 122                 if (rundowns == null) {\r
 123                         logger.warn(getMarker(), "Nem található adástükör a {} napra.", CalendarUtils.toDateString(scheduledDate));\r
 124                         return null;\r
 125                 }\r
 126                 int index = 1;\r
 127 \r
 128                 for (DBObject r : rundowns) {\r
 129                         setProgress(index * 100 / rundowns.size());\r
 130                         try {\r
 131                                 RundownArchive rundownArchive = processRundow(r);\r
 132                                 if (rundownArchive == null || rundownArchive.isEmpty())\r
 133                                         continue;\r
 134                                 copy(rundownArchive);\r
 135 \r
 136                         } catch (Exception e) {\r
 137                                 logger.catching(e);\r
 138                                 logger.error(getMarker(),\r
 139                                                 "A {} tükör archiválása nem lehetséges, mert a annak ellenőrzése hibát jelzett. A rendszer üzenete: " + e.getMessage());\r
 140                         }\r
 141                         index++;\r
 142                 }\r
 143                 return null;\r
 144         }\r
 145 \r
 146         private String processMosObject(BasicDBObject rundown, BasicDBObject story, BasicDBObject mosObject) throws Exception {\r
 147                 String mosID = mosObject.getString(IOctopusAPI.OBJ_ID);\r
 148                 if (MetadataTypeDetector.GuessMetadataType(mosID) != MetadataType.OCTOPUSPLACEHOLDER) {\r
 149                         logger.warn("Skipping MOS object {}", mosID);\r
 150                         return null;\r
 151                 }\r
 152                 DBCollection clips = db.getCollection(NEXIOCLIPS);\r
 153                 DBObject clip = clips.findOne(new BasicDBObject(LONGNAMEID, mosID));\r
 154                 if (clip == null) {\r
 155                         logger.info("File NOT exists {}", mosID);\r
 156                         throw new Exception(String.format("File NOT exists %s"));\r
 157                 } else {\r
 158                         logger.info("File exists {}", mosID);\r
 159                 }\r
 160                 return mosID + MXFEXT;\r
 161         }\r
 162 \r
 163         private RundownArchive processRundow(DBObject r) throws Exception {\r
 164                 BasicDBObject rundown = (BasicDBObject) r;\r
 165                 long rundownID = rundown.getLong(ID);\r
 166                 logger.info("Processing rundown {} {}", rundownID, rundown.getString(IOctopusAPI.NAME));\r
 167                 List<DBObject> stories = octopusAPI.getRundownFullStories(rundownID);\r
 168                 if (stories == null)\r
 169                         return null;\r
 170                 RundownArchive result = new RundownArchive();\r
 171                 //              {\r
 172                 //                      "id":22358752,\r
 173                 //                      "rundownType":{"name":"Híradó"},\r
 174                 //                      "scheduledStart":{"$date":"2017-12-09T19:00:00.000Z"}\r
 175                 //              }\r
 176 \r
 177                 long id = NoSQLUtils.asLong(rundown, IOctopusAPI.ID);\r
 178                 if (id == 0)\r
 179                         return null;\r
 180                 String name = NoSQLUtils.asString(NoSQLUtils.asDBObject(rundown, IOctopusAPI.RUNDOWN_TYPE), IOctopusAPI.NAME);\r
 181                 if (StringUtils.isBlank(name))\r
 182                         return null;\r
 183                 Date scheduledStart = rundown.getDate(IOctopusAPI.SCHEDULED_START);\r
 184                 if (scheduledStart == null)\r
 185                         return null;\r
 186                 String start = CalendarUtils.toHunString(CalendarUtils.createCalendar(scheduledStart));\r
 187 \r
 188                 result.setItemHouseId(String.valueOf(id));\r
 189                 result.setItemTitle(String.format("%s - %s", start, name));\r
 190 \r
 191                 for (DBObject s : stories) {\r
 192                         StoryArchive storyArchive = processStory(rundown, s);\r
 193                         if (storyArchive == null)\r
 194                                 continue;\r
 195                         result.addStoryArchive(storyArchive);\r
 196                 }\r
 197                 return result;\r
 198         }\r
 199 \r
 200         private StoryArchive processStory(BasicDBObject rundown, DBObject s) throws Exception {\r
 201                 BasicDBObject story = (BasicDBObject) s;\r
 202                 String parentStoryID = story.getString(IOctopusAPI.PARENT_STORY_ID);\r
 203                 if (StringUtils.isBlank(parentStoryID)) {\r
 204                         logger.warn("Story parentStoryID is null: {}", story.toPrettyString(null));\r
 205                         return null;\r
 206                 } else\r
 207                         logger.info("Processing story {}", parentStoryID);\r
 208                 List<BasicDBObject> mosObjects = NoSQLUtils.asList(story, IOctopusAPI.MOS_OBJECTS);\r
 209                 if (mosObjects == null)\r
 210                         return null;\r
 211                 StoryArchive storyArchive = null;\r
 212                 for (BasicDBObject mosObject : mosObjects) {\r
 213                         String fileName = processMosObject(rundown, story, mosObject);\r
 214                         if (fileName == null)\r
 215                                 continue;\r
 216                         if (storyArchive == null) {\r
 217                                 storyArchive = new StoryArchive();\r
 218                                 storyArchive.setMediaHouseId(parentStoryID);\r
 219                                 storyArchive.setMediaTitle(story.getString(IOctopusAPI.NAME));\r
 220                                 storyArchive.setMediaDesc(story.getString(IOctopusAPI.SCRIPT_CONTENT));\r
 221                         }\r
 222                         storyArchive.addFile(fileName);\r
 223                 }\r
 224                 return null;\r
 225 \r
 226         }\r
 227 \r
 228         private void setAndCheck(int nexioPort, String nexioUserName, String nexioPassword, String archiveFtp, String archiveUserName, String archivePassword,\r
 229                         IJobEngine jobEngine) throws JobEngineException, IOException, URISyntaxException {\r
 230                 db = NoSQLUtils.getNoSQLDB();\r
 231                 if (db == null) {\r
 232                         logger.error(getMarker(), "Az NoSQL adatkezelő réteg nem elérhető.");\r
 233                         throw new NullPointerException("Internal error, missing NoSQL DB reference.");\r
 234                 }\r
 235 \r
 236                 if (jobEngine == null) {\r
 237                         logger.error(getMarker(), "Az folyamatkezelő réteg nem elérhető.");\r
 238                         throw new NullPointerException("Internal error, missing JobEngine reference.");\r
 239                 }\r
 240                 manager = jobEngine.getItemManager();\r
 241                 if (manager == null) {\r
 242                         logger.error(getMarker(), "Az adatbáziskezelő réteg nem elérhető.");\r
 243                         throw new NullPointerException("Internal error, missing ItemManager reference.");\r
 244                 }\r
 245                 String nexioHost = System.getProperty("nexio.host");\r
 246                 if (StringUtils.isBlank(nexioHost)) {\r
 247                         logger.error(getMarker(), "A 'nexio.host' rendszer paraméter nem található.");\r
 248                         throw new NullPointerException("System is not configured properly, 'jobengine.selenio.address' startup parameter missing.");\r
 249                 }\r
 250                 check(nexioPort, "nexioPort");\r
 251                 check(nexioUserName, "nexioUserName");\r
 252                 check(nexioPassword, "nexioPassword");\r
 253 \r
 254                 sourceUri = manager.createStoreUri(RemoteStoreProtocol.FTP, nexioHost);\r
 255                 sourceUri.setPortNumber(nexioPort);\r
 256                 sourceUri.setUserName(nexioUserName);\r
 257                 sourceUri.setPassword(nexioPassword);\r
 258 \r
 259                 check(archiveFtp, "archiveFtp");\r
 260                 check(archiveUserName, "archiveUserName");\r
 261                 check(archivePassword, "archivePassword");\r
 262 \r
 263                 targetUri = manager.createStoreUri(new URI(archiveFtp));\r
 264                 targetUri.setUserName(archiveUserName);\r
 265                 targetUri.setPassword(archivePassword);\r
 266         }\r
 267 \r
 268         private void transferFile(FTPClient source, FTPClient target, String fileName) throws Exception {\r
 269                 int reply = 0;\r
 270 \r
 271                 if (!target.enterRemotePassiveMode())\r
 272                         throw new Exception("!PASV");\r
 273 \r
 274                 reply = source.port(InetAddress.getByName(target.getPassiveHost()), target.getPassivePort());\r
 275                 if (!FTPReply.isPositiveCompletion(reply))\r
 276                         throw new Exception("!PORT");\r
 277 \r
 278                 if (!source.setFileType(FTP.BINARY_FILE_TYPE))\r
 279                         throw new Exception("!SOURCE TYPE");\r
 280 \r
 281                 if (!target.setFileType(FTP.BINARY_FILE_TYPE))\r
 282                         throw new Exception("!TARGET TYPE");\r
 283 \r
 284                 source.retr(fileName);\r
 285                 target.stor(fileName);\r
 286 \r
 287                 while (true) {\r
 288                         reply = source.stat();\r
 289                         if (!FTPReply.isPositiveCompletion(reply))\r
 290                                 throw new Exception("!STAT");\r
 291                         String replyText = source.getReplyString();\r
 292                         if ("Transaction completed".equals(replyText))\r
 293                                 break;\r
 294                 }\r
 295         }\r
 296 \r
 297         private void transferMetadata(FTPClient target, BasicDBObject metadata) {\r
 298         }\r
 299 \r
 300 }\r