git.useribm.hu - mediacube.git/blob

   1 package user.jobengine.server.steps;\r
   2 \r
   3 import java.io.OutputStream;\r
   4 import java.net.InetAddress;\r
   5 import java.net.URI;\r
   6 import java.util.Calendar;\r
   7 import java.util.Date;\r
   8 import java.util.List;\r
   9 \r
  10 import org.apache.commons.lang.StringUtils;\r
  11 import org.apache.commons.net.ftp.FTP;\r
  12 import org.apache.commons.net.ftp.FTPClient;\r
  13 import org.apache.commons.net.ftp.FTPReply;\r
  14 import org.apache.logging.log4j.LogManager;\r
  15 import org.apache.logging.log4j.Logger;\r
  16 \r
  17 import com.ibm.nosql.json.api.BasicDBObject;\r
  18 import com.ibm.nosql.json.api.DB;\r
  19 import com.ibm.nosql.json.api.DBCollection;\r
  20 import com.ibm.nosql.json.api.DBCursor;\r
  21 import com.ibm.nosql.json.api.DBObject;\r
  22 \r
  23 import user.commons.CalendarUtils;\r
  24 import user.commons.ListUtils;\r
  25 import user.commons.StoreUri;\r
  26 import user.commons.nosql.NoSQLUtils;\r
  27 import user.commons.octopus.IOctopusAPI;\r
  28 import user.commons.octopus.OctopusAPI;\r
  29 import user.commons.remotestore.FtpDirectoryLister;\r
  30 import user.commons.remotestore.RemoteFileHandler;\r
  31 import user.commons.remotestore.RemoteStoreProtocol;\r
  32 import user.jobengine.db.IItemManager;\r
  33 import user.jobengine.server.IJobEngine;\r
  34 import user.jobengine.server.IJobRuntime;\r
  35 import user.jobengine.server.steps.MetadataTypeDetector.MetadataType;\r
  36 \r
  37 public class CopyForArchiveNEXIOMaterialsStep extends JobStep {\r
  38         private static final String SCHEDULED_FORMAT = "yyyy.MM.dd HH:mm";\r
  39         private static final Logger logger = LogManager.getLogger();\r
  40         private static final String UTF_8 = "utf-8";\r
  41         private static final String JSON_EXT = ".json";\r
  42         private static final String XML_EXT = ".xml";\r
  43         private static final String DURATION = "duration";\r
  44         private static final String MXFEXT = ".MXF";\r
  45         private static final String NEXIOCLIPS = "nexioclips";\r
  46         private static final String LONGNAMEID = "longnameid";\r
  47         private static final String ARCHIVEDRUNDOWNS = "archivedrundowns";\r
  48         private static final String ID = "id";\r
  49         private OctopusAPI octopusAPI;\r
  50         private IItemManager manager;\r
  51 \r
  52         private DB db;\r
  53         private FTPClient sourceFtp;\r
  54         private FTPClient targetFtp;\r
  55         private StoreUri sourceUri;\r
  56         private StoreUri targetUri;\r
  57         private int nexioKillDateDays;\r
  58         private String nexioAgency;\r
  59 \r
  60         private int check(int value, String name) {\r
  61                 if (value == 0) {\r
  62                         logger.error(getMarker(), "A folyamat '{}' bemeneti paramétere 0.", name);\r
  63                         throw new NullPointerException(String.format("System is not configured properly, missing '%s' input parameter.", name));\r
  64                 }\r
  65                 return value;\r
  66         }\r
  67 \r
  68         private String check(String value, String name) {\r
  69                 if (value == null) {\r
  70                         logger.error(getMarker(), "A folyamat '{}' bemeneti paramétere üres.", name);\r
  71                         throw new NullPointerException(String.format("System is not configured properly, missing '%s' input parameter.", name));\r
  72                 }\r
  73                 return value;\r
  74         }\r
  75 \r
  76         private void copy(RundownArchive rundownArchive) throws Exception {\r
  77                 for (StoryArchive storyArchive : rundownArchive.getStoryArchives()) {\r
  78                         for (FileArchive fileArchive : storyArchive.getFileArchives()) {\r
  79                                 copyFile(fileArchive, rundownArchive, storyArchive);\r
  80                         }\r
  81                 }\r
  82         }\r
  83 \r
  84         private void copyFile(FileArchive fileArchive, RundownArchive rundownArchive, StoryArchive storyArchive) throws Exception {\r
  85                 String fileName = fileArchive.getFileName();\r
  86                 //TODO file existance\r
  87                 //              targetFtp = ((FtpDirectoryLister) RemoteFileHandler.createLister(targetUri)).connect();\r
  88                 //              int version = 1;\r
  89                 //              while (true) {\r
  90                 //                      targetFtp.listNames(arg0)\r
  91                 //              }\r
  92                 //\r
  93                 //              if (targetUri != null)\r
  94                 //                      targetUri.cleanUp();\r
  95                 String videoFileName = fileName + MXFEXT;\r
  96 \r
  97                 transferFile(videoFileName);\r
  98                 BasicDBObject metadata = createMetadata(rundownArchive, storyArchive, fileArchive);\r
  99                 transferMetadata(videoFileName, metadata);\r
 100                 //targetFtp.changeToParentDirectory();\r
 101                 createSourceKillDateFile(rundownArchive, fileName);\r
 102 \r
 103         }\r
 104 \r
 105         private BasicDBObject createMetadata(RundownArchive rundownArchive, StoryArchive storyArchive, FileArchive fileArchive) {\r
 106                 //              {\r
 107                 //                        "itemHouseId": "43",\r
 108                 //                        "itemTitle": "Hazahúzó",\r
 109                 //                        "itemDescription": null,\r
 110                 //                        "mediaHouseId": "002570",\r
 111                 //                        "mediaTitle": "2017.12.13",\r
 112                 //                        "mediaDescription": null,\r
 113                 //                        "ok": false,\r
 114                 //                        "userName": "echotest"\r
 115                 //              }\r
 116                 BasicDBObject result = new BasicDBObject();\r
 117                 result.put("itemHouseID", rundownArchive.getItemHouseId());\r
 118                 result.put("itemTitle", rundownArchive.getItemTitle());\r
 119                 result.put("itemDescription", rundownArchive.getItemDesc());\r
 120                 result.put("userName", "mediacube");\r
 121 \r
 122                 result.put("mediaHouseId", storyArchive.getMediaHouseId());\r
 123                 result.put("mediaTitle", storyArchive.getMediaTitle());\r
 124                 result.put("mediaDescription", storyArchive.getMediaDesc());\r
 125 \r
 126                 result.put("duration", fileArchive.getDuration());\r
 127                 return result;\r
 128         }\r
 129 \r
 130         private void createSourceKillDateFile(RundownArchive rundownArchive, String fileName) throws Exception {\r
 131                 //sourceFtp = ((FtpDirectoryLister) RemoteFileHandler.createLister(sourceUri)).connect();\r
 132                 OutputStream outStream = null;\r
 133                 try {\r
 134                         sourceFtp.enterLocalActiveMode();\r
 135                         Calendar killDate = CalendarUtils.createCalendar(rundownArchive.getScheduleDate());\r
 136                         killDate.add(Calendar.DAY_OF_YEAR, nexioKillDateDays);\r
 137                         byte[] killDateFile = EscortFiles.createNEXIOKillDateFile(fileName, killDate.getTime(), null, nexioAgency);\r
 138                         outStream = sourceFtp.storeFileStream(fileName + XML_EXT);\r
 139                         if (outStream == null) {\r
 140                                 throw new NullPointerException("Can not open: " + fileName + XML_EXT + " Reply:" + sourceFtp.getReplyString());\r
 141                         }\r
 142                         outStream.write(killDateFile);\r
 143                         outStream.flush();\r
 144                 } catch (Exception e) {\r
 145                         logger.catching(e);\r
 146                         throw e;\r
 147                 } finally {\r
 148                         if (outStream != null)\r
 149                                 outStream.close();\r
 150                         //                      if (sourceUri != null)\r
 151                         //                              sourceUri.cleanUp();\r
 152                 }\r
 153         }\r
 154 \r
 155         @StepEntry\r
 156         public Object[] execute(int nexioPort, String nexioUserName, String nexioPassword, String archiveFtp, String archiveUserName, String archivePassword,\r
 157                         int daysBeforeNow, int nexioKillDateDays, String nexioAgency, IJobEngine jobEngine, IJobRuntime jobRuntime) throws Exception {\r
 158                 setAndCheck(nexioPort, nexioUserName, nexioPassword, archiveFtp, archiveUserName, archivePassword, nexioKillDateDays, nexioAgency, jobEngine);\r
 159                 octopusAPI = new OctopusAPI();\r
 160                 Calendar scheduledDate = Calendar.getInstance();\r
 161                 scheduledDate.add(Calendar.DAY_OF_YEAR, -1 * daysBeforeNow);\r
 162                 List<DBObject> rundowns = octopusAPI.getRundowns(scheduledDate.getTime());\r
 163                 if (rundowns == null) {\r
 164                         logger.warn(getMarker(), "Nem található adástükör a {} napra.", CalendarUtils.toDateString(scheduledDate));\r
 165                         return null;\r
 166                 }\r
 167 \r
 168                 sourceFtp = ((FtpDirectoryLister) RemoteFileHandler.createLister(sourceUri)).connect();\r
 169                 targetFtp = ((FtpDirectoryLister) RemoteFileHandler.createLister(targetUri)).connect();\r
 170                 processRundowns(rundowns);\r
 171                 if (sourceUri != null)\r
 172                         sourceUri.cleanUp();\r
 173                 if (targetUri != null)\r
 174                         targetUri.cleanUp();\r
 175                 return null;\r
 176         }\r
 177 \r
 178         private FileArchive processMosObject(BasicDBObject rundown, BasicDBObject story, BasicDBObject mosObject) throws Exception {\r
 179                 String mosID = mosObject.getString(IOctopusAPI.OBJ_ID);\r
 180                 if (MetadataTypeDetector.GuessMetadataType(mosID) != MetadataType.OCTOPUSPLACEHOLDER) {\r
 181                         logger.trace("Skipping MOS object {}", mosID);\r
 182                         return null;\r
 183                 }\r
 184                 DBCollection clips = db.getCollection(NEXIOCLIPS);\r
 185                 BasicDBObject clip = (BasicDBObject) clips.findOne(new BasicDBObject(LONGNAMEID, mosID));\r
 186                 if (clip == null) {\r
 187                         logger.debug("File NOT exists {}", mosID);\r
 188                         throw new Exception(String.format("File NOT exists %s", mosID));\r
 189                 } else {\r
 190                         logger.debug("File exists {}", mosID);\r
 191                 }\r
 192                 long duration = NoSQLUtils.asLong(clip, DURATION);\r
 193                 return new FileArchive(mosID, duration);\r
 194         }\r
 195 \r
 196         private RundownArchive processRundow(DBObject r) throws Exception {\r
 197                 BasicDBObject rundown = (BasicDBObject) r;\r
 198                 long rundownID = rundown.getLong(ID);\r
 199                 logger.info("Processing rundown {} {}", rundownID, rundown.getString(IOctopusAPI.NAME));\r
 200                 List<DBObject> stories = octopusAPI.getRundownFullStories(rundownID);\r
 201                 if (stories == null)\r
 202                         return null;\r
 203                 RundownArchive result = new RundownArchive();\r
 204 \r
 205                 long id = NoSQLUtils.asLong(rundown, IOctopusAPI.ID);\r
 206                 if (id == 0)\r
 207                         return null;\r
 208                 String name = NoSQLUtils.asString(NoSQLUtils.asDBObject(rundown, IOctopusAPI.RUNDOWN_TYPE), IOctopusAPI.NAME);\r
 209                 if (StringUtils.isBlank(name))\r
 210                         return null;\r
 211                 String channel = NoSQLUtils.asString(NoSQLUtils.asDBObject(rundown, IOctopusAPI.CHANNEL), IOctopusAPI.NAME);\r
 212                 Date scheduledStart = rundown.getDate(IOctopusAPI.SCHEDULED_START);\r
 213                 if (scheduledStart == null)\r
 214                         return null;\r
 215                 String start = CalendarUtils.toString(CalendarUtils.createCalendar(scheduledStart), SCHEDULED_FORMAT);\r
 216                 result.setScheduleDate(scheduledStart);\r
 217                 result.setItemHouseId(String.valueOf(id));\r
 218                 result.setItemTitle(String.format("%s %s %s", start, name, channel));\r
 219 \r
 220                 for (DBObject s : stories) {\r
 221                         StoryArchive storyArchive = processStory(rundown, s);\r
 222                         if (storyArchive == null)\r
 223                                 continue;\r
 224                         result.addStoryArchive(storyArchive);\r
 225                 }\r
 226                 return result;\r
 227         }\r
 228 \r
 229         private void processRundowns(List<DBObject> rundowns) {\r
 230                 db.getCollection(ARCHIVEDRUNDOWNS).drop();\r
 231                 List<BasicDBObject> archivedRundowns = queryArchivedRundowns();\r
 232 \r
 233                 int index = 1;\r
 234 \r
 235                 for (DBObject r : rundowns) {\r
 236                         BasicDBObject rundown = (BasicDBObject) r;\r
 237                         setProgress(index * 100 / rundowns.size());\r
 238                         try {\r
 239                                 long rundownID = NoSQLUtils.asLong(rundown, IOctopusAPI.ID);\r
 240                                 BasicDBObject currentRundownID = new BasicDBObject(IOctopusAPI.ID, rundownID);\r
 241                                 String rundownName = rundown.getString(IOctopusAPI.NAME);\r
 242                                 if (archivedRundowns != null && archivedRundowns.contains(currentRundownID)) {\r
 243                                         logger.info("Skipping archived rundown {} {}", rundownID, rundownName);\r
 244                                         continue;\r
 245                                 }\r
 246 \r
 247                                 RundownArchive rundownArchive = processRundow(r);\r
 248                                 if (rundownArchive == null || rundownArchive.isEmpty()) {\r
 249                                         logger.info("Skipping rundown {} {}", NoSQLUtils.asLong(rundown, IOctopusAPI.ID), rundown.getString(IOctopusAPI.NAME));\r
 250                                         continue;\r
 251                                 }\r
 252 \r
 253                                 logger.info("Saving rundown {} {}", rundownID, rundownName);\r
 254                                 copy(rundownArchive);\r
 255 \r
 256                                 db.getCollection(ARCHIVEDRUNDOWNS).save(currentRundownID);\r
 257                         } catch (Exception e) {\r
 258                                 logger.catching(e);\r
 259                                 logger.error(getMarker(),\r
 260                                                 "A {} tükör archiválása nem lehetséges, mert a annak ellenőrzése hibát jelzett. A rendszer üzenete: " + e.getMessage());\r
 261                         }\r
 262                         index++;\r
 263                 }\r
 264 \r
 265         }\r
 266 \r
 267         private StoryArchive processStory(BasicDBObject rundown, DBObject s) throws Exception {\r
 268                 BasicDBObject story = (BasicDBObject) s;\r
 269                 String parentStoryID = story.getString(IOctopusAPI.PARENT_STORY_ID);\r
 270                 if (StringUtils.isBlank(parentStoryID)) {\r
 271                         logger.warn("Story parentStoryID is null: {}", story.toPrettyString(null));\r
 272                         return null;\r
 273                 } else\r
 274                         logger.debug("Processing story {}", parentStoryID);\r
 275                 List<BasicDBObject> mosObjects = NoSQLUtils.asList(story, IOctopusAPI.MOS_OBJECTS);\r
 276                 if (mosObjects == null)\r
 277                         return null;\r
 278                 StoryArchive storyArchive = null;\r
 279                 for (BasicDBObject mosObject : mosObjects) {\r
 280                         FileArchive fileArchive = processMosObject(rundown, story, mosObject);\r
 281                         if (fileArchive == null)\r
 282                                 continue;\r
 283                         if (storyArchive == null) {\r
 284                                 storyArchive = new StoryArchive();\r
 285                                 storyArchive.setMediaHouseId(parentStoryID);\r
 286                                 storyArchive.setMediaTitle(story.getString(IOctopusAPI.NAME));\r
 287                                 storyArchive.setMediaDesc(story.getString(IOctopusAPI.SCRIPT_CONTENT));\r
 288 \r
 289                         }\r
 290                         storyArchive.addFileArchive(fileArchive);\r
 291                 }\r
 292                 return storyArchive;\r
 293 \r
 294         }\r
 295 \r
 296         private List<BasicDBObject> queryArchivedRundowns() {\r
 297                 List<BasicDBObject> result = null;\r
 298                 DBCollection collection = db.getCollection(ARCHIVEDRUNDOWNS);\r
 299                 DBCursor find = collection.find(new BasicDBObject(), new BasicDBObject(IOctopusAPI._ID, 0).append(IOctopusAPI.ID, 1));\r
 300                 if (find.hasNext())\r
 301                         result = ListUtils.cast(find.toArray());\r
 302                 return result;\r
 303         }\r
 304 \r
 305         private void setAndCheck(int nexioPort, String nexioUserName, String nexioPassword, String archiveFtp, String archiveUserName, String archivePassword,\r
 306                         int nexioKillDateDays, String nexioAgency, IJobEngine jobEngine) throws Exception {\r
 307                 db = NoSQLUtils.getNoSQLDB();\r
 308                 if (db == null) {\r
 309                         logger.error(getMarker(), "Az NoSQL adatkezelő réteg nem elérhető.");\r
 310                         throw new NullPointerException("Internal error, missing NoSQL DB reference.");\r
 311                 }\r
 312 \r
 313                 if (jobEngine == null) {\r
 314                         logger.error(getMarker(), "Az folyamatkezelő réteg nem elérhető.");\r
 315                         throw new NullPointerException("Internal error, missing JobEngine reference.");\r
 316                 }\r
 317                 manager = jobEngine.getItemManager();\r
 318                 if (manager == null) {\r
 319                         logger.error(getMarker(), "Az adatbáziskezelő réteg nem elérhető.");\r
 320                         throw new NullPointerException("Internal error, missing ItemManager reference.");\r
 321                 }\r
 322                 String nexioHost = System.getProperty("nexio.host");\r
 323                 if (StringUtils.isBlank(nexioHost)) {\r
 324                         logger.error(getMarker(), "A 'nexio.host' rendszer paraméter nem található.");\r
 325                         throw new NullPointerException("System is not configured properly, 'jobengine.selenio.address' startup parameter missing.");\r
 326                 }\r
 327                 check(nexioPort, "nexioPort");\r
 328                 check(nexioUserName, "nexioUserName");\r
 329                 check(nexioPassword, "nexioPassword");\r
 330 \r
 331                 check(nexioKillDateDays, "nexioKillDateDays");\r
 332                 this.nexioKillDateDays = nexioKillDateDays;\r
 333                 check(nexioAgency, "nexioAgency");\r
 334                 this.nexioAgency = nexioAgency;\r
 335 \r
 336                 sourceUri = manager.createStoreUri(RemoteStoreProtocol.FTP, nexioHost);\r
 337                 sourceUri.setPortNumber(nexioPort);\r
 338                 sourceUri.setUserName(nexioUserName);\r
 339                 sourceUri.setPassword(nexioPassword);\r
 340 \r
 341                 check(archiveFtp, "archiveFtp");\r
 342                 check(archiveUserName, "archiveUserName");\r
 343                 check(archivePassword, "archivePassword");\r
 344 \r
 345                 targetUri = manager.createStoreUri(new URI(archiveFtp));\r
 346                 targetUri.setUserName(archiveUserName);\r
 347                 targetUri.setPassword(archivePassword);\r
 348 \r
 349         }\r
 350 \r
 351         private void transferFile(String fileName) throws Exception {\r
 352                 int reply = 0;\r
 353                 //              sourceFtp = ((FtpDirectoryLister) RemoteFileHandler.createLister(sourceUri)).connect();\r
 354                 //              targetFtp = ((FtpDirectoryLister) RemoteFileHandler.createLister(targetUri)).connect();\r
 355                 //              if (sourceUri != null)\r
 356                 //                      sourceUri.cleanUp();\r
 357                 //              if (targetUri != null)\r
 358                 //                      targetUri.cleanUp();\r
 359 \r
 360                 if (!targetFtp.enterRemotePassiveMode())\r
 361                         throw new Exception("!PASV");\r
 362 \r
 363                 reply = sourceFtp.port(InetAddress.getByName(targetFtp.getPassiveHost()), targetFtp.getPassivePort());\r
 364                 if (!FTPReply.isPositiveCompletion(reply))\r
 365                         throw new Exception("!PORT");\r
 366 \r
 367                 if (!sourceFtp.setFileType(FTP.BINARY_FILE_TYPE))\r
 368                         throw new Exception("!SOURCE TYPE");\r
 369 \r
 370                 sourceFtp.retr(fileName);\r
 371 \r
 372                 if (!targetFtp.setFileType(FTP.BINARY_FILE_TYPE))\r
 373                         throw new Exception("!TARGET TYPE");\r
 374 \r
 375                 targetFtp.stor(fileName);\r
 376 \r
 377                 while (true) {\r
 378                         reply = sourceFtp.stat();\r
 379                         if (!FTPReply.isPositiveCompletion(reply))\r
 380                                 throw new Exception("!STAT");\r
 381 \r
 382                         String replyText = sourceFtp.getReplyString();\r
 383                         //logger.info("Status: {}", replyText);\r
 384                         if (reply == 226) {\r
 385                                 break;\r
 386                         }\r
 387                         Thread.sleep(100);\r
 388                 }\r
 389 \r
 390                 //              if (sourceUri != null)\r
 391                 //                      sourceUri.cleanUp();\r
 392                 //              if (targetUri != null)\r
 393                 //                      targetUri.cleanUp();\r
 394 \r
 395         }\r
 396 \r
 397         private void transferMetadata(String fileName, BasicDBObject metadata) throws Exception {\r
 398                 OutputStream outStream = null;\r
 399                 if (targetUri != null)\r
 400                         targetUri.cleanUp();\r
 401                 targetFtp = ((FtpDirectoryLister) RemoteFileHandler.createLister(targetUri)).connect();\r
 402                 try {\r
 403                         if (!targetFtp.changeWorkingDirectory(EscortFiles.STATUSFOLDER)) {\r
 404                                 targetFtp.makeDirectory(EscortFiles.STATUSFOLDER);\r
 405                                 if (!targetFtp.changeWorkingDirectory(EscortFiles.STATUSFOLDER))\r
 406                                         throw new Exception("!STATUSFOLDER");\r
 407                         }\r
 408 \r
 409                         outStream = targetFtp.storeFileStream(fileName + JSON_EXT);\r
 410                         if (outStream == null) {\r
 411                                 throw new NullPointerException("Can not open: " + fileName + JSON_EXT + " Reply:" + targetFtp.getReplyString());\r
 412                         }\r
 413                         outStream.write(metadata.toString().getBytes(UTF_8));\r
 414                         outStream.flush();\r
 415                         targetFtp.changeToParentDirectory();\r
 416                 } catch (Exception e) {\r
 417                         logger.catching(e);\r
 418                         throw e;\r
 419                 } finally {\r
 420                         if (outStream != null)\r
 421                                 outStream.close();\r
 422                         //                      if (targetUri != null)\r
 423                         //                              targetUri.cleanUp();\r
 424                 }\r
 425         }\r
 426 \r
 427 }\r