git.useribm.hu - mediacube.git/blob

   1 package hu.user.mediacube.executors.tests;\r
   2 \r
   3 import java.io.OutputStream;\r
   4 import java.net.InetAddress;\r
   5 import java.net.URI;\r
   6 import java.util.ArrayList;\r
   7 import java.util.Calendar;\r
   8 import java.util.Date;\r
   9 import java.util.List;\r
  10 \r
  11 import org.apache.commons.lang.StringUtils;\r
  12 import org.apache.commons.net.ftp.FTP;\r
  13 import org.apache.commons.net.ftp.FTPClient;\r
  14 import org.apache.commons.net.ftp.FTPReply;\r
  15 import org.apache.logging.log4j.LogManager;\r
  16 import org.apache.logging.log4j.Logger;\r
  17 import org.apache.logging.log4j.Marker;\r
  18 \r
  19 import com.ibm.nosql.json.api.BasicDBObject;\r
  20 import com.ibm.nosql.json.api.DB;\r
  21 import com.ibm.nosql.json.api.DBObject;\r
  22 \r
  23 import user.commons.CalendarUtils;\r
  24 import user.commons.StoreUri;\r
  25 import user.commons.nosql.NoSQLUtils;\r
  26 import user.commons.octopus.IOctopusAPI;\r
  27 import user.commons.octopus.OctopusAPI;\r
  28 import user.commons.remotestore.FtpDirectoryLister;\r
  29 import user.commons.remotestore.RemoteStoreProtocol;\r
  30 import user.jobengine.db.IItemManager;\r
  31 import user.jobengine.server.steps.EscortFiles;\r
  32 import user.jobengine.server.steps.FileArchive;\r
  33 import user.jobengine.server.steps.RundownArchive;\r
  34 import user.jobengine.server.steps.StoryArchive;\r
  35 \r
  36 public class RescueNEXIOMaterials {\r
  37         private static final String SCHEDULED_FORMAT = "yyyy.MM.dd HH:mm";\r
  38         private static final Logger logger = LogManager.getLogger();\r
  39         private static final String UTF_8 = "utf-8";\r
  40         private static final String JSON_EXT = ".json";\r
  41         private static final String XML_EXT = ".xml";\r
  42         private static final String DURATION = "duration";\r
  43         private static final String MXFEXT = ".MXF";\r
  44         private static final String NEXIOCLIPS = "nexioclips";\r
  45         private static final String LONGNAMEID = "longnameid";\r
  46         private static final String ID = "id";\r
  47         private static final String MEDIATYPE = "Hír bejátszó";\r
  48         private OctopusAPI octopusAPI;\r
  49 \r
  50         private DB db;\r
  51         private FTPClient sourceFtp;\r
  52         private FTPClient targetFtp;\r
  53         private StoreUri sourceUri;\r
  54         private StoreUri targetUri;\r
  55         private int nexioKillDateDays;\r
  56         private String nexioAgency;\r
  57         private Marker systemMarker;\r
  58         private List<String> transferredFileNames = null;\r
  59         private boolean demo = false;\r
  60 \r
  61         public void copy(RundownArchive rundownArchive) throws Exception {\r
  62                 for (StoryArchive storyArchive : rundownArchive.getStoryArchives()) {\r
  63                         for (FileArchive fileArchive : storyArchive.getFileArchives()) {\r
  64                                 try {\r
  65                                         copyFile(fileArchive, rundownArchive, storyArchive);\r
  66                                 } catch (Exception e) {\r
  67                                         logger.error(systemMarker, "A '{}' clip archiválása sikertelen. A rendszer üzenete: {}", fileArchive.getFileName(), e.getMessage());\r
  68                                         throw e;\r
  69                                 }\r
  70                         }\r
  71                 }\r
  72         }\r
  73 \r
  74         private void copyFile(FileArchive fileArchive, RundownArchive rundownArchive, StoryArchive storyArchive) throws Exception {\r
  75                 String origFileName = fileArchive.getFileName();\r
  76                 String fileName = String.format("%s-%s", origFileName, rundownArchive.getItemHouseId());\r
  77                 String videoFileName = fileName + MXFEXT;\r
  78 \r
  79                 if (transferredFileNames == null)\r
  80                         transferredFileNames = new ArrayList<>();\r
  81 \r
  82                 //A mar letezo mozikat nem archivaljuk le ujra, csak a metaadatot\r
  83                 long existingMediaId = 0;\r
  84                 if (!transferredFileNames.contains(origFileName)) {\r
  85                         transferredFileNames.add(origFileName);\r
  86                         if (!demo)\r
  87                                 transferFile(origFileName + MXFEXT, videoFileName);\r
  88                         logger.info(systemMarker, "A '{}' file archiválásra felkészítése sikeres volt.", origFileName);\r
  89                 }\r
  90 \r
  91                 if (!demo) {\r
  92                         BasicDBObject metadata = createMetadata(rundownArchive, storyArchive, fileArchive, existingMediaId);\r
  93                         transferMetadata(videoFileName, metadata);\r
  94                         createSourceKillDateFile(rundownArchive, origFileName);\r
  95                 }\r
  96         }\r
  97 \r
  98         private BasicDBObject createMetadata(RundownArchive rundownArchive, StoryArchive storyArchive, FileArchive fileArchive, long existingMediaId) {\r
  99                 BasicDBObject result = new BasicDBObject();\r
 100                 result.put("itemHouseId", rundownArchive.getItemHouseId());\r
 101                 result.put("itemTitle", rundownArchive.getItemTitle());\r
 102                 result.put("itemDescription", rundownArchive.getItemDesc());\r
 103                 result.put("userName", "mediacube");\r
 104                 result.put("mediaHouseId", storyArchive.getMediaHouseId());\r
 105                 result.put("mediaTitle", storyArchive.getMediaTitle());\r
 106                 result.put("mediaDescription", storyArchive.getMediaDesc());\r
 107                 result.put("mediaType", MEDIATYPE);\r
 108                 result.put("duration", fileArchive.getDuration());\r
 109                 result.put("existingMediaId", existingMediaId);\r
 110                 return result;\r
 111         }\r
 112 \r
 113         private void createSourceKillDateFile(RundownArchive rundownArchive, String fileName) throws Exception {\r
 114                 logger.info("Create killdate/agency for {}", fileName);\r
 115                 OutputStream outStream = null;\r
 116                 try {\r
 117                         sourceFtp = ((FtpDirectoryLister) sourceUri.getLister()).connect();\r
 118                         Calendar killDate = CalendarUtils.createCalendar(rundownArchive.getScheduleDate());\r
 119                         killDate.add(Calendar.DAY_OF_YEAR, nexioKillDateDays);\r
 120                         byte[] killDateFile = EscortFiles.createNEXIOKillDateFile(fileName, killDate.getTime(), null, nexioAgency);\r
 121                         outStream = sourceFtp.storeFileStream(fileName + XML_EXT);\r
 122                         if (outStream == null) {\r
 123                                 throw new NullPointerException("Can not open: " + fileName + XML_EXT + " Reply:" + sourceFtp.getReplyString());\r
 124                         }\r
 125                         outStream.write(killDateFile);\r
 126                         outStream.flush();\r
 127                 } catch (Exception e) {\r
 128                         logger.catching(e);\r
 129                         throw e;\r
 130                 } finally {\r
 131                         if (outStream != null)\r
 132                                 outStream.close();\r
 133                         sourceUri.cleanUp();\r
 134                 }\r
 135         }\r
 136 \r
 137         private FileArchive processMosObject(BasicDBObject rundown, BasicDBObject story, BasicDBObject mosObject, String clipName, long duration) throws Exception {\r
 138                 String mosID = mosObject.getString(IOctopusAPI.OBJ_ID);\r
 139                 if (!mosID.equals(clipName))\r
 140                         return null;\r
 141                 return new FileArchive(mosID, duration);\r
 142         }\r
 143 \r
 144         public RundownArchive processRundow(String clipName, long duration) throws Exception {\r
 145                 octopusAPI = new OctopusAPI();\r
 146                 List<DBObject> rds = octopusAPI.getRundownsByPlaceHolderID(clipName);\r
 147                 if (rds == null || rds.size() == 0)\r
 148                         return null;\r
 149                 BasicDBObject rundown = (BasicDBObject) rds.get(0);\r
 150                 long rundownID = rundown.getLong(ID);\r
 151                 if (!demo)\r
 152                         logger.info("Processing rundown {} {}", rundownID, rundown.getString(IOctopusAPI.NAME));\r
 153                 List<DBObject> stories = octopusAPI.getRundownFullStories(rundownID);\r
 154                 if (stories == null)\r
 155                         return null;\r
 156                 RundownArchive result = new RundownArchive();\r
 157 \r
 158                 long id = NoSQLUtils.asLong(rundown, IOctopusAPI.ID);\r
 159                 if (id == 0)\r
 160                         return null;\r
 161                 String name = NoSQLUtils.asString(NoSQLUtils.asDBObject(rundown, IOctopusAPI.RUNDOWN_TYPE), IOctopusAPI.NAME);\r
 162                 if (StringUtils.isBlank(name))\r
 163                         return null;\r
 164                 String channel = NoSQLUtils.asString(NoSQLUtils.asDBObject(rundown, IOctopusAPI.CHANNEL), IOctopusAPI.NAME);\r
 165                 Date scheduledStart = rundown.getDate(IOctopusAPI.SCHEDULED_START);\r
 166                 if (scheduledStart == null)\r
 167                         return null;\r
 168                 String start = CalendarUtils.toString(CalendarUtils.createCalendar(scheduledStart), SCHEDULED_FORMAT);\r
 169                 result.setScheduleDate(scheduledStart);\r
 170                 result.setItemHouseId(String.valueOf(id));\r
 171                 result.setItemTitle(String.format("%s %s %s", start, name, channel));\r
 172 \r
 173                 for (DBObject s : stories) {\r
 174                         StoryArchive storyArchive = processStory(rundown, s, clipName, duration);\r
 175                         if (storyArchive == null)\r
 176                                 continue;\r
 177                         result.addStoryArchive(storyArchive);\r
 178                         break;\r
 179                 }\r
 180                 return result;\r
 181         }\r
 182 \r
 183         private StoryArchive processStory(BasicDBObject rundown, DBObject s, String clipName, long duration) throws Exception {\r
 184                 BasicDBObject story = (BasicDBObject) s;\r
 185                 String parentStoryID = story.getString(IOctopusAPI.PARENT_STORY_ID);\r
 186                 if (StringUtils.isBlank(parentStoryID)) {\r
 187                         logger.warn("Story parentStoryID is null: {}", story.toPrettyString(null));\r
 188                         return null;\r
 189                 } else\r
 190                         logger.debug("Processing story {}", parentStoryID);\r
 191                 List<BasicDBObject> mosObjects = NoSQLUtils.asList(story, IOctopusAPI.MOS_OBJECTS);\r
 192                 if (mosObjects == null)\r
 193                         return null;\r
 194 \r
 195                 StoryArchive storyArchive = null;\r
 196                 for (BasicDBObject mosObject : mosObjects) {\r
 197                         FileArchive fileArchive = processMosObject(rundown, story, mosObject, clipName, duration);\r
 198                         if (fileArchive == null)\r
 199                                 continue;\r
 200                         if (storyArchive == null) {\r
 201                                 storyArchive = new StoryArchive();\r
 202                                 storyArchive.setMediaHouseId(parentStoryID);\r
 203                                 storyArchive.setMediaTitle(story.getString(IOctopusAPI.NAME));\r
 204                                 storyArchive.setMediaDesc(story.getString(IOctopusAPI.SCRIPT_CONTENT));\r
 205 \r
 206                         }\r
 207                         storyArchive.addFileArchive(fileArchive);\r
 208                 }\r
 209                 return storyArchive;\r
 210 \r
 211         }\r
 212 \r
 213         public void set(int nexioPort, String nexioUserName, String nexioPassword, String archiveFtp, String archiveUserName, String archivePassword,\r
 214                         int nexioKillDateDays, String nexioAgency, IItemManager manager) throws Exception {\r
 215                 db = NoSQLUtils.getNoSQLDB();\r
 216                 if (db == null) {\r
 217                         logger.error(systemMarker, "Az NoSQL adatkezelő réteg nem elérhető.");\r
 218                         throw new NullPointerException("Internal error, missing NoSQL DB reference.");\r
 219                 }\r
 220 \r
 221                 if (manager == null) {\r
 222                         logger.error(systemMarker, "Az adatbáziskezelő réteg nem elérhető.");\r
 223                         throw new NullPointerException("Internal error, missing ItemManager reference.");\r
 224                 }\r
 225                 String nexioHost = System.getProperty("nexio.host");\r
 226                 if (StringUtils.isBlank(nexioHost)) {\r
 227                         logger.error(systemMarker, "A 'nexio.host' rendszer paraméter nem található.");\r
 228                         throw new NullPointerException("System is not configured properly, 'jobengine.selenio.address' startup parameter missing.");\r
 229                 }\r
 230                 this.nexioKillDateDays = nexioKillDateDays;\r
 231                 this.nexioAgency = nexioAgency;\r
 232 \r
 233                 sourceUri = manager.createStoreUri(RemoteStoreProtocol.FTP, nexioHost);\r
 234                 sourceUri.setPortNumber(nexioPort);\r
 235                 sourceUri.setUserName(nexioUserName);\r
 236                 sourceUri.setPassword(nexioPassword);\r
 237                 if (sourceUri == null) {\r
 238                         logger.error(systemMarker, "A forrás nem elérhető.");\r
 239                         throw new NullPointerException("Internal error, missing 'sourceUri'.");\r
 240                 }\r
 241 \r
 242                 targetUri = manager.createStoreUri(new URI(archiveFtp));\r
 243                 targetUri.setUserName(archiveUserName);\r
 244                 targetUri.setPassword(archivePassword);\r
 245                 if (targetUri == null) {\r
 246                         logger.error(systemMarker, "A cél nem elérhető.");\r
 247                         throw new NullPointerException("Internal error, missing 'targetUri'.");\r
 248                 }\r
 249 \r
 250         }\r
 251 \r
 252         private void transferFile(String sourceFileName, String targetFileName) throws Exception {\r
 253                 int reply = 0;\r
 254                 logger.info("Transfer clip {}", sourceFileName);\r
 255                 try {\r
 256                         sourceFtp = ((FtpDirectoryLister) sourceUri.getLister()).connect();\r
 257                         targetFtp = ((FtpDirectoryLister) targetUri.getLister()).connect();\r
 258                         if (!targetFtp.enterRemotePassiveMode())\r
 259                                 throw new Exception("!PASV");\r
 260 \r
 261                         reply = sourceFtp.port(InetAddress.getByName(targetFtp.getPassiveHost()), targetFtp.getPassivePort());\r
 262                         if (!FTPReply.isPositiveCompletion(reply))\r
 263                                 throw new Exception("PORT parancs válasza: " + sourceFtp.getReplyString());\r
 264 \r
 265                         if (!sourceFtp.setFileType(FTP.BINARY_FILE_TYPE))\r
 266                                 throw new Exception("!SOURCE TYPE");\r
 267 \r
 268                         reply = sourceFtp.retr(sourceFileName);\r
 269                         if (!FTPReply.isPositivePreliminary(reply))\r
 270                                 throw new Exception("RETR parancs válasza: " + sourceFtp.getReplyString());\r
 271 \r
 272                         if (!targetFtp.setFileType(FTP.BINARY_FILE_TYPE))\r
 273                                 throw new Exception("!TARGET TYPE");\r
 274 \r
 275                         reply = targetFtp.stor(targetFileName);\r
 276                         if (!FTPReply.isPositivePreliminary(reply))\r
 277                                 throw new Exception("STOR parancs válasza: " + sourceFtp.getReplyString());\r
 278 \r
 279                         while (true) {\r
 280                                 reply = sourceFtp.stat();\r
 281                                 if (!FTPReply.isPositiveCompletion(reply))\r
 282                                         throw new Exception("STAT parancs válasza: " + sourceFtp.getReplyString());\r
 283 \r
 284                                 logger.info("Status: {}", sourceFtp.getReplyString());\r
 285                                 if (reply == 226) {\r
 286                                         break;\r
 287                                 }\r
 288                                 Thread.sleep(1000);\r
 289                         }\r
 290                 } catch (Exception e) {\r
 291                         logger.catching(e);\r
 292                         throw e;\r
 293                 } finally {\r
 294                         sourceUri.cleanUp();\r
 295                         targetUri.cleanUp();\r
 296                 }\r
 297 \r
 298         }\r
 299 \r
 300         private void transferMetadata(String fileName, BasicDBObject metadata) throws Exception {\r
 301                 logger.info("Transfer metadata {}", fileName);\r
 302                 OutputStream outStream = null;\r
 303                 try {\r
 304                         targetFtp = ((FtpDirectoryLister) targetUri.getLister()).connect();\r
 305                         if (!targetFtp.changeWorkingDirectory(EscortFiles.STATUSFOLDER)) {\r
 306                                 targetFtp.makeDirectory(EscortFiles.STATUSFOLDER);\r
 307                                 if (!targetFtp.changeWorkingDirectory(EscortFiles.STATUSFOLDER))\r
 308                                         throw new Exception("!STATUSFOLDER");\r
 309                         }\r
 310 \r
 311                         outStream = targetFtp.storeFileStream(fileName + JSON_EXT);\r
 312                         if (outStream == null) {\r
 313                                 throw new NullPointerException("Can not open: " + fileName + JSON_EXT + " Reply:" + targetFtp.getReplyString());\r
 314                         }\r
 315                         outStream.write(metadata.toString().getBytes(UTF_8));\r
 316                         outStream.flush();\r
 317                         //targetFtp.changeToParentDirectory();\r
 318                 } catch (Exception e) {\r
 319                         logger.catching(e);\r
 320                         throw e;\r
 321                 } finally {\r
 322                         if (outStream != null)\r
 323                                 outStream.close();\r
 324                         targetUri.cleanUp();\r
 325                 }\r
 326         }\r
 327 \r
 328 }\r