git.useribm.hu - mediacube.git/blob

   1 package user.jobengine.server.steps;\r
   2 \r
   3 import java.nio.file.DirectoryStream;\r
   4 import java.nio.file.Files;\r
   5 import java.nio.file.Path;\r
   6 import java.nio.file.Paths;\r
   7 import java.util.ArrayList;\r
   8 import java.util.Arrays;\r
   9 import java.util.Collection;\r
  10 import java.util.LinkedHashSet;\r
  11 import java.util.List;\r
  12 \r
  13 import org.apache.commons.lang.StringUtils;\r
  14 import org.apache.commons.net.ftp.FTPClient;\r
  15 import org.apache.logging.log4j.LogManager;\r
  16 import org.apache.logging.log4j.Logger;\r
  17 \r
  18 import com.ibm.nosql.json.api.BasicDBList;\r
  19 \r
  20 import user.commons.DownloadableMedia;\r
  21 import user.commons.StoreUri;\r
  22 import user.commons.remotestore.FtpDirectoryLister;\r
  23 import user.commons.remotestore.RemoteStoreProtocol;\r
  24 import user.jobengine.db.Media;\r
  25 import user.jobengine.server.steps.shared.EscortFiles;\r
  26 import user.jobengine.server.steps.shared.PBQuery;\r
  27 import user.jobengine.server.steps.shared.PBQuery.MMMedia;\r
  28 \r
  29 public class PeableBeachMissingMaterialCheckerStep extends JobStep {\r
  30         private static final Logger logger = LogManager.getLogger();\r
  31         private static final String appendExtension = ".mxf";\r
  32 \r
  33         @StepEntry\r
  34         public Object[] execute(String escortStoreName, BasicDBList lookupStoreNames, String targetStoreName,\r
  35                         String targetProtocol, String primaryEndPoint, String primaryUserName, String primaryPassword,\r
  36                         String secondaryEndPoint, String secondaryUserName, String secondaryPassword, int rangeForwardHours)\r
  37                         throws Exception {\r
  38                 StoreUri escortStoreUri = null;\r
  39                 try {\r
  40                         getJobRuntime().setRelated(targetStoreName + " restore");\r
  41 \r
  42                         escortStoreUri = getManager().getStoreUri(escortStoreName, RemoteStoreProtocol.LOCAL);\r
  43                         StoreUri sourceStoreUri = getManager().getStoreUri("TSM", RemoteStoreProtocol.TSM);\r
  44                         StoreUri targetStoreUri = getManager().getStoreUri(targetStoreName,\r
  45                                         Enum.valueOf(RemoteStoreProtocol.class, targetProtocol));\r
  46 \r
  47                         LinkedHashSet<String> poolContent = new LinkedHashSet<>();\r
  48 \r
  49                         int progress = 0;\r
  50                         int count = lookupStoreNames.size();\r
  51                         int processed = 0;\r
  52 \r
  53                         for (Object lookup : lookupStoreNames) {\r
  54                                 if (getJobRuntime().isWaitingCancel()) {\r
  55                                         cancel();\r
  56                                         return null;\r
  57                                 }\r
  58                                 String lookupStoreName = String.valueOf(lookup);\r
  59 \r
  60                                 StoreUri lookupStoreUri = getManager().getStoreUri(lookupStoreName,\r
  61                                                 Enum.valueOf(RemoteStoreProtocol.class, targetProtocol));\r
  62                                 logger.info(getMarker(), "Checking pool {}", lookupStoreName);\r
  63                                 List<String> lookupContent = getPoolContent(lookupStoreName, lookupStoreUri);\r
  64                                 poolContent.addAll(lookupContent);\r
  65                                 processed++;\r
  66                                 progress = processed * 50 / count;\r
  67                                 setProgress(progress);\r
  68                         }\r
  69                         /*\r
  70                          * if (1 == 1) {\r
  71                          * \r
  72                          * logger.info(getMarker(), "Kikapcsolva"); return null; }\r
  73                          */\r
  74 \r
  75                         PBQuery query = new PBQuery();\r
  76                         query.init(primaryEndPoint, primaryUserName, primaryPassword, secondaryEndPoint, secondaryUserName,\r
  77                                         secondaryPassword, rangeForwardHours);\r
  78                         Collection<MMMedia> medias = query.getPossibelMissingMaterialNames(new ArrayList<>(poolContent));\r
  79                         logger.info(getMarker(), "API returned {} items", medias == null ? 0 : medias.size());\r
  80 \r
  81                         if (medias != null && medias.size() > 0) {\r
  82                                 count = medias.size();\r
  83                                 for (MMMedia media : medias) {\r
  84                                         logger.info(getMarker(), "API response contains {}", media.getName());\r
  85                                 }\r
  86 \r
  87                                 for (MMMedia media : medias) {\r
  88                                         if (getJobRuntime().isWaitingCancel()) {\r
  89                                                 cancel();\r
  90                                                 break;\r
  91                                         }\r
  92                                         processRecord(media, sourceStoreUri, targetStoreName, targetStoreUri, escortStoreUri);\r
  93                                         processed++;\r
  94                                         progress = 50 + (processed * 50 / count);\r
  95                                         setProgress(progress);\r
  96                                 }\r
  97 \r
  98                         }\r
  99                         setProgress(100);\r
 100                 } catch (Exception e) {\r
 101                         logger.error(getSessionMarker(), e.getMessage());\r
 102                         throw e;\r
 103                 } finally {\r
 104                         if (escortStoreUri != null)\r
 105                                 escortStoreUri.cleanUp();\r
 106                 }\r
 107 \r
 108                 return null;\r
 109         }\r
 110 \r
 111         private List<String> getPoolContent(String storeName, StoreUri targetStoreUri) throws Exception {\r
 112                 List<String> poolContent = new ArrayList<>();\r
 113 \r
 114                 FtpDirectoryLister lister = (FtpDirectoryLister) targetStoreUri.getLister();\r
 115                 FTPClient client = lister.connect();\r
 116                 logger.info(getMarker(), "Working dir is {}", client.printWorkingDirectory());\r
 117                 String[] listNames = client.listNames();\r
 118                 targetStoreUri.cleanUp();\r
 119 \r
 120                 if (listNames != null) {\r
 121                         List<String> remoteFileNames = Arrays.asList(listNames);\r
 122                         logger.info(getMarker(), "Remote file list size :{}", remoteFileNames.size());\r
 123                         for (String name : remoteFileNames) {\r
 124                                 try {\r
 125                                         if (name.toLowerCase().endsWith(".mxf")) {\r
 126                                                 int pos = name.lastIndexOf(".");\r
 127                                                 if (pos > -1) {\r
 128                                                         name = name.substring(0, pos);\r
 129                                                         // logger.info(getMarker(), "Adding {}", name);\r
 130                                                         poolContent.add(name);\r
 131                                                 }\r
 132                                         }\r
 133                                 } catch (Exception e) {\r
 134                                         logger.error(e.getMessage());\r
 135                                 }\r
 136 \r
 137                         }\r
 138                 }\r
 139 \r
 140                 // List<RemoteFile> remoteFiles = targetStoreUri.getRemoteFiles();\r
 141                 // if (remoteFiles != null) {\r
 142                 // logger.info(getMarker(), "Remote file list size :{}", remoteFiles.size());\r
 143                 // for (RemoteFile rf : remoteFiles) {\r
 144                 // try {\r
 145                 // String name = rf.getName();\r
 146                 // if (name.toLowerCase().endsWith(".mxf")) {\r
 147                 // int pos = name.lastIndexOf(".");\r
 148                 // if (pos > -1) {\r
 149                 // name = name.substring(0, pos);\r
 150                 // //logger.info(getMarker(), "Adding {}", name);\r
 151                 // poolContent.add(name);\r
 152                 // }\r
 153                 // }\r
 154                 // } catch (Exception e) {\r
 155                 // logger.error(e.getMessage());\r
 156                 // }\r
 157                 //\r
 158                 // }\r
 159                 // }\r
 160                 logger.info(getMarker(), "Pool {} contains {} items", storeName, poolContent == null ? 0 : poolContent.size());\r
 161                 return poolContent;\r
 162         }\r
 163 \r
 164         // csak visszaterunk\r
 165         private String normalizeName(String mediaName) {\r
 166                 String result = mediaName;\r
 167                 if (StringUtils.isNotBlank(appendExtension))\r
 168                         result += appendExtension;\r
 169                 // if (StringUtils.isNotBlank(mediaName)) {\r
 170                 // result = mediaName.trim().toLowerCase();\r
 171                 // }\r
 172                 return result;\r
 173         }\r
 174 \r
 175         private void processRecord(MMMedia mmedia, StoreUri sourceStoreUri, String targetStoreName, StoreUri targetStoreUri,\r
 176                         StoreUri escortStoreUri) {\r
 177                 // TODO kisbetu/nagybetu problema kezelese\r
 178 \r
 179                 Media media = null;\r
 180                 // duplikatum miatt\r
 181                 try {\r
 182                         media = getManager().getMedia(mmedia.getName());\r
 183                 } catch (Exception e) {\r
 184                         logger.error(getMarker(), "Can't identify missing material {}. System message is: {}", mmedia.getName(),\r
 185                                         e.getMessage());\r
 186                         return;\r
 187                 }\r
 188 \r
 189                 String fileName = normalizeName(mmedia.getName());\r
 190 \r
 191                 if (media == null) {\r
 192                         logger.warn(getMarker(), "File {} not archived yet", mmedia.getName());\r
 193                         return;\r
 194                 }\r
 195 \r
 196                 DownloadableMedia downloadable = DownloadableMedia.create(mmedia.getName(), fileName, media.getModified(),\r
 197                                 media.getCreated(), media.getLength(), 0L, sourceStoreUri.getId(), targetStoreUri.getId(),\r
 198                                 media.getId());\r
 199                 downloadable.put("priority", 50);\r
 200                 String usage = mmedia.getUsage();\r
 201                 usage = usage.replace("-", "").replace(":", "");\r
 202                 String escortFileName = usage + "." + targetStoreName + "." + fileName;\r
 203                 Path outputPath = null;\r
 204                 try {\r
 205                         outputPath = Paths.get(escortStoreUri.toString(true));\r
 206                         boolean exists = statusFileExists(outputPath, fileName);\r
 207                         if (exists)\r
 208                                 return;\r
 209                         EscortFiles.createMetadata(outputPath.toString(), escortFileName, downloadable.toPrettyString(""));\r
 210                         logger.info(getMarker(), "Status file created {}", escortFileName);\r
 211                 } catch (Exception e) {\r
 212                         logger.error(getSessionMarker(), "Can't create status file {}",\r
 213                                         Paths.get(outputPath.toString(), escortFileName));\r
 214                 }\r
 215 \r
 216         }\r
 217 \r
 218         private boolean statusFileExists(Path outputPath, String fileName) {\r
 219                 boolean exists = false;\r
 220                 Path statusPath = Paths.get(outputPath.toString(), EscortFiles.STATUSFOLDER);\r
 221                 try (DirectoryStream<Path> p = Files.newDirectoryStream(statusPath, "*" + fileName + EscortFiles.DOT_JSON)) {\r
 222                         if (p.iterator().hasNext()) {\r
 223                                 logger.info(getMarker(), "Status file for {} already exists", fileName);\r
 224                                 exists = true;\r
 225                         }\r
 226 \r
 227                 } catch (Exception e1) {\r
 228                         logger.error(getSessionMarker(), e1.getMessage());\r
 229                 }\r
 230                 return exists;\r
 231         }\r
 232 \r
 233 }\r