git.useribm.hu - mediacube.git/blob

   1 package user.jobengine.server.steps;\r
   2 \r
   3 import java.nio.file.DirectoryStream;\r
   4 import java.nio.file.Files;\r
   5 import java.nio.file.Path;\r
   6 import java.nio.file.Paths;\r
   7 import java.util.ArrayList;\r
   8 import java.util.Collection;\r
   9 import java.util.LinkedHashSet;\r
  10 import java.util.List;\r
  11 \r
  12 import org.apache.commons.lang.StringUtils;\r
  13 import org.apache.logging.log4j.LogManager;\r
  14 import org.apache.logging.log4j.Logger;\r
  15 \r
  16 import com.ibm.nosql.json.api.BasicDBList;\r
  17 \r
  18 import user.commons.DownloadableMedia;\r
  19 import user.commons.RemoteFile;\r
  20 import user.commons.StoreUri;\r
  21 import user.commons.remotestore.RemoteStoreProtocol;\r
  22 import user.jobengine.db.Media;\r
  23 import user.jobengine.server.steps.PBQuery.MMMedia;\r
  24 \r
  25 public class PeableBeachMissingMaterialCheckerStep extends JobStep {\r
  26         private static final Logger logger = LogManager.getLogger();\r
  27         private static final String appendExtension = System.getProperty("missingmaterial.appendextension", ".mxf");\r
  28 \r
  29         @StepEntry\r
  30         public Object[] execute(String escortStoreName, BasicDBList lookupStoreNames, String targetStoreName, String targetProtocol, String primaryEndPoint,\r
  31                         String primaryUserName, String primaryPassword, String secondaryEndPoint, String secondaryUserName, String secondaryPassword, int rangeForwardHours)\r
  32                         throws Exception {\r
  33                 StoreUri escortStoreUri = null;\r
  34                 try {\r
  35                         getJobRuntime().setRelated(targetStoreName + " restore");\r
  36 \r
  37                         escortStoreUri = getManager().getStoreUri(escortStoreName, RemoteStoreProtocol.LOCAL);\r
  38                         StoreUri sourceStoreUri = getManager().getStoreUri("TSM", RemoteStoreProtocol.TSM);\r
  39                         StoreUri targetStoreUri = getManager().getStoreUri(targetStoreName, Enum.valueOf(RemoteStoreProtocol.class, targetProtocol));\r
  40 \r
  41                         LinkedHashSet<String> poolContent = new LinkedHashSet<>();\r
  42 \r
  43                         int progress = 0;\r
  44                         int count = lookupStoreNames.size();\r
  45                         int processed = 0;\r
  46 \r
  47                         for (Object lookup : lookupStoreNames) {\r
  48                                 if (getJobRuntime().isWaitingCancel()) {\r
  49                                         cancel();\r
  50                                         return null;\r
  51                                 }\r
  52                                 String lookupStoreName = String.valueOf(lookup);\r
  53 \r
  54                                 StoreUri lookupStoreUri = getManager().getStoreUri(lookupStoreName, RemoteStoreProtocol.LOCAL);\r
  55                                 logger.info(getMarker(), "Checking pool {}", lookupStoreName);\r
  56                                 List<String> lookupContent = getPoolContent(lookupStoreName, lookupStoreUri);\r
  57                                 poolContent.addAll(lookupContent);\r
  58                                 processed++;\r
  59                                 progress = processed * 50 / count;\r
  60                                 setProgress(progress);\r
  61                         }\r
  62                         /*\r
  63                                                 if (1 == 1) {\r
  64 \r
  65                                                         logger.info(getMarker(), "Kikapcsolva");\r
  66                                                         return null;\r
  67                                                 }\r
  68                         */\r
  69 \r
  70                         PBQuery query = new PBQuery();\r
  71                         query.init(primaryEndPoint, primaryUserName, primaryPassword, secondaryEndPoint, secondaryUserName, secondaryPassword, rangeForwardHours);\r
  72                         Collection<MMMedia> medias = query.getPossibelMissingMaterialNames(new ArrayList<>(poolContent));\r
  73                         logger.info(getMarker(), "API returned {} items", medias == null ? 0 : medias.size());\r
  74 \r
  75                         if (medias != null && medias.size() > 0) {\r
  76                                 count = medias.size();\r
  77 \r
  78                                 for (MMMedia media : medias) {\r
  79                                         logger.info(getMarker(), "API response contains {}", media.getName());\r
  80                                 }\r
  81 \r
  82                                 for (MMMedia media : medias) {\r
  83                                         if (getJobRuntime().isWaitingCancel()) {\r
  84                                                 cancel();\r
  85                                                 break;\r
  86                                         }\r
  87                                         processRecord(media, sourceStoreUri, targetStoreName, targetStoreUri, escortStoreUri);\r
  88                                         processed++;\r
  89                                         progress = 50 + (processed * 50 / count);\r
  90                                         setProgress(progress);\r
  91                                 }\r
  92 \r
  93                         }\r
  94                         setProgress(100);\r
  95                 } catch (Exception e) {\r
  96                         logger.error(getSessionMarker(), e.getMessage());\r
  97                         throw e;\r
  98                 } finally {\r
  99                         if (escortStoreUri != null)\r
 100                                 escortStoreUri.cleanUp();\r
 101                 }\r
 102 \r
 103                 return null;\r
 104         }\r
 105 \r
 106         private List<String> getPoolContent(String storeName, StoreUri targetStoreUri) throws Exception {\r
 107                 List<String> poolContent = new ArrayList<>();\r
 108                 List<RemoteFile> remoteFiles = targetStoreUri.getRemoteFiles();\r
 109                 if (remoteFiles != null) {\r
 110                         for (RemoteFile rf : remoteFiles) {\r
 111                                 try {\r
 112                                         String name = rf.getName();\r
 113                                         int pos = name.lastIndexOf(".");\r
 114                                         if (pos > -1) {\r
 115                                                 name = name.substring(0, pos);\r
 116                                                 //logger.info(getMarker(), "Adding {}", name);\r
 117                                                 poolContent.add(name);\r
 118                                         }\r
 119                                 } catch (Exception e) {\r
 120                                         logger.error(e.getMessage());\r
 121                                 }\r
 122 \r
 123                         }\r
 124                 }\r
 125                 logger.info(getMarker(), "Pool {} contains {} items", storeName, poolContent == null ? 0 : poolContent.size());\r
 126                 return poolContent;\r
 127         }\r
 128 \r
 129         //csak visszaterunk\r
 130         private String normalizeName(String mediaName) {\r
 131                 String result = mediaName;\r
 132                 if (StringUtils.isNotBlank(appendExtension))\r
 133                         result += appendExtension;\r
 134                 //              if (StringUtils.isNotBlank(mediaName)) {\r
 135                 //                      result = mediaName.trim().toLowerCase();\r
 136                 //              }\r
 137                 return result;\r
 138         }\r
 139 \r
 140         private void processRecord(MMMedia mmedia, StoreUri sourceStoreUri, String targetStoreName, StoreUri targetStoreUri, StoreUri escortStoreUri) {\r
 141                 //TODO kisbetu/nagybetu problema kezelese\r
 142 \r
 143                 Media media = null;\r
 144                 //duplikatum miatt\r
 145                 try {\r
 146                         media = getManager().getMedia(mmedia.getName());\r
 147                 } catch (Exception e) {\r
 148                         logger.error(getMarker(), "Can't identify missing material {}. System message is: {}", mmedia.getName(), e.getMessage());\r
 149                         return;\r
 150                 }\r
 151 \r
 152                 String fileName = normalizeName(mmedia.getName());\r
 153 \r
 154                 if (media == null) {\r
 155                         logger.error(getMarker(), "File {} not archived yet", mmedia.getName());\r
 156                         return;\r
 157                 }\r
 158 \r
 159                 DownloadableMedia downloadable = DownloadableMedia.create(mmedia.getName(), fileName, media.getModified(), media.getCreated(), media.getLength(), 0L,\r
 160                                 sourceStoreUri.getId(), targetStoreUri.getId(), media.getId());\r
 161                 downloadable.put("priority", 50);\r
 162                 String usage = mmedia.getUsage();\r
 163                 usage = usage.replace("-", "").replace(":", "");\r
 164                 String escortFileName = usage + "." + targetStoreName + "." + fileName;\r
 165                 Path outputPath = null;\r
 166                 try {\r
 167                         outputPath = Paths.get(escortStoreUri.toString(true));\r
 168                         boolean exsists = statusFileExists(outputPath, fileName);\r
 169                         if (exsists)\r
 170                                 return;\r
 171                         EscortFiles.createMetadata(outputPath.toString(), escortFileName, downloadable.toPrettyString(""));\r
 172                         logger.info(getMarker(), "Status file created {}", escortFileName);\r
 173                 } catch (Exception e) {\r
 174                         logger.error(getSessionMarker(), "Can't create status file {}", Paths.get(outputPath.toString(), escortFileName));\r
 175                 }\r
 176 \r
 177         }\r
 178 \r
 179         private boolean statusFileExists(Path outputPath, String fileName) {\r
 180                 boolean exsists = false;\r
 181                 Path statusPath = Paths.get(outputPath.toString(), EscortFiles.STATUSFOLDER);\r
 182                 try (DirectoryStream<Path> p = Files.newDirectoryStream(statusPath, "*" + fileName + EscortFiles.DOT_JSON)) {\r
 183                         if (p.iterator().hasNext()) {\r
 184                                 logger.info(getMarker(), "Status file for {} already exists", fileName);\r
 185                                 exsists = true;\r
 186                         }\r
 187 \r
 188                 } catch (Exception e1) {\r
 189                         logger.error(getSessionMarker(), e1.getMessage());\r
 190                 }\r
 191                 return exsists;\r
 192         }\r
 193 \r
 194 }\r