import hu.user.mcvodsync.service.data.CompositeSummary;
import hu.user.mcvodsync.service.data.Summary;
import hu.user.mcvodsync.service.in.AssetImportService;
-import hu.user.mcvodsync.service.in.VodXlsProcessor;
+import hu.user.mcvodsync.service.in.XlsProcessor;
import hu.user.mcvodsync.service.out.SyncProcessor;
import hu.user.mcvodsync.service.out.VideoMapper;
import lombok.extern.log4j.Log4j2;
import org.apache.commons.lang3.StringUtils;
import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.beans.factory.annotation.Qualifier;
import java.io.IOException;
import java.nio.file.Files;
@Log4j2
public class AssetImportExportBase {
@Autowired
- private VodXlsProcessor vodXlsProcessor;
+ @Qualifier("fastXLSProcessor")
+ private XlsProcessor xlsProcessor;
@Autowired
private AssetRepository assetRepository;
CompositeSummary summary;
try {
Path input = Paths.get(xlsFile);
- summary = vodXlsProcessor.process(input.getFileName().toString(), Files.readAllBytes(input));
+ summary = xlsProcessor.process(input.getFileName().toString(), Files.readAllBytes(input));
} catch (IOException e) {
log.error(e);
throw new RuntimeException(e);
--- /dev/null
+package hu.user.mcvodsync.service.in;
+
+import hu.user.mcvodsync.db.Asset;
+import hu.user.mcvodsync.service.data.CompositeSummary;
+import lombok.extern.log4j.Log4j2;
+import org.apache.commons.lang3.StringUtils;
+import org.apache.poi.hssf.usermodel.HSSFWorkbook;
+import org.apache.poi.ss.usermodel.Cell;
+import org.apache.poi.ss.usermodel.Row;
+import org.apache.poi.ss.usermodel.Sheet;
+import org.apache.poi.ss.usermodel.Workbook;
+import org.apache.poi.xssf.usermodel.XSSFWorkbook;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.beans.factory.annotation.Qualifier;
+import org.springframework.stereotype.Component;
+import org.springframework.transaction.annotation.Transactional;
+
+import java.io.ByteArrayInputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.time.Instant;
+import java.time.LocalDate;
+import java.util.*;
+import java.util.stream.Collectors;
+import java.util.stream.StreamSupport;
+
+@Log4j2
+@Component
+@Qualifier("poiXLSProcessor")
+public class VodXlsPOIProcessor implements XlsProcessor {
+ private List<Cell> headers;
+
+ @Autowired
+ private AssetMapper assetMapper;
+
+ @Autowired
+ private AssetImportService assetImportService;
+
+
+ @Transactional
+ public CompositeSummary process(String fileName, byte[] xlsData) {
+ CompositeSummary summary = CompositeSummary.builder().started(Instant.now()).build();
+ try (InputStream is = new ByteArrayInputStream(xlsData);
+ Workbook wb = getWorkbook(fileName, is)) {
+ Sheet sheet = wb.getSheetAt(0);
+
+ Iterator<Row> rowIterator = sheet.iterator();
+ while (rowIterator.hasNext()) {
+ Row row = rowIterator.next();
+
+ if (row.getRowNum() == 0) {
+ assetImportService.prepare(LocalDate.now());
+ headers = getHeaders(row);
+ } else {
+ log.info("Processing {}", row.getRowNum());
+ summary.getVideo().incAll();
+ processRow(row, summary);
+ }
+
+ }
+ assetImportService.processPlaylists(summary);
+
+ } catch (IOException e) {
+ log.error("Excel file reading error from {}. System message: {}", fileName, e.getMessage());
+ }
+ summary.setFinished(Instant.now());
+ log.info(summary);
+ return summary;
+ }
+
+ private Workbook getWorkbook(String fileName, InputStream is) throws IOException {
+ if (fileName.toUpperCase().endsWith("XLS")) {
+ return new HSSFWorkbook(is);
+ } else {
+ return new XSSFWorkbook(is);
+ }
+ }
+
+ private List<Cell> getHeaders(Row row) {
+ Iterator<Cell> cellIterator = row.cellIterator();
+ Iterable<Cell> iterable = () -> cellIterator;
+ return StreamSupport.stream(iterable.spliterator(), false).collect(Collectors.toList());
+ }
+
+ private void processRow(Row row, CompositeSummary summary) {
+ Map<String, String> rowData = new HashMap<>();
+ Iterator<Cell> cellIterator = row.cellIterator();
+ Iterable<Cell> iterable = () -> cellIterator;
+
+ StreamSupport.stream(iterable.spliterator(), false).filter(Objects::nonNull).forEach(c -> {
+ try {
+ String header = headers.get(c.getColumnIndex()).getStringCellValue();
+
+ String data = null;
+ switch (c.getCellType()) {
+ case NUMERIC:
+ data = String.valueOf(c.getNumericCellValue());
+ break;
+ default:
+ data = c.getStringCellValue();
+ break;
+ }
+ rowData.put(header, data);
+ } catch (Exception e) {
+ log.error("Cell error!", e);
+ }
+ });
+
+ Asset asset = null;
+ try {
+ asset = assetMapper.toEntity(rowData);
+ if (isAssetValid(asset)) {
+ assetImportService.processAsset(asset, summary);
+ } else {
+ summary.getVideo().incSkip();
+ }
+ summary.getVideo().incSuccess();
+ } catch (Exception e) {
+ summary.getVideo().incError();
+ log.error("Processing error for {}", asset, e);
+ }
+ }
+
+ private boolean isAssetValid(Asset asset) {
+ if (StringUtils.isBlank(asset.getCatalogId())) {
+ log.warn("Catalog ID not defined for {}", asset);
+ return false;
+ }
+ if (StringUtils.isBlank(asset.getPlaylist())) {
+ log.warn("Playlist not defined for {}", asset);
+ return false;
+ }
+ if (StringUtils.isBlank(asset.getProgLocalTitle())) {
+ log.warn("Prog local title not defined for {}", asset);
+ return false;
+ }
+ if (Objects.isNull(asset.getSunset())) {
+ log.warn("Sunset not defined for {}", asset);
+ return false;
+ }
+ if (Objects.isNull(asset.getSunrise())) {
+ log.warn("Sunrise not defined for {}", asset);
+ return false;
+ }
+ return true;
+ }
+
+}
import hu.user.mcvodsync.service.data.CompositeSummary;
import hu.user.mcvodsync.service.event.ImportCompletedEvent;
import hu.user.mcvodsync.service.event.ImportStartedEvent;
-import hu.user.mcvodsync.service.in.VodXlsProcessor;
+import hu.user.mcvodsync.service.in.XlsProcessor;
import lombok.extern.log4j.Log4j2;
import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.beans.factory.annotation.Qualifier;
import org.springframework.context.ApplicationEventPublisher;
import org.springframework.stereotype.Component;
private UploadFileRepository uploadFileRepository;
@Autowired
- private VodXlsProcessor vodXlsProcessor;
+ @Qualifier("poiXLSProcessor")
+ private XlsProcessor xlsProcessor;
@Override
public void run() {
if (opUploadFile.isPresent()) {
applicationEventPublisher.publishEvent(new ImportStartedEvent(this, opUploadFile.get().getName()));
log.info("ScheduledImport started from {}", opUploadFile.get().getName());
- summary = vodXlsProcessor.process(opUploadFile.get().getName(), opUploadFile.get().getFile());
+ summary = xlsProcessor.process(opUploadFile.get().getName(), opUploadFile.get().getFile());
}
} catch (Exception e) {
log.error("ScheduledImport error!", e);
https://git.useribm.hu / mediacube.git / commitdiff