From: elgekko Date: Mon, 25 Sep 2023 21:29:09 +0000 (+0200) Subject: XLS import with sync data X-Git-Url: http://git.useribm.hu/?a=commitdiff_plain;h=e262f3648077e4d72c51a3b6077b73e93eb439a9;p=mediacube.git XLS import with sync data --- diff --git a/mc-vod-sync/QUESTIONS.md b/mc-vod-sync/QUESTIONS.md new file mode 100644 index 00000000..08494491 --- /dev/null +++ b/mc-vod-sync/QUESTIONS.md @@ -0,0 +1,4 @@ +* Ha hiba történik az import vagy a szinkronizálás során a teljes fájl tartalmat el kell vetni (rollback), vagy ami jó + az frissüljön be? +* Ha hibás egy sor a táblázatban, akkor a teljes importot el kelll dobni, vagy csak a sort kell kihagyni? pl. üres + hubinfo vagy episodetitle \ No newline at end of file diff --git a/mc-vod-sync/mc-vod-sync-app/src/main/java/hu/user/mcvodsync/VodSyncEntry.java b/mc-vod-sync/mc-vod-sync-app/src/main/java/hu/user/mcvodsync/VodSyncEntry.java index 9093e9bf..9bf7e9dd 100644 --- a/mc-vod-sync/mc-vod-sync-app/src/main/java/hu/user/mcvodsync/VodSyncEntry.java +++ b/mc-vod-sync/mc-vod-sync-app/src/main/java/hu/user/mcvodsync/VodSyncEntry.java @@ -6,15 +6,11 @@ package hu.user.mcvodsync; import lombok.extern.log4j.Log4j2; -import org.apache.logging.log4j.LogManager; -import org.apache.logging.log4j.Logger; import org.springframework.boot.SpringApplication; import org.springframework.boot.autoconfigure.SpringBootApplication; import org.springframework.boot.builder.SpringApplicationBuilder; import org.springframework.boot.web.servlet.support.SpringBootServletInitializer; import org.springframework.context.ApplicationContext; -import org.springframework.context.ApplicationListener; -import org.springframework.context.event.ContextRefreshedEvent; import org.springframework.core.SpringVersion; import org.springframework.data.jpa.repository.config.EnableJpaRepositories; import org.springframework.web.servlet.config.annotation.EnableWebMvc; @@ -23,6 +19,7 @@ import org.springframework.web.servlet.config.annotation.EnableWebMvc; @Log4j2 @EnableJpaRepositories("hu.user.mcvodsync.db.repository") @SpringBootApplication(scanBasePackages = {"hu.user.mcvodsync"}) + public class VodSyncEntry extends SpringBootServletInitializer { diff --git a/mc-vod-sync/mc-vod-sync-app/src/main/resources/application-dev.yaml b/mc-vod-sync/mc-vod-sync-app/src/main/resources/application-dev.yaml index ce99c570..dcc1edac 100644 --- a/mc-vod-sync/mc-vod-sync-app/src/main/resources/application-dev.yaml +++ b/mc-vod-sync/mc-vod-sync-app/src/main/resources/application-dev.yaml @@ -21,6 +21,7 @@ spring: datasource: type: com.zaxxer.hikari.HikariDataSource url: jdbc:db2://localhost:50000/vodsync + retrieveMessagesFromServerOnGetMessage: true username: db2admin password: password camunda.bpm: diff --git a/mc-vod-sync/mc-vod-sync-app/src/test/java/hu/user/mcvodsync/VodXlsProcessorIT.java b/mc-vod-sync/mc-vod-sync-app/src/test/java/hu/user/mcvodsync/VodXlsProcessorIT.java index 89476882..0baa47bb 100644 --- a/mc-vod-sync/mc-vod-sync-app/src/test/java/hu/user/mcvodsync/VodXlsProcessorIT.java +++ b/mc-vod-sync/mc-vod-sync-app/src/test/java/hu/user/mcvodsync/VodXlsProcessorIT.java @@ -5,7 +5,6 @@ package hu.user.mcvodsync; -import hu.user.mcvodsync.db.UploadFile; import hu.user.mcvodsync.service.xls.VodXlsProcessor; import lombok.extern.log4j.Log4j2; import org.junit.Test; @@ -32,13 +31,9 @@ public class VodXlsProcessorIT { VodXlsProcessor vodXlsProcessor; @Test - public void listProjects() throws IOException { - Path xlsFile = Paths.get("src", "test", "resources", "AMC_Selekt_jogositott_tartalmak_20230906.xlsx"); - log.info(xlsFile.toAbsolutePath()); - UploadFile uploadFile = UploadFile.builder() - .file(Files.readAllBytes(xlsFile)) - .build(); - vodXlsProcessor.process(uploadFile); + public void processXLS() throws IOException { + Path xlsFile = Paths.get("src/test/resources/AMC_Selekt_jogositott_tartalmak_20230906.xlsx"); + vodXlsProcessor.process(xlsFile.getFileName().toString(), Files.readAllBytes(xlsFile)); } } diff --git a/mc-vod-sync/mc-vod-sync-app/src/test/resources/AMC_Selekt_jogositott_tartalmak_20230906.xlsx b/mc-vod-sync/mc-vod-sync-app/src/test/resources/AMC_Selekt_jogositott_tartalmak_20230906.xlsx index ac221fbb..3c28ea29 100644 Binary files a/mc-vod-sync/mc-vod-sync-app/src/test/resources/AMC_Selekt_jogositott_tartalmak_20230906.xlsx and b/mc-vod-sync/mc-vod-sync-app/src/test/resources/AMC_Selekt_jogositott_tartalmak_20230906.xlsx differ diff --git a/mc-vod-sync/mc-vod-sync-db/migrations/scripts/002_create_initial_schema.sql b/mc-vod-sync/mc-vod-sync-db/migrations/scripts/002_create_system_tables.sql similarity index 56% rename from mc-vod-sync/mc-vod-sync-db/migrations/scripts/002_create_initial_schema.sql rename to mc-vod-sync/mc-vod-sync-db/migrations/scripts/002_create_system_tables.sql index f6f8da99..db08459c 100644 --- a/mc-vod-sync/mc-vod-sync-db/migrations/scripts/002_create_initial_schema.sql +++ b/mc-vod-sync/mc-vod-sync-db/migrations/scripts/002_create_system_tables.sql @@ -1,4 +1,4 @@ --- // create_initial_schema +-- // Create system tables -- Migration SQL that makes the change goes here. CREATE TABLE ASSOCIATE ( id BIGINT NOT NULL GENERATED BY DEFAULT AS IDENTITY, @@ -9,17 +9,11 @@ CREATE TABLE ASSOCIATE ( CONSTRAINT pk_associate PRIMARY KEY (id) ); -CREATE TABLE UPLOAD_FILE ( - id BIGINT NOT NULL GENERATED BY DEFAULT AS IDENTITY, - created TIMESTAMP, - name VARCHAR(255), - size INTEGER NOT NULL, - file BLOB, - CONSTRAINT pk_uploadfile PRIMARY KEY (id) -); - +INSERT INTO ASSOCIATE + (name, login, password, active) +VALUES + ('Test User', 'user', 'password', 1); -- //@UNDO -- SQL to undo the change goes here. -DROP TABLE UPLOAD_FILE; DROP TABLE ASSOCIATE; diff --git a/mc-vod-sync/mc-vod-sync-db/migrations/scripts/003_create_business_tables.sql b/mc-vod-sync/mc-vod-sync-db/migrations/scripts/003_create_business_tables.sql new file mode 100644 index 00000000..d6e5c228 --- /dev/null +++ b/mc-vod-sync/mc-vod-sync-db/migrations/scripts/003_create_business_tables.sql @@ -0,0 +1,43 @@ +-- // Create business tables +-- Migration SQL that makes the change goes here. + +CREATE TABLE UPLOAD_FILE ( + id BIGINT NOT NULL GENERATED BY DEFAULT AS IDENTITY, + created TIMESTAMP, + name VARCHAR(255), + size INTEGER NOT NULL, + file BLOB, + CONSTRAINT pk_uploadfile PRIMARY KEY (id) +); + +CREATE TABLE ASSET ( + episode_id VARCHAR(20) NOT NULL, + season_id BIGINT NOT NULL, + series_id BIGINT NOT NULL, + hub_info VARCHAR(255) NOT NULL, + episode_title VARCHAR(255) NOT NULL, + season_title VARCHAR(255), + series_title VARCHAR(255), + season_nr INTEGER, + episode_nr INTEGER, + production_year INTEGER, + country VARCHAR(1000), + age_rating VARCHAR(10), + CONSTRAINT pk_asset PRIMARY KEY (episode_id, season_id, series_id) +); + +CREATE TABLE ASSET_SYNC ( + execution_id VARCHAR(36) NOT NULL, + episode_id VARCHAR(20) NOT NULL, + season_id BIGINT NOT NULL, + series_id BIGINT NOT NULL, + sync_type VARCHAR(6) NOT NULL, + completed TIMESTAMP, + CONSTRAINT pk_asset_sync PRIMARY KEY (episode_id, season_id, series_id) +); + +-- //@UNDO +-- SQL to undo the change goes here. +DROP TABLE ASSET; +DROP TABLE ASSET_SYNC; +DROP TABLE UPLOAD_FILE; diff --git a/mc-vod-sync/mc-vod-sync-db/migrations/scripts/004_create_test_data.sql b/mc-vod-sync/mc-vod-sync-db/migrations/scripts/004_create_test_data.sql deleted file mode 100644 index 81e01ea4..00000000 --- a/mc-vod-sync/mc-vod-sync-db/migrations/scripts/004_create_test_data.sql +++ /dev/null @@ -1,13 +0,0 @@ --- // create test data --- Migration SQL that makes the change goes here. - -INSERT INTO ASSOCIATE - (name, login, password, active) -VALUES - ('Test User', 'user', 'password', 1); - - --- //@UNDO --- SQL to undo the change goes here. - -DELETE FROM ASSOCIATE; diff --git a/mc-vod-sync/mc-vod-sync-db/src/main/java/hu/user/mcvodsync/db/Asset.java b/mc-vod-sync/mc-vod-sync-db/src/main/java/hu/user/mcvodsync/db/Asset.java new file mode 100644 index 00000000..314e4cd9 --- /dev/null +++ b/mc-vod-sync/mc-vod-sync-db/src/main/java/hu/user/mcvodsync/db/Asset.java @@ -0,0 +1,57 @@ +package hu.user.mcvodsync.db; + +import lombok.*; + +import javax.persistence.Column; +import javax.persistence.Entity; +import javax.persistence.Id; +import javax.persistence.IdClass; +import java.io.Serializable; + +@Getter +@Setter +@Entity +@Builder +@ToString +@NoArgsConstructor +@AllArgsConstructor +@IdClass(AssetId.class) +public class Asset implements Serializable { + @Id + private String episodeId; + + @Id + private Long seasonId; + + @Id + private Long seriesId; + + + @Column(nullable = false) + private String hubInfo; + + + @Column(nullable = false) + private String episodeTitle; + + + private String seasonTitle; + + + private String seriesTitle; + + + private Integer seasonNr; + + + private Integer episodeNr; + + + private Integer productionYear; + + + private String country; + + + private String ageRating; +} diff --git a/mc-vod-sync/mc-vod-sync-db/src/main/java/hu/user/mcvodsync/db/AssetId.java b/mc-vod-sync/mc-vod-sync-db/src/main/java/hu/user/mcvodsync/db/AssetId.java new file mode 100644 index 00000000..5ff15f98 --- /dev/null +++ b/mc-vod-sync/mc-vod-sync-db/src/main/java/hu/user/mcvodsync/db/AssetId.java @@ -0,0 +1,25 @@ +package hu.user.mcvodsync.db; + +import lombok.*; + +import javax.persistence.Column; +import java.io.Serializable; + +@Getter +@Setter +@Builder +@ToString +@NoArgsConstructor +@AllArgsConstructor +public class AssetId implements Serializable { + + @Column(nullable = false) + private String episodeId; + + @Column(nullable = false) + private Long seasonId; + + @Column(nullable = false) + private Long seriesId; + +} diff --git a/mc-vod-sync/mc-vod-sync-db/src/main/java/hu/user/mcvodsync/db/AssetSync.java b/mc-vod-sync/mc-vod-sync-db/src/main/java/hu/user/mcvodsync/db/AssetSync.java new file mode 100644 index 00000000..f9fc2a41 --- /dev/null +++ b/mc-vod-sync/mc-vod-sync-db/src/main/java/hu/user/mcvodsync/db/AssetSync.java @@ -0,0 +1,36 @@ +package hu.user.mcvodsync.db; + +import lombok.*; + +import javax.persistence.*; +import java.io.Serializable; +import java.util.Date; + +@Getter +@Setter +@Entity +@Builder +@ToString +@NoArgsConstructor +@AllArgsConstructor +@IdClass(AssetId.class) +public class AssetSync implements Serializable { + + @Column(nullable = false) + private String executionId; + + @Id + private String episodeId; + + @Id + private Long seasonId; + + @Id + private Long seriesId; + + @Enumerated(EnumType.STRING) + @Column(nullable = false) + private SyncType syncType; + + private Date completed; +} diff --git a/mc-vod-sync/mc-vod-sync-db/src/main/java/hu/user/mcvodsync/db/SyncType.java b/mc-vod-sync/mc-vod-sync-db/src/main/java/hu/user/mcvodsync/db/SyncType.java new file mode 100644 index 00000000..1622e91e --- /dev/null +++ b/mc-vod-sync/mc-vod-sync-db/src/main/java/hu/user/mcvodsync/db/SyncType.java @@ -0,0 +1,5 @@ +package hu.user.mcvodsync.db; + +public enum SyncType { + INSERT, UPDATE, DELETE, ERROR +} diff --git a/mc-vod-sync/mc-vod-sync-db/src/main/java/hu/user/mcvodsync/db/UploadFile.java b/mc-vod-sync/mc-vod-sync-db/src/main/java/hu/user/mcvodsync/db/UploadFile.java index 79885a0c..4043e59e 100644 --- a/mc-vod-sync/mc-vod-sync-db/src/main/java/hu/user/mcvodsync/db/UploadFile.java +++ b/mc-vod-sync/mc-vod-sync-db/src/main/java/hu/user/mcvodsync/db/UploadFile.java @@ -19,8 +19,12 @@ public class UploadFile implements Serializable { @Id @GeneratedValue(strategy = GenerationType.IDENTITY) Long id; + Date created; + String name; + int size; + byte[] file; } diff --git a/mc-vod-sync/mc-vod-sync-db/src/main/java/hu/user/mcvodsync/db/repository/AssetRepository.java b/mc-vod-sync/mc-vod-sync-db/src/main/java/hu/user/mcvodsync/db/repository/AssetRepository.java new file mode 100644 index 00000000..6fb2e770 --- /dev/null +++ b/mc-vod-sync/mc-vod-sync-db/src/main/java/hu/user/mcvodsync/db/repository/AssetRepository.java @@ -0,0 +1,8 @@ +package hu.user.mcvodsync.db.repository; + +import hu.user.mcvodsync.db.Asset; +import hu.user.mcvodsync.db.AssetId; +import org.springframework.data.jpa.repository.JpaRepository; + +public interface AssetRepository extends JpaRepository { +} diff --git a/mc-vod-sync/mc-vod-sync-db/src/main/java/hu/user/mcvodsync/db/repository/AssetSyncRepository.java b/mc-vod-sync/mc-vod-sync-db/src/main/java/hu/user/mcvodsync/db/repository/AssetSyncRepository.java new file mode 100644 index 00000000..048fa038 --- /dev/null +++ b/mc-vod-sync/mc-vod-sync-db/src/main/java/hu/user/mcvodsync/db/repository/AssetSyncRepository.java @@ -0,0 +1,8 @@ +package hu.user.mcvodsync.db.repository; + +import hu.user.mcvodsync.db.AssetId; +import hu.user.mcvodsync.db.AssetSync; +import org.springframework.data.jpa.repository.JpaRepository; + +public interface AssetSyncRepository extends JpaRepository { +} diff --git a/mc-vod-sync/mc-vod-sync-db/src/main/resources/fields.md b/mc-vod-sync/mc-vod-sync-db/src/main/resources/fields.md new file mode 100644 index 00000000..d1688ea6 --- /dev/null +++ b/mc-vod-sync/mc-vod-sync-db/src/main/resources/fields.md @@ -0,0 +1,47 @@ +0 CATALOG_ID +1 SEASON_CATALOG_ID +2 SERIES_GROUP_CATALOG_ID +3 HUB_INFO +4 PROG_ORIG_TITLE +5 PROG_LOCAL_TITLE +6 SEASON_ORIG_TITLE +7 SEASON_LOCAL_TITLE +8 SER_GROUP_ORIG_TITLE +9 SERIES_GROUP_LOCAL_TITLE +10 SEASON_NUMBER +11 EPISODE_NR +12 PROG_SYNOPSIS_FULL_ENG +13 PROG_SYNOPSIS_FULL_HUN +14 PROG_SYNOPSIS_SHORT_ENG +15 PROG_SYNOPSIS_SHORT_HUN +16 SEASON_SYNOPSIS_FULL_ENG +17 SEASON_SYNOPSIS_FULL_HUN +18 SEASON_SYNOPSIS_SHORT_ENG +19 SEASON_SYNOPSIS_SHORT_HUN +20 SERIES_GROUP_SYNOPSIS_FULL_ENG +21 SERIES_GROUP_SYNOPSIS_FULL_HUN +22 SERIES_GROUP_SYNOPSIS_SHORT_ENG +23 SERIES_GROUP_SYNOPSIS_SHORT_HUN +24 ORIGINAL_LANGUAGE +25 CONTENT_TYPE +26 CONTENT_SOURCE +27 AGE_RATING +28 PG_SEX +29 PG_LANGUAGE +30 PG_VIOLENCE +31 PG_SUBSTANCES +32 PG_FEAR +33 PG_RELIGION +34 COUNTRY_OF_ORIGIN +35 PRODUCTION_YEAR +36 DIRECTOR +37 X_CAST +38 SERIES_GROUP_GENRE_1 +39 SERIES_SEASON_GENRE_1 +40 PROG_GENRE_1 +41 SERIES_GROUP_GENRE_2 +42 SERIES_SEASON_GENRE_2 +43 PROG_GENRE_2 +44 SERIES_GROUP_GENRE_3 +45 SERIES_SEASON_GENRE_3 +46 PROG_GENRE_3 diff --git a/mc-vod-sync/mc-vod-sync-service/pom.xml b/mc-vod-sync/mc-vod-sync-service/pom.xml index ee71d40a..5d0ed5a1 100644 --- a/mc-vod-sync/mc-vod-sync-service/pom.xml +++ b/mc-vod-sync/mc-vod-sync-service/pom.xml @@ -8,10 +8,6 @@ mc-vod-sync 0.0.1-SNAPSHOT - - - - org.springframework diff --git a/mc-vod-sync/mc-vod-sync-service/src/main/java/hu/user/mcvodsync/service/data/EntityDataService.java b/mc-vod-sync/mc-vod-sync-service/src/main/java/hu/user/mcvodsync/service/data/EntityDataService.java index e7fc6e15..bacc853f 100644 --- a/mc-vod-sync/mc-vod-sync-service/src/main/java/hu/user/mcvodsync/service/data/EntityDataService.java +++ b/mc-vod-sync/mc-vod-sync-service/src/main/java/hu/user/mcvodsync/service/data/EntityDataService.java @@ -60,7 +60,7 @@ public class EntityDataService extends EntityDataService JSONAssert.assertEquals(json1, json2, JSONCompareMode.LENIENT); } catch (Throwable e) { result = false; - log.error(e); + //log.error(e); } return result; @@ -78,7 +78,7 @@ public class EntityDataService extends EntityDataService JSONAssert.assertEquals(json1, json2, JSONCompareMode.LENIENT); } catch (Throwable e) { result = true; - log.error(e); + //log.error(e); } } return result; diff --git a/mc-vod-sync/mc-vod-sync-service/src/main/java/hu/user/mcvodsync/service/xls/AssetImportService.java b/mc-vod-sync/mc-vod-sync-service/src/main/java/hu/user/mcvodsync/service/xls/AssetImportService.java new file mode 100644 index 00000000..723d866b --- /dev/null +++ b/mc-vod-sync/mc-vod-sync-service/src/main/java/hu/user/mcvodsync/service/xls/AssetImportService.java @@ -0,0 +1,64 @@ +package hu.user.mcvodsync.service.xls; + + +import hu.user.mcvodsync.db.Asset; +import hu.user.mcvodsync.db.AssetId; +import hu.user.mcvodsync.db.AssetSync; +import hu.user.mcvodsync.db.SyncType; +import hu.user.mcvodsync.db.repository.AssetRepository; +import hu.user.mcvodsync.db.repository.AssetSyncRepository; +import hu.user.mcvodsync.service.data.EntityDataService; +import lombok.extern.log4j.Log4j2; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Service; +import org.springframework.transaction.annotation.Transactional; + +import java.util.Objects; +import java.util.Optional; + +@Log4j2 +@Service +public class AssetImportService { + @Autowired + AssetRepository assetRepository; + + @Autowired + AssetSyncRepository assetSyncRepository; + + @Autowired + EntityDataService entityDataService; + + @Transactional + public void processAsset(String executionId, Asset asset) { + AssetId assetId = AssetId.builder() + .episodeId(asset.getEpisodeId()) + .seasonId(asset.getSeasonId()) + .seriesId(asset.getSeriesId()) + .build(); + Optional optionalEntity = assetRepository.findById(assetId); + + AssetSync assetSync = AssetSync.builder() + .executionId(executionId) + .episodeId(asset.getEpisodeId()) + .seasonId(asset.getSeasonId()) + .seriesId(asset.getSeriesId()) + .build(); + + if (optionalEntity.isPresent()) { + boolean areDifferent = entityDataService.areDifferent(asset, optionalEntity.get()); + if (areDifferent) { +// log.info("History UPDATE registered for {}", assetId); + assetSync.setSyncType(SyncType.UPDATE); + } + } else { +// log.info("History INSERT registered for {}", assetId); + assetSync.setSyncType(SyncType.INSERT); + } + + if (Objects.nonNull(assetSync.getSyncType())) { + assetSyncRepository.save(assetSync); + } + assetRepository.save(asset); + } + +} diff --git a/mc-vod-sync/mc-vod-sync-service/src/main/java/hu/user/mcvodsync/service/xls/AssetMapper.java b/mc-vod-sync/mc-vod-sync-service/src/main/java/hu/user/mcvodsync/service/xls/AssetMapper.java new file mode 100644 index 00000000..26a2e84e --- /dev/null +++ b/mc-vod-sync/mc-vod-sync-service/src/main/java/hu/user/mcvodsync/service/xls/AssetMapper.java @@ -0,0 +1,27 @@ +package hu.user.mcvodsync.service.xls; + +import hu.user.mcvodsync.db.Asset; +import org.mapstruct.Mapper; +import org.mapstruct.Mapping; +import org.mapstruct.ReportingPolicy; + +import java.util.Map; + + +@Mapper(unmappedTargetPolicy = ReportingPolicy.IGNORE) +public interface AssetMapper { + @Mapping(target = "episodeId", source = "CATALOG_ID") + @Mapping(target = "seasonId", source = "SEASON_CATALOG_ID") + @Mapping(target = "seriesId", source = "SERIES_GROUP_CATALOG_ID") + @Mapping(target = "hubInfo", source = "HUB_INFO") + @Mapping(target = "episodeTitle", source = "PROG_ORIG_TITLE") + @Mapping(target = "seasonTitle", source = "SEASON_ORIG_TITLE") + @Mapping(target = "seriesTitle", source = "SER_GROUP_ORIG_TITLE") + @Mapping(target = "seasonNr", source = "SEASON_NUMBER") + @Mapping(target = "episodeNr", source = "EPISODE_NR") + @Mapping(target = "ageRating", source = "AGE_RATING") + @Mapping(target = "country", source = "COUNTRY_OF_ORIGIN") + @Mapping(target = "productionYear", source = "PRODUCTION_YEAR") + Asset toEntity(Map xlsRowData); + +} diff --git a/mc-vod-sync/mc-vod-sync-service/src/main/java/hu/user/mcvodsync/service/xls/VodXlsProcessor.java b/mc-vod-sync/mc-vod-sync-service/src/main/java/hu/user/mcvodsync/service/xls/VodXlsProcessor.java index d0b60fd1..6eb4bfc6 100644 --- a/mc-vod-sync/mc-vod-sync-service/src/main/java/hu/user/mcvodsync/service/xls/VodXlsProcessor.java +++ b/mc-vod-sync/mc-vod-sync-service/src/main/java/hu/user/mcvodsync/service/xls/VodXlsProcessor.java @@ -1,45 +1,92 @@ package hu.user.mcvodsync.service.xls; -import hu.user.mcvodsync.db.UploadFile; +import hu.user.mcvodsync.db.Asset; import lombok.extern.log4j.Log4j2; +import org.apache.commons.lang3.StringUtils; +import org.apache.commons.lang3.time.DurationFormatUtils; +import org.dhatim.fastexcel.reader.Cell; import org.dhatim.fastexcel.reader.ReadableWorkbook; import org.dhatim.fastexcel.reader.Row; import org.dhatim.fastexcel.reader.Sheet; +import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Service; import java.io.ByteArrayInputStream; import java.io.IOException; import java.io.InputStream; -import java.util.Collections; -import java.util.List; +import java.time.Duration; +import java.time.Instant; +import java.util.*; +import java.util.stream.Collectors; import java.util.stream.Stream; @Log4j2 @Service public class VodXlsProcessor { - private final boolean firstRowAsHeader = true; private final List mappings = Collections.emptyList(); + private List headers; - public void process(UploadFile uploadFile) { - try (InputStream is = new ByteArrayInputStream(uploadFile.getFile()); + @Autowired + AssetMapper assetMapper; + + @Autowired + AssetImportService assetImportService; + + public void process(String fileName, byte[] xlsData) { + Instant start = Instant.now(); + String executionId = UUID.randomUUID().toString(); + try (InputStream is = new ByteArrayInputStream(xlsData); ReadableWorkbook wb = new ReadableWorkbook(is)) { Sheet sheet = wb.getFirstSheet(); + try (Stream rows = sheet.openStream()) { - rows.forEach(r -> { -// if (firstRowAsHeader && r.getRowNum() == 1) { -// continue; -// } - String str = r.getCellAsString(0).orElse(null); - log.info("{} {}", r.getRowNum(), str); -// BigDecimal num = r.getCellAsNumber(0).orElse(null); -// String str = r.getCellAsString(1).orElse(null); -// LocalDateTime date = r.getCellAsDate(2).orElse(null); + rows.forEach(row -> { + if (row.getRowNum() == 1) { + headers = getHeaders(row); + } else { + processRow(executionId, row); + } }); } } catch (IOException e) { - log.error("Excel file reading error from {}. System message: {}", uploadFile.getName(), e.getMessage()); + log.error("Excel file reading error from {}. System message: {}", fileName, e.getMessage()); + } + Duration executionDuration = Duration.between(start, Instant.now()); + String duration = DurationFormatUtils.formatDuration(executionDuration.toMillis(), "H:mm:ss", true); + log.info("Processing took {}", duration); + } + + private List getHeaders(Row row) { + return row.stream().collect(Collectors.toList()); + } + + private void processRow(String executionId, Row r) { + Map rowData = new HashMap<>(); + r.stream().filter(Objects::nonNull).forEach(c -> { + try { + String data = c.getRawValue(); + rowData.put(headers.get(c.getColumnIndex()).getRawValue(), data); + } catch (Exception e) { + log.error("Cell error!", e); + } + }); + Asset asset = assetMapper.toEntity(rowData); + if (Objects.isNull(asset.getSeasonId())) { + asset.setSeasonId(0L); + } + if (Objects.isNull(asset.getSeriesId())) { + asset.setSeriesId(0L); } + if (StringUtils.isBlank(asset.getHubInfo()) || StringUtils.isBlank(asset.getEpisodeTitle())) { + //log.error("Hub not defined for {}", asset); + } else { + try { + assetImportService.processAsset(executionId, asset); + } catch (Exception e) { + log.error("Processing error for {}", asset); + } + } } } diff --git a/mc-vod-sync/pom.xml b/mc-vod-sync/pom.xml index 8b4ddbeb..243eef56 100644 --- a/mc-vod-sync/pom.xml +++ b/mc-vod-sync/pom.xml @@ -18,6 +18,8 @@ 1.8 1.8 7.19.0 + 1.5.5.Final + 1.18.26 org.springframework.boot @@ -27,13 +29,51 @@ + + + + org.apache.maven.plugins + maven-compiler-plugin + 3.8.0 + + + + org.mapstruct + mapstruct-processor + ${org.mapstruct.version} + + + org.projectlombok + lombok + ${lombok.version} + + + org.projectlombok + lombok-mapstruct-binding + 0.2.0 + + + + + -Amapstruct.defaultComponentModel=spring + + + + + + org.projectlombok lombok - 1.18.26 + ${lombok.version} provided + + org.mapstruct + mapstruct + ${org.mapstruct.version} + org.apache.logging.log4j log4j-api