--- /dev/null
+* Ha hiba történik az import vagy a szinkronizálás során a teljes fájl tartalmat el kell vetni (rollback), vagy ami jó
+ az frissüljön be?
+* Ha hibás egy sor a táblázatban, akkor a teljes importot el kelll dobni, vagy csak a sort kell kihagyni? pl. üres
+ hubinfo vagy episodetitle
\ No newline at end of file
package hu.user.mcvodsync;
import lombok.extern.log4j.Log4j2;
-import org.apache.logging.log4j.LogManager;
-import org.apache.logging.log4j.Logger;
import org.springframework.boot.SpringApplication;
import org.springframework.boot.autoconfigure.SpringBootApplication;
import org.springframework.boot.builder.SpringApplicationBuilder;
import org.springframework.boot.web.servlet.support.SpringBootServletInitializer;
import org.springframework.context.ApplicationContext;
-import org.springframework.context.ApplicationListener;
-import org.springframework.context.event.ContextRefreshedEvent;
import org.springframework.core.SpringVersion;
import org.springframework.data.jpa.repository.config.EnableJpaRepositories;
import org.springframework.web.servlet.config.annotation.EnableWebMvc;
@Log4j2
@EnableJpaRepositories("hu.user.mcvodsync.db.repository")
@SpringBootApplication(scanBasePackages = {"hu.user.mcvodsync"})
+
public class VodSyncEntry extends SpringBootServletInitializer {
datasource:
type: com.zaxxer.hikari.HikariDataSource
url: jdbc:db2://localhost:50000/vodsync
+ retrieveMessagesFromServerOnGetMessage: true
username: db2admin
password: password
camunda.bpm:
package hu.user.mcvodsync;
-import hu.user.mcvodsync.db.UploadFile;
import hu.user.mcvodsync.service.xls.VodXlsProcessor;
import lombok.extern.log4j.Log4j2;
import org.junit.Test;
VodXlsProcessor vodXlsProcessor;
@Test
- public void listProjects() throws IOException {
- Path xlsFile = Paths.get("src", "test", "resources", "AMC_Selekt_jogositott_tartalmak_20230906.xlsx");
- log.info(xlsFile.toAbsolutePath());
- UploadFile uploadFile = UploadFile.builder()
- .file(Files.readAllBytes(xlsFile))
- .build();
- vodXlsProcessor.process(uploadFile);
+ public void processXLS() throws IOException {
+ Path xlsFile = Paths.get("src/test/resources/AMC_Selekt_jogositott_tartalmak_20230906.xlsx");
+ vodXlsProcessor.process(xlsFile.getFileName().toString(), Files.readAllBytes(xlsFile));
}
}
--- // create_initial_schema
+-- // Create system tables
-- Migration SQL that makes the change goes here.
CREATE TABLE ASSOCIATE (
id BIGINT NOT NULL GENERATED BY DEFAULT AS IDENTITY,
CONSTRAINT pk_associate PRIMARY KEY (id)
);
-CREATE TABLE UPLOAD_FILE (
- id BIGINT NOT NULL GENERATED BY DEFAULT AS IDENTITY,
- created TIMESTAMP,
- name VARCHAR(255),
- size INTEGER NOT NULL,
- file BLOB,
- CONSTRAINT pk_uploadfile PRIMARY KEY (id)
-);
-
+INSERT INTO ASSOCIATE
+ (name, login, password, active)
+VALUES
+ ('Test User', 'user', 'password', 1);
-- //@UNDO
-- SQL to undo the change goes here.
-DROP TABLE UPLOAD_FILE;
DROP TABLE ASSOCIATE;
--- /dev/null
+-- // Create business tables
+-- Migration SQL that makes the change goes here.
+
+CREATE TABLE UPLOAD_FILE (
+ id BIGINT NOT NULL GENERATED BY DEFAULT AS IDENTITY,
+ created TIMESTAMP,
+ name VARCHAR(255),
+ size INTEGER NOT NULL,
+ file BLOB,
+ CONSTRAINT pk_uploadfile PRIMARY KEY (id)
+);
+
+CREATE TABLE ASSET (
+ episode_id VARCHAR(20) NOT NULL,
+ season_id BIGINT NOT NULL,
+ series_id BIGINT NOT NULL,
+ hub_info VARCHAR(255) NOT NULL,
+ episode_title VARCHAR(255) NOT NULL,
+ season_title VARCHAR(255),
+ series_title VARCHAR(255),
+ season_nr INTEGER,
+ episode_nr INTEGER,
+ production_year INTEGER,
+ country VARCHAR(1000),
+ age_rating VARCHAR(10),
+ CONSTRAINT pk_asset PRIMARY KEY (episode_id, season_id, series_id)
+);
+
+CREATE TABLE ASSET_SYNC (
+ execution_id VARCHAR(36) NOT NULL,
+ episode_id VARCHAR(20) NOT NULL,
+ season_id BIGINT NOT NULL,
+ series_id BIGINT NOT NULL,
+ sync_type VARCHAR(6) NOT NULL,
+ completed TIMESTAMP,
+ CONSTRAINT pk_asset_sync PRIMARY KEY (episode_id, season_id, series_id)
+);
+
+-- //@UNDO
+-- SQL to undo the change goes here.
+DROP TABLE ASSET;
+DROP TABLE ASSET_SYNC;
+DROP TABLE UPLOAD_FILE;
+++ /dev/null
--- // create test data
--- Migration SQL that makes the change goes here.
-
-INSERT INTO ASSOCIATE
- (name, login, password, active)
-VALUES
- ('Test User', 'user', 'password', 1);
-
-
--- //@UNDO
--- SQL to undo the change goes here.
-
-DELETE FROM ASSOCIATE;
--- /dev/null
+package hu.user.mcvodsync.db;
+
+import lombok.*;
+
+import javax.persistence.Column;
+import javax.persistence.Entity;
+import javax.persistence.Id;
+import javax.persistence.IdClass;
+import java.io.Serializable;
+
+@Getter
+@Setter
+@Entity
+@Builder
+@ToString
+@NoArgsConstructor
+@AllArgsConstructor
+@IdClass(AssetId.class)
+public class Asset implements Serializable {
+ @Id
+ private String episodeId;
+
+ @Id
+ private Long seasonId;
+
+ @Id
+ private Long seriesId;
+
+
+ @Column(nullable = false)
+ private String hubInfo;
+
+
+ @Column(nullable = false)
+ private String episodeTitle;
+
+
+ private String seasonTitle;
+
+
+ private String seriesTitle;
+
+
+ private Integer seasonNr;
+
+
+ private Integer episodeNr;
+
+
+ private Integer productionYear;
+
+
+ private String country;
+
+
+ private String ageRating;
+}
--- /dev/null
+package hu.user.mcvodsync.db;
+
+import lombok.*;
+
+import javax.persistence.Column;
+import java.io.Serializable;
+
+@Getter
+@Setter
+@Builder
+@ToString
+@NoArgsConstructor
+@AllArgsConstructor
+public class AssetId implements Serializable {
+
+ @Column(nullable = false)
+ private String episodeId;
+
+ @Column(nullable = false)
+ private Long seasonId;
+
+ @Column(nullable = false)
+ private Long seriesId;
+
+}
--- /dev/null
+package hu.user.mcvodsync.db;
+
+import lombok.*;
+
+import javax.persistence.*;
+import java.io.Serializable;
+import java.util.Date;
+
+@Getter
+@Setter
+@Entity
+@Builder
+@ToString
+@NoArgsConstructor
+@AllArgsConstructor
+@IdClass(AssetId.class)
+public class AssetSync implements Serializable {
+
+ @Column(nullable = false)
+ private String executionId;
+
+ @Id
+ private String episodeId;
+
+ @Id
+ private Long seasonId;
+
+ @Id
+ private Long seriesId;
+
+ @Enumerated(EnumType.STRING)
+ @Column(nullable = false)
+ private SyncType syncType;
+
+ private Date completed;
+}
--- /dev/null
+package hu.user.mcvodsync.db;
+
+public enum SyncType {
+ INSERT, UPDATE, DELETE, ERROR
+}
@Id
@GeneratedValue(strategy = GenerationType.IDENTITY)
Long id;
+
Date created;
+
String name;
+
int size;
+
byte[] file;
}
--- /dev/null
+package hu.user.mcvodsync.db.repository;
+
+import hu.user.mcvodsync.db.Asset;
+import hu.user.mcvodsync.db.AssetId;
+import org.springframework.data.jpa.repository.JpaRepository;
+
+public interface AssetRepository extends JpaRepository<Asset, AssetId> {
+}
--- /dev/null
+package hu.user.mcvodsync.db.repository;
+
+import hu.user.mcvodsync.db.AssetId;
+import hu.user.mcvodsync.db.AssetSync;
+import org.springframework.data.jpa.repository.JpaRepository;
+
+public interface AssetSyncRepository extends JpaRepository<AssetSync, AssetId> {
+}
--- /dev/null
+0 CATALOG_ID
+1 SEASON_CATALOG_ID
+2 SERIES_GROUP_CATALOG_ID
+3 HUB_INFO
+4 PROG_ORIG_TITLE
+5 PROG_LOCAL_TITLE
+6 SEASON_ORIG_TITLE
+7 SEASON_LOCAL_TITLE
+8 SER_GROUP_ORIG_TITLE
+9 SERIES_GROUP_LOCAL_TITLE
+10 SEASON_NUMBER
+11 EPISODE_NR
+12 PROG_SYNOPSIS_FULL_ENG
+13 PROG_SYNOPSIS_FULL_HUN
+14 PROG_SYNOPSIS_SHORT_ENG
+15 PROG_SYNOPSIS_SHORT_HUN
+16 SEASON_SYNOPSIS_FULL_ENG
+17 SEASON_SYNOPSIS_FULL_HUN
+18 SEASON_SYNOPSIS_SHORT_ENG
+19 SEASON_SYNOPSIS_SHORT_HUN
+20 SERIES_GROUP_SYNOPSIS_FULL_ENG
+21 SERIES_GROUP_SYNOPSIS_FULL_HUN
+22 SERIES_GROUP_SYNOPSIS_SHORT_ENG
+23 SERIES_GROUP_SYNOPSIS_SHORT_HUN
+24 ORIGINAL_LANGUAGE
+25 CONTENT_TYPE
+26 CONTENT_SOURCE
+27 AGE_RATING
+28 PG_SEX
+29 PG_LANGUAGE
+30 PG_VIOLENCE
+31 PG_SUBSTANCES
+32 PG_FEAR
+33 PG_RELIGION
+34 COUNTRY_OF_ORIGIN
+35 PRODUCTION_YEAR
+36 DIRECTOR
+37 X_CAST
+38 SERIES_GROUP_GENRE_1
+39 SERIES_SEASON_GENRE_1
+40 PROG_GENRE_1
+41 SERIES_GROUP_GENRE_2
+42 SERIES_SEASON_GENRE_2
+43 PROG_GENRE_2
+44 SERIES_GROUP_GENRE_3
+45 SERIES_SEASON_GENRE_3
+46 PROG_GENRE_3
<artifactId>mc-vod-sync</artifactId>
<version>0.0.1-SNAPSHOT</version>
</parent>
- <build>
- <plugins>
- </plugins>
- </build>
<dependencies>
<dependency>
<groupId>org.springframework</groupId>
JSONAssert.assertEquals(json1, json2, JSONCompareMode.LENIENT);
} catch (Throwable e) {
result = false;
- log.error(e);
+ //log.error(e);
}
return result;
JSONAssert.assertEquals(json1, json2, JSONCompareMode.LENIENT);
} catch (Throwable e) {
result = true;
- log.error(e);
+ //log.error(e);
}
}
return result;
--- /dev/null
+package hu.user.mcvodsync.service.xls;
+
+
+import hu.user.mcvodsync.db.Asset;
+import hu.user.mcvodsync.db.AssetId;
+import hu.user.mcvodsync.db.AssetSync;
+import hu.user.mcvodsync.db.SyncType;
+import hu.user.mcvodsync.db.repository.AssetRepository;
+import hu.user.mcvodsync.db.repository.AssetSyncRepository;
+import hu.user.mcvodsync.service.data.EntityDataService;
+import lombok.extern.log4j.Log4j2;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.stereotype.Service;
+import org.springframework.transaction.annotation.Transactional;
+
+import java.util.Objects;
+import java.util.Optional;
+
+@Log4j2
+@Service
+public class AssetImportService {
+ @Autowired
+ AssetRepository assetRepository;
+
+ @Autowired
+ AssetSyncRepository assetSyncRepository;
+
+ @Autowired
+ EntityDataService<Asset> entityDataService;
+
+ @Transactional
+ public void processAsset(String executionId, Asset asset) {
+ AssetId assetId = AssetId.builder()
+ .episodeId(asset.getEpisodeId())
+ .seasonId(asset.getSeasonId())
+ .seriesId(asset.getSeriesId())
+ .build();
+ Optional<Asset> optionalEntity = assetRepository.findById(assetId);
+
+ AssetSync assetSync = AssetSync.builder()
+ .executionId(executionId)
+ .episodeId(asset.getEpisodeId())
+ .seasonId(asset.getSeasonId())
+ .seriesId(asset.getSeriesId())
+ .build();
+
+ if (optionalEntity.isPresent()) {
+ boolean areDifferent = entityDataService.areDifferent(asset, optionalEntity.get());
+ if (areDifferent) {
+// log.info("History UPDATE registered for {}", assetId);
+ assetSync.setSyncType(SyncType.UPDATE);
+ }
+ } else {
+// log.info("History INSERT registered for {}", assetId);
+ assetSync.setSyncType(SyncType.INSERT);
+ }
+
+ if (Objects.nonNull(assetSync.getSyncType())) {
+ assetSyncRepository.save(assetSync);
+ }
+ assetRepository.save(asset);
+ }
+
+}
--- /dev/null
+package hu.user.mcvodsync.service.xls;
+
+import hu.user.mcvodsync.db.Asset;
+import org.mapstruct.Mapper;
+import org.mapstruct.Mapping;
+import org.mapstruct.ReportingPolicy;
+
+import java.util.Map;
+
+
+@Mapper(unmappedTargetPolicy = ReportingPolicy.IGNORE)
+public interface AssetMapper {
+ @Mapping(target = "episodeId", source = "CATALOG_ID")
+ @Mapping(target = "seasonId", source = "SEASON_CATALOG_ID")
+ @Mapping(target = "seriesId", source = "SERIES_GROUP_CATALOG_ID")
+ @Mapping(target = "hubInfo", source = "HUB_INFO")
+ @Mapping(target = "episodeTitle", source = "PROG_ORIG_TITLE")
+ @Mapping(target = "seasonTitle", source = "SEASON_ORIG_TITLE")
+ @Mapping(target = "seriesTitle", source = "SER_GROUP_ORIG_TITLE")
+ @Mapping(target = "seasonNr", source = "SEASON_NUMBER")
+ @Mapping(target = "episodeNr", source = "EPISODE_NR")
+ @Mapping(target = "ageRating", source = "AGE_RATING")
+ @Mapping(target = "country", source = "COUNTRY_OF_ORIGIN")
+ @Mapping(target = "productionYear", source = "PRODUCTION_YEAR")
+ Asset toEntity(Map<String, String> xlsRowData);
+
+}
package hu.user.mcvodsync.service.xls;
-import hu.user.mcvodsync.db.UploadFile;
+import hu.user.mcvodsync.db.Asset;
import lombok.extern.log4j.Log4j2;
+import org.apache.commons.lang3.StringUtils;
+import org.apache.commons.lang3.time.DurationFormatUtils;
+import org.dhatim.fastexcel.reader.Cell;
import org.dhatim.fastexcel.reader.ReadableWorkbook;
import org.dhatim.fastexcel.reader.Row;
import org.dhatim.fastexcel.reader.Sheet;
+import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Service;
import java.io.ByteArrayInputStream;
import java.io.IOException;
import java.io.InputStream;
-import java.util.Collections;
-import java.util.List;
+import java.time.Duration;
+import java.time.Instant;
+import java.util.*;
+import java.util.stream.Collectors;
import java.util.stream.Stream;
@Log4j2
@Service
public class VodXlsProcessor {
- private final boolean firstRowAsHeader = true;
private final List<String> mappings = Collections.emptyList();
+ private List<Cell> headers;
- public void process(UploadFile uploadFile) {
- try (InputStream is = new ByteArrayInputStream(uploadFile.getFile());
+ @Autowired
+ AssetMapper assetMapper;
+
+ @Autowired
+ AssetImportService assetImportService;
+
+ public void process(String fileName, byte[] xlsData) {
+ Instant start = Instant.now();
+ String executionId = UUID.randomUUID().toString();
+ try (InputStream is = new ByteArrayInputStream(xlsData);
ReadableWorkbook wb = new ReadableWorkbook(is)) {
Sheet sheet = wb.getFirstSheet();
+
try (Stream<Row> rows = sheet.openStream()) {
- rows.forEach(r -> {
-// if (firstRowAsHeader && r.getRowNum() == 1) {
-// continue;
-// }
- String str = r.getCellAsString(0).orElse(null);
- log.info("{} {}", r.getRowNum(), str);
-// BigDecimal num = r.getCellAsNumber(0).orElse(null);
-// String str = r.getCellAsString(1).orElse(null);
-// LocalDateTime date = r.getCellAsDate(2).orElse(null);
+ rows.forEach(row -> {
+ if (row.getRowNum() == 1) {
+ headers = getHeaders(row);
+ } else {
+ processRow(executionId, row);
+ }
});
}
} catch (IOException e) {
- log.error("Excel file reading error from {}. System message: {}", uploadFile.getName(), e.getMessage());
+ log.error("Excel file reading error from {}. System message: {}", fileName, e.getMessage());
+ }
+ Duration executionDuration = Duration.between(start, Instant.now());
+ String duration = DurationFormatUtils.formatDuration(executionDuration.toMillis(), "H:mm:ss", true);
+ log.info("Processing took {}", duration);
+ }
+
+ private List<Cell> getHeaders(Row row) {
+ return row.stream().collect(Collectors.toList());
+ }
+
+ private void processRow(String executionId, Row r) {
+ Map<String, String> rowData = new HashMap<>();
+ r.stream().filter(Objects::nonNull).forEach(c -> {
+ try {
+ String data = c.getRawValue();
+ rowData.put(headers.get(c.getColumnIndex()).getRawValue(), data);
+ } catch (Exception e) {
+ log.error("Cell error!", e);
+ }
+ });
+ Asset asset = assetMapper.toEntity(rowData);
+ if (Objects.isNull(asset.getSeasonId())) {
+ asset.setSeasonId(0L);
+ }
+ if (Objects.isNull(asset.getSeriesId())) {
+ asset.setSeriesId(0L);
}
+ if (StringUtils.isBlank(asset.getHubInfo()) || StringUtils.isBlank(asset.getEpisodeTitle())) {
+ //log.error("Hub not defined for {}", asset);
+ } else {
+ try {
+ assetImportService.processAsset(executionId, asset);
+ } catch (Exception e) {
+ log.error("Processing error for {}", asset);
+ }
+ }
}
}
<maven.compiler.source>1.8</maven.compiler.source>
<maven.compiler.target>1.8</maven.compiler.target>
<camunda.spring-boot.version>7.19.0</camunda.spring-boot.version>
+ <org.mapstruct.version>1.5.5.Final</org.mapstruct.version>
+ <lombok.version>1.18.26</lombok.version>
</properties>
<parent>
<groupId>org.springframework.boot</groupId>
</parent>
<repositories>
</repositories>
+ <build>
+ <plugins>
+ <plugin>
+ <groupId>org.apache.maven.plugins</groupId>
+ <artifactId>maven-compiler-plugin</artifactId>
+ <version>3.8.0</version>
+ <configuration>
+ <annotationProcessorPaths>
+ <path>
+ <groupId>org.mapstruct</groupId>
+ <artifactId>mapstruct-processor</artifactId>
+ <version>${org.mapstruct.version}</version>
+ </path>
+ <path>
+ <groupId>org.projectlombok</groupId>
+ <artifactId>lombok</artifactId>
+ <version>${lombok.version}</version>
+ </path>
+ <dependency>
+ <groupId>org.projectlombok</groupId>
+ <artifactId>lombok-mapstruct-binding</artifactId>
+ <version>0.2.0</version>
+ </dependency>
+ </annotationProcessorPaths>
+ <compilerArgs>
+ <compilerArg>
+ -Amapstruct.defaultComponentModel=spring
+ </compilerArg>
+ </compilerArgs>
+ </configuration>
+ </plugin>
+ </plugins>
+ </build>
<dependencies>
<dependency>
<groupId>org.projectlombok</groupId>
<artifactId>lombok</artifactId>
- <version>1.18.26</version>
+ <version>${lombok.version}</version>
<scope>provided</scope>
</dependency>
+ <dependency>
+ <groupId>org.mapstruct</groupId>
+ <artifactId>mapstruct</artifactId>
+ <version>${org.mapstruct.version}</version>
+ </dependency>
<dependency>
<groupId>org.apache.logging.log4j</groupId>
<artifactId>log4j-api</artifactId>
https://git.useribm.hu / mediacube.git / commitdiff