From: Vásáry Dániel Date: Tue, 7 Nov 2023 14:23:44 +0000 (+0100) Subject: XLS import playlist logic completed, some mapping missing X-Git-Url: http://git.useribm.hu/?a=commitdiff_plain;h=a05b345d6f750f8c4a9924df088d14f40c9e41c9;p=mediacube.git XLS import playlist logic completed, some mapping missing --- diff --git a/mc-vod-sync/QUESTIONS.md b/mc-vod-sync/QUESTIONS.md index 4fd619a6..2bff3a6c 100644 --- a/mc-vod-sync/QUESTIONS.md +++ b/mc-vod-sync/QUESTIONS.md @@ -1,22 +1,29 @@ +* A sunset kezelése is kell, tehát jöhet adat későbbi sunset-el? +* A sunset és sunrise formátuma legyen egyforma (mindegy melyik) minden sorra! +* Mik a kötelező mezők? Most: playlist, sunset, sunrise, title +* A sunrise alapján lejárt video-kat a BrightCove API-n keresztül törölni kell, vagy csak a playlist-ből kell + eltávolítani? +* Feltételeze, hogy a sunrise dátum inkluzív, tehát aznap még jogosult a kijátszás, a törlés az aktuális időponttal + egyező vagy régebbi sunrise dátumú videókat törli. A manuális működés miatt kellene figyelmeztetés, hogy futtatni kell + a szinkronzációt a törlés után (ha volt + + * Ha hiba történik az import vagy a szinkronizálás során a teljes fájl tartalmat el kell vetni (rollback), vagy ami jó - az frissüljön be? + az frissüljön be? ami jó, az frissüljön * Ha hibás egy sor a táblázatban, akkor a teljes importot el kelll dobni, vagy csak a sort kell kihagyni? pl. üres - hubinfo vagy episodetitle -* Mi az Excel mező összerendelés? -* Milyen fix értéket kapnak az Excelből nem jövő video mezők? -* Mi a videoId, az API osztja? igen -* Mi a playlist rendezési elve? -* Playlist-en belül a sorrend változhat? -* Egy video több playlist-en is szerepelhet? -* Mi video asset metadata frissítés API-ja? -* Video vs Media asset -* Folder kezelés lesz? + hubinfo vagy episodetitle: sor kihagyása +* Mi az Excel mező összerendelés? mapping tábla +* Milyen fix értéket kapnak az Excelből nem jövő video mezők? mapping megmondja +* Mi a videoId, az API osztja? nem, a catalogId az +* Mi a playlist rendezési elve? seasonnr, epizodenr alapján +* Playlist-en belül a sorrend változhat? igen +* Egy video több playlist-en is szerepelhet? igen -* Video törlése minden listáról: -DEL https://cms.api.brightcove.com/v1/accounts/{{account_id}}/videos/{{video_id}}/references +* Video törlése minden listáról: + DEL https://cms.api.brightcove.com/v1/accounts/{{account_id}}/videos/{{video_id}}/references * Video törlése -https://cms.api.brightcove.com/v1/accounts/{{account_id}}/videos{video_ids}} + https://cms.api.brightcove.com/v1/accounts/{{account_id}}/videos{video_ids}} https://apis.support.brightcove.com/getting-started/getting-started-brightcove-apis.html https://apis.support.brightcove.com/cms/index.html diff --git a/mc-vod-sync/mc-vod-sync-app/src/test/java/hu/user/mcvodsync/RepositoryIT.java b/mc-vod-sync/mc-vod-sync-app/src/test/java/hu/user/mcvodsync/RepositoryIT.java index cc7ebb3d..55d5a7b9 100644 --- a/mc-vod-sync/mc-vod-sync-app/src/test/java/hu/user/mcvodsync/RepositoryIT.java +++ b/mc-vod-sync/mc-vod-sync-app/src/test/java/hu/user/mcvodsync/RepositoryIT.java @@ -5,15 +5,28 @@ package hu.user.mcvodsync; +import hu.user.mcvodsync.db.repository.AssetRepository; +import hu.user.mcvodsync.db.repository.AssetSyncRepository; +import hu.user.mcvodsync.service.xls.AssetImportService; +import hu.user.mcvodsync.service.xls.ImportSummary; import lombok.extern.log4j.Log4j2; import org.junit.Test; import org.junit.runner.RunWith; +import org.springframework.beans.factory.annotation.Autowired; import org.springframework.boot.test.context.SpringBootTest; import org.springframework.context.annotation.ComponentScan; import org.springframework.test.context.ActiveProfiles; import org.springframework.test.context.TestPropertySource; import org.springframework.test.context.junit4.SpringRunner; +import java.sql.Date; +import java.time.LocalDate; +import java.time.format.DateTimeFormatter; +import java.util.List; + +import static org.junit.Assert.assertEquals; +import static org.junit.Assert.assertNotNull; + @Log4j2 @RunWith(SpringRunner.class) @@ -24,21 +37,38 @@ import org.springframework.test.context.junit4.SpringRunner; @TestPropertySource("classpath:application-dev.yaml") //@AutoConfigureMockMvc public class RepositoryIT { -// @Autowired -// ServiceRecordRepository serviceRecordRepository; -// @Autowired -// ProjectRepository projectRepository; -// @Autowired -// ProjectService projectService; -// @Autowired -// TreasuryService treasuryService; -// @Autowired -// private TreasuryRepository treasuryRepository; + @Autowired + private AssetRepository assetRepository; + + @Autowired + private AssetSyncRepository assetSyncRepository; + + @Autowired + private AssetImportService assetImportService; @Test - public void listProjects() { -// List allItems = projectRepository.findAll(); -// log.info("Found {} items", allItems.size()); + public void queryPlayLists() { + List playlists = assetRepository.queryDistinctPlaylists(); + assertNotNull(playlists); + + List assets = assetRepository.queryPlaylistsAssets(playlists.get(0)); + assertNotNull(assets); + } + + @Test + public void queryExpired() { + LocalDate currentDate = LocalDate.parse("2000-01-02", DateTimeFormatter.ISO_LOCAL_DATE); + List ids = assetRepository.queryExpired(Date.valueOf(currentDate)); + assertNotNull(ids); + assertEquals(1, ids.size()); + + ImportSummary summary = ImportSummary.builder().build(); + assetImportService.removeExpired(currentDate); + + ids = assetRepository.queryExpired(Date.valueOf(currentDate)); + assertNotNull(ids); + assertEquals(0, ids.size()); + } } diff --git a/mc-vod-sync/mc-vod-sync-app/src/test/java/hu/user/mcvodsync/VodXlsProcessorIT.java b/mc-vod-sync/mc-vod-sync-app/src/test/java/hu/user/mcvodsync/VodXlsProcessorIT.java index 4b602bd2..b1de9ce4 100644 --- a/mc-vod-sync/mc-vod-sync-app/src/test/java/hu/user/mcvodsync/VodXlsProcessorIT.java +++ b/mc-vod-sync/mc-vod-sync-app/src/test/java/hu/user/mcvodsync/VodXlsProcessorIT.java @@ -5,11 +5,18 @@ package hu.user.mcvodsync; +import hu.user.mcvodsync.db.SyncType; import hu.user.mcvodsync.db.repository.AssetRepository; +import hu.user.mcvodsync.db.repository.AssetSyncRepository; +import hu.user.mcvodsync.db.repository.PlaylistSyncRepository; +import hu.user.mcvodsync.service.xls.AssetImportService; import hu.user.mcvodsync.service.xls.ImportSummary; import hu.user.mcvodsync.service.xls.VodXlsProcessor; import lombok.extern.log4j.Log4j2; +import org.junit.AfterClass; +import org.junit.BeforeClass; import org.junit.Test; +import org.junit.jupiter.api.Order; import org.junit.runner.RunWith; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.boot.test.context.SpringBootTest; @@ -17,14 +24,13 @@ import org.springframework.test.context.ActiveProfiles; import org.springframework.test.context.TestPropertySource; import org.springframework.test.context.junit4.SpringRunner; +import javax.annotation.PostConstruct; import java.io.IOException; import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.Paths; -import java.util.List; import static org.junit.Assert.assertEquals; -import static org.junit.Assert.assertNotNull; @Log4j2 @@ -36,23 +42,126 @@ public class VodXlsProcessorIT { @Autowired private VodXlsProcessor vodXlsProcessor; + @Autowired + private AssetImportService assetImportService; + @Autowired private AssetRepository assetRepository; + @Autowired + private AssetSyncRepository assetSyncRepository; + + @Autowired + private PlaylistSyncRepository playlistSyncRepository; + + @BeforeClass + public static void beforeClass() { + } + + @AfterClass + public static void afterClass() { + } + + @PostConstruct + public void init() { + } + @Test - public void queryPlayLists() { - List playlists = assetRepository.queryDistinctPlaylists(); - assertNotNull(playlists); + public void initialLoadTest() throws IOException { +// Path xlsFile = Paths.get("src/test/resources/testdata-big.xlsx"); + log.info("initialLoadTest"); + cleanup(); + ImportSummary summary = process("src/test/resources/testdata.xlsx"); + checkSummary(summary, 9, 9, 0, 0); + checkAsset(9, SyncType.INSERT, 9); + checkPlaylist(3, SyncType.INSERT, 3); + } - List assets = assetRepository.queryPlaylistsAssets(playlists.get(0)); - assertNotNull(assets); + @Test + public void changeVideoTest() throws IOException { + log.info("changeVideoTest"); + ImportSummary summary = process("src/test/resources/testdata.xlsx"); + cleanupSync(); + summary = process("src/test/resources/testdata-change-video.xlsx"); + checkSummary(summary, 9, 0, 1, 0); + checkAsset(1, SyncType.UPDATE, 1); } @Test - public void processXLS() throws IOException { - Path xlsFile = Paths.get("src/test/resources/testdata.xlsx"); - ImportSummary process = vodXlsProcessor.process(xlsFile.getFileName().toString(), Files.readAllBytes(xlsFile)); - assertEquals(process.getAll(), process.getSuccess()); + public void deleteVideoTest() { + log.info("deleteVideoTest"); + ImportSummary summary = process("src/test/resources/testdata.xlsx"); + cleanupSync(); + summary = process("src/test/resources/testdata-delete-video.xlsx"); + checkSummary(summary, 9, 0, 1, 1); + checkAsset(1, SyncType.UPDATE, 2); + checkAsset(1, SyncType.DELETE, 2); + checkPlaylist(1, SyncType.UPDATE, 1); + } + + @Test + public void changePlaylistOrderTest() { + log.info("changePlaylistOrderTest"); + ImportSummary summary = process("src/test/resources/testdata.xlsx"); + cleanupSync(); + summary = process("src/test/resources/testdata-change-playlist-order.xlsx"); + checkSummary(summary, 8, 0, 1, 0); + checkAsset(1, SyncType.UPDATE, 1); + checkPlaylist(1, SyncType.UPDATE, 1); + } + + @Test + @Order(5) + public void changePlaylistAddAssetTest() { + log.info("changePlaylistAddAssetTest"); +// "src/test/resources/testdata-playlist-add-asset.xlsx" + } + + @Test + @Order(6) + public void changePlaylistRemoveAssetTest() { + log.info("changePlaylistRemoveAssetTest"); +// "src/test/resources/testdata-playlist-remove-asset.xlsx" + } + + public void cleanup() { + cleanupSync(); + assetRepository.deleteAllInBatch(); + } + + public void cleanupSync() { + playlistSyncRepository.deleteAllInBatch(); + assetSyncRepository.deleteAllInBatch(); + } + + private ImportSummary process(String xlsFile) { + ImportSummary summary = null; + try { + Path input = Paths.get(xlsFile); + summary = vodXlsProcessor.process(input.getFileName().toString(), Files.readAllBytes(input)); + } catch (IOException e) { + log.error(e); + throw new RuntimeException(e); + } + return summary; + } + + private void checkSummary(ImportSummary summary, long expectedAll, long expectedInsert, long expectedUpdate, long expectedDelete) { + assertEquals(expectedAll, summary.getAll()); + assertEquals(expectedAll, summary.getSuccess()); + assertEquals(expectedInsert, summary.getInserted()); + assertEquals(expectedUpdate, summary.getUpdated()); + assertEquals(expectedDelete, summary.getDeleted()); + } + + private void checkAsset(long expected, SyncType syncType, long expectedAllCount) { + assertEquals(expected, assetSyncRepository.countBySyncType(syncType)); + assertEquals(expectedAllCount, assetSyncRepository.count()); + } + + private void checkPlaylist(long expected, SyncType syncType, long expectedAllCount) { + assertEquals(expected, playlistSyncRepository.countBySyncType(syncType)); + assertEquals(expectedAllCount, playlistSyncRepository.count()); } } diff --git a/mc-vod-sync/mc-vod-sync-app/src/test/resources/testdata-change-playlist-order.xlsx b/mc-vod-sync/mc-vod-sync-app/src/test/resources/testdata-change-playlist-order.xlsx new file mode 100644 index 00000000..ad99c38a Binary files /dev/null and b/mc-vod-sync/mc-vod-sync-app/src/test/resources/testdata-change-playlist-order.xlsx differ diff --git a/mc-vod-sync/mc-vod-sync-app/src/test/resources/testdata-change-video.xlsx b/mc-vod-sync/mc-vod-sync-app/src/test/resources/testdata-change-video.xlsx new file mode 100644 index 00000000..b20367aa Binary files /dev/null and b/mc-vod-sync/mc-vod-sync-app/src/test/resources/testdata-change-video.xlsx differ diff --git a/mc-vod-sync/mc-vod-sync-app/src/test/resources/testdata-delete-video.xlsx b/mc-vod-sync/mc-vod-sync-app/src/test/resources/testdata-delete-video.xlsx new file mode 100644 index 00000000..77d8a951 Binary files /dev/null and b/mc-vod-sync/mc-vod-sync-app/src/test/resources/testdata-delete-video.xlsx differ diff --git a/mc-vod-sync/mc-vod-sync-app/src/test/resources/testdata.xlsx b/mc-vod-sync/mc-vod-sync-app/src/test/resources/testdata.xlsx index 51429a78..bfe30365 100644 Binary files a/mc-vod-sync/mc-vod-sync-app/src/test/resources/testdata.xlsx and b/mc-vod-sync/mc-vod-sync-app/src/test/resources/testdata.xlsx differ diff --git a/mc-vod-sync/mc-vod-sync-db/migrations/bootstrap.sql b/mc-vod-sync/mc-vod-sync-db/migrations/bootstrap.sql index 99b70ec0..c0eae3c6 100644 --- a/mc-vod-sync/mc-vod-sync-db/migrations/bootstrap.sql +++ b/mc-vod-sync/mc-vod-sync-db/migrations/bootstrap.sql @@ -1,3 +1,7 @@ -- // Bootstrap.sql -DROP DATABASE IF EXISTS VODSYNC; -CREATE DATABASE VODSYNC AUTOMATIC STORAGE YES USING CODESET UTF-8 TERRITORY hu COLLATE USING UCA500R1_S2 PAGESIZE 32 K; +-- DROP DATABASE IF EXISTS VODSYNC; +-- CREATE DATABASE VODSYNC AUTOMATIC STORAGE YES USING CODESET UTF-8 TERRITORY hu COLLATE USING UCA500R1_S2 PAGESIZE 32 K; + +TRUNCATE TABLE asset IMMEDIATE; +TRUNCATE TABLE asset_sync IMMEDIATE; +TRUNCATE TABLE playlist_sync IMMEDIATE; diff --git a/mc-vod-sync/mc-vod-sync-db/migrations/scripts/003_create_business_tables.sql b/mc-vod-sync/mc-vod-sync-db/migrations/scripts/003_create_business_tables.sql index e7695e11..0cbf11a8 100644 --- a/mc-vod-sync/mc-vod-sync-db/migrations/scripts/003_create_business_tables.sql +++ b/mc-vod-sync/mc-vod-sync-db/migrations/scripts/003_create_business_tables.sql @@ -36,7 +36,7 @@ CREATE INDEX idx_asset_playlist ON asset(playlist); CREATE TABLE asset_sync ( id BIGINT NOT NULL GENERATED BY DEFAULT AS IDENTITY, catalog_id VARCHAR(20) NOT NULL, - imported TIMESTAMP NOT NULL, + created TIMESTAMP NOT NULL, sync_type VARCHAR(6) NOT NULL, exported_success TIMESTAMP, exported_error TIMESTAMP, @@ -47,7 +47,7 @@ CREATE INDEX idx_asset_sync_exported_success ON asset_sync(exported_success); CREATE TABLE playlist_sync ( id BIGINT NOT NULL GENERATED BY DEFAULT AS IDENTITY, playlist VARCHAR(100) NOT NULL, - imported TIMESTAMP NOT NULL, + created TIMESTAMP NOT NULL, sync_type VARCHAR(6) NOT NULL, exported_success TIMESTAMP, exported_error TIMESTAMP, diff --git a/mc-vod-sync/mc-vod-sync-db/src/main/java/hu/user/mcvodsync/db/AssetSync.java b/mc-vod-sync/mc-vod-sync-db/src/main/java/hu/user/mcvodsync/db/AssetSync.java index b7bdcb1b..96482d75 100644 --- a/mc-vod-sync/mc-vod-sync-db/src/main/java/hu/user/mcvodsync/db/AssetSync.java +++ b/mc-vod-sync/mc-vod-sync-db/src/main/java/hu/user/mcvodsync/db/AssetSync.java @@ -21,7 +21,7 @@ public class AssetSync implements Serializable { private String catalogId; @Column(nullable = false) - private Date imported; + private Date created; @Enumerated(EnumType.STRING) @Column(nullable = false) diff --git a/mc-vod-sync/mc-vod-sync-db/src/main/java/hu/user/mcvodsync/db/PlaylistSync.java b/mc-vod-sync/mc-vod-sync-db/src/main/java/hu/user/mcvodsync/db/PlaylistSync.java index 9d7072ef..df4efa6f 100644 --- a/mc-vod-sync/mc-vod-sync-db/src/main/java/hu/user/mcvodsync/db/PlaylistSync.java +++ b/mc-vod-sync/mc-vod-sync-db/src/main/java/hu/user/mcvodsync/db/PlaylistSync.java @@ -21,7 +21,7 @@ public class PlaylistSync implements Serializable { private String playlist; @Column(nullable = false) - private Date imported; + private Date created; @Enumerated(EnumType.STRING) @Column(nullable = false) diff --git a/mc-vod-sync/mc-vod-sync-db/src/main/java/hu/user/mcvodsync/db/repository/AssetRepository.java b/mc-vod-sync/mc-vod-sync-db/src/main/java/hu/user/mcvodsync/db/repository/AssetRepository.java index de660d3c..fd44b2b3 100644 --- a/mc-vod-sync/mc-vod-sync-db/src/main/java/hu/user/mcvodsync/db/repository/AssetRepository.java +++ b/mc-vod-sync/mc-vod-sync-db/src/main/java/hu/user/mcvodsync/db/repository/AssetRepository.java @@ -2,8 +2,10 @@ package hu.user.mcvodsync.db.repository; import hu.user.mcvodsync.db.Asset; import org.springframework.data.jpa.repository.JpaRepository; +import org.springframework.data.jpa.repository.Modifying; import org.springframework.data.jpa.repository.Query; +import java.sql.Date; import java.util.List; public interface AssetRepository extends JpaRepository { @@ -14,4 +16,10 @@ public interface AssetRepository extends JpaRepository { @Query("SELECT a.catalogId FROM Asset a WHERE a.playlist = :playlist ORDER BY a.seasonNr, a.episodeNr") List queryPlaylistsAssets(String playlist); + @Query("SELECT a.catalogId FROM Asset a WHERE a.sunset <= :currentDate ORDER BY a.catalogId") + List queryExpired(Date currentDate); + + @Modifying + @Query("DELETE FROM Asset a WHERE a.sunset <= :currentDate") + void deleteExpired(Date currentDate); } diff --git a/mc-vod-sync/mc-vod-sync-db/src/main/java/hu/user/mcvodsync/db/repository/AssetSyncRepository.java b/mc-vod-sync/mc-vod-sync-db/src/main/java/hu/user/mcvodsync/db/repository/AssetSyncRepository.java index bf27cd19..0c372c1c 100644 --- a/mc-vod-sync/mc-vod-sync-db/src/main/java/hu/user/mcvodsync/db/repository/AssetSyncRepository.java +++ b/mc-vod-sync/mc-vod-sync-db/src/main/java/hu/user/mcvodsync/db/repository/AssetSyncRepository.java @@ -1,7 +1,11 @@ package hu.user.mcvodsync.db.repository; import hu.user.mcvodsync.db.AssetSync; +import hu.user.mcvodsync.db.SyncType; import org.springframework.data.jpa.repository.JpaRepository; public interface AssetSyncRepository extends JpaRepository { + + long countBySyncType(SyncType syncType); + } diff --git a/mc-vod-sync/mc-vod-sync-db/src/main/java/hu/user/mcvodsync/db/repository/PlaylistSyncRepository.java b/mc-vod-sync/mc-vod-sync-db/src/main/java/hu/user/mcvodsync/db/repository/PlaylistSyncRepository.java new file mode 100644 index 00000000..2e752e36 --- /dev/null +++ b/mc-vod-sync/mc-vod-sync-db/src/main/java/hu/user/mcvodsync/db/repository/PlaylistSyncRepository.java @@ -0,0 +1,11 @@ +package hu.user.mcvodsync.db.repository; + +import hu.user.mcvodsync.db.PlaylistSync; +import hu.user.mcvodsync.db.SyncType; +import org.springframework.data.jpa.repository.JpaRepository; + +public interface PlaylistSyncRepository extends JpaRepository { + + long countBySyncType(SyncType syncType); + +} diff --git a/mc-vod-sync/mc-vod-sync-service/src/main/java/hu/user/mcvodsync/service/xls/AssetImportService.java b/mc-vod-sync/mc-vod-sync-service/src/main/java/hu/user/mcvodsync/service/xls/AssetImportService.java index 6539be83..a33167c8 100644 --- a/mc-vod-sync/mc-vod-sync-service/src/main/java/hu/user/mcvodsync/service/xls/AssetImportService.java +++ b/mc-vod-sync/mc-vod-sync-service/src/main/java/hu/user/mcvodsync/service/xls/AssetImportService.java @@ -3,17 +3,21 @@ package hu.user.mcvodsync.service.xls; import hu.user.mcvodsync.db.Asset; import hu.user.mcvodsync.db.AssetSync; +import hu.user.mcvodsync.db.PlaylistSync; import hu.user.mcvodsync.db.SyncType; import hu.user.mcvodsync.db.repository.AssetRepository; import hu.user.mcvodsync.db.repository.AssetSyncRepository; +import hu.user.mcvodsync.db.repository.PlaylistSyncRepository; import hu.user.mcvodsync.service.data.EntityDataService; import lombok.extern.log4j.Log4j2; +import org.apache.commons.collections4.CollectionUtils; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Service; -import org.springframework.transaction.annotation.Transactional; import java.time.Instant; +import java.time.LocalDate; import java.util.*; +import java.util.stream.Collectors; @Log4j2 @Service @@ -24,12 +28,18 @@ public class AssetImportService { @Autowired private AssetSyncRepository assetSyncRepository; + @Autowired + private PlaylistSyncRepository playlistSyncRepository; + @Autowired private EntityDataService entityDataService; private Map> currentPlaylists; - public void prepare() { + private ImportSummary summary; + + public void prepare(ImportSummary summary) { + this.summary = summary; currentPlaylists = getPlayLists(); } @@ -43,14 +53,12 @@ public class AssetImportService { return result; } - @Transactional - public void processAsset(Asset asset, ImportSummary summary) { + public void processAsset(Asset asset) { Optional optionalEntity = assetRepository.findById(asset.getCatalogId()); AssetSync assetSync = AssetSync.builder() - .imported(Date.from(Instant.now())) + .created(Date.from(Instant.now())) .catalogId(asset.getCatalogId()) - .syncType(SyncType.INSERT) .build(); if (optionalEntity.isPresent()) { @@ -58,10 +66,14 @@ public class AssetImportService { if (areDifferent) { assetSync.setSyncType(SyncType.UPDATE); } + } else { + assetSync.setSyncType(SyncType.INSERT); } - assetRepository.saveAndFlush(asset); - assetSyncRepository.saveAndFlush(assetSync); + if (Objects.nonNull(assetSync.getSyncType())) { + assetRepository.save(asset); + assetSyncRepository.save(assetSync); + } if (assetSync.getSyncType() == SyncType.UPDATE) { summary.incUpdated(); @@ -71,9 +83,78 @@ public class AssetImportService { } } + public void removeExpired(LocalDate currentDate) { + List ids = assetRepository.queryExpired(java.sql.Date.valueOf(currentDate)); + List syncs = ids.stream().map(id -> AssetSync.builder() + .catalogId(id) + .syncType(SyncType.DELETE) + .created(Date.from(Instant.now())) + .build()).collect(Collectors.toList()); + assetSyncRepository.saveAll(syncs); + assetRepository.deleteExpired(java.sql.Date.valueOf(currentDate)); + summary.setDeleted(summary.getDeleted() + ids.size()); + } + public void finish() { + removeExpired(LocalDate.now()); + processPlaylists(); + } + + public void processPlaylists() { Map> newPlaylists = getPlayLists(); + Set currentPlaylistNames = currentPlaylists.keySet(); + Set newPlaylistNames = newPlaylists.keySet(); + + Collection playlistNamesDifference = CollectionUtils.subtract(currentPlaylistNames, newPlaylistNames); + if (!playlistNamesDifference.isEmpty()) { + playlistNamesDifference.forEach(playlist -> { + PlaylistSync sync = PlaylistSync.builder() + .playlist(playlist) + .syncType(SyncType.DELETE) + .created(Date.from(Instant.now())) + .build(); + playlistSyncRepository.save(sync); + summary.incDeletedPlaylist(); + }); + } - //CollectionUtils.isEqualCollection(Collection a, Collection b) + playlistNamesDifference = CollectionUtils.subtract(newPlaylistNames, currentPlaylistNames); + if (!playlistNamesDifference.isEmpty()) { + playlistNamesDifference.forEach(playlist -> { + PlaylistSync sync = PlaylistSync.builder() + .playlist(playlist) + .syncType(SyncType.INSERT) + .created(Date.from(Instant.now())) + .build(); + playlistSyncRepository.save(sync); + summary.incInsertedPlaylist(); + }); + } + + currentPlaylistNames.forEach(playlist -> { + List currentContent = currentPlaylists.get(playlist); + List newContent = newPlaylists.get(playlist); + if (isContentChanged(currentContent, newContent)) { + PlaylistSync sync = PlaylistSync.builder() + .playlist(playlist) + .syncType(SyncType.UPDATE) + .created(Date.from(Instant.now())) + .build(); + playlistSyncRepository.save(sync); + summary.incUpdatedPlaylist(); + } + }); + } + + private boolean isContentChanged(List currentContent, List newContent) { + if (currentContent.size() != newContent.size()) { + return true; + } + for (int i = 0; i < currentContent.size(); i++) { + if (!currentContent.get(i).equals(newContent.get(i))) { + return true; + } + } + return false; } } diff --git a/mc-vod-sync/mc-vod-sync-service/src/main/java/hu/user/mcvodsync/service/xls/AssetMapper.java b/mc-vod-sync/mc-vod-sync-service/src/main/java/hu/user/mcvodsync/service/xls/AssetMapper.java index 74b4920c..67cbb3a9 100644 --- a/mc-vod-sync/mc-vod-sync-service/src/main/java/hu/user/mcvodsync/service/xls/AssetMapper.java +++ b/mc-vod-sync/mc-vod-sync-service/src/main/java/hu/user/mcvodsync/service/xls/AssetMapper.java @@ -61,14 +61,21 @@ public interface AssetMapper { } @Named("mapSunset") - static Date mapSunset(String dateValue) { - Date result = null; + static Date mapSunset(String numberValue) { +// Date result = null; +// try { +// if (StringUtils.isNotBlank(dateValue)) { +// result = new Date(formatter.parse(dateValue).getTime()); +// } +// } catch (Exception ignored) { +// } +// return result; + long longValue = 0; try { - if (StringUtils.isNotBlank(dateValue)) { - result = new Date(formatter.parse(dateValue).getTime()); - } + longValue = Long.parseLong(numberValue); } catch (Exception ignored) { } - return result; + long javaDateValue = (longValue - 25569) * 86400 * 1000; + return new Date(javaDateValue); } } diff --git a/mc-vod-sync/mc-vod-sync-service/src/main/java/hu/user/mcvodsync/service/xls/ImportSummary.java b/mc-vod-sync/mc-vod-sync-service/src/main/java/hu/user/mcvodsync/service/xls/ImportSummary.java index 172206da..0620208a 100644 --- a/mc-vod-sync/mc-vod-sync-service/src/main/java/hu/user/mcvodsync/service/xls/ImportSummary.java +++ b/mc-vod-sync/mc-vod-sync-service/src/main/java/hu/user/mcvodsync/service/xls/ImportSummary.java @@ -28,26 +28,36 @@ public class ImportSummary { private long deleted; + private long insertedPlaylist; + + private long updatedPlaylist; + + private long deletedPlaylist; + public String toString() { Duration executionDuration = Duration.between(started, finished); String duration = DurationFormatUtils.formatDuration(executionDuration.toMillis(), "H:mm:ss", true); return String.format("Execution started at %s, finished at %s.", started, finished) + System.lineSeparator() + - String.format("Process execution took %s on %d rows", duration, all) + + String.format("Process execution took %s on %d rows.", duration, all) + System.lineSeparator() + - String.format("Success count is %d, error count is %d", success, error) + + String.format("Success count: %d, error count: %d", success, error) + System.lineSeparator() + - String.format("Insert count is %d, update count is %d, delete count is %d", inserted, updated, deleted); + String.format("Insert count: %d, update count: %d, delete count: %d", inserted, updated, deleted); } - public void incError() { - error++; + public void incAll() { + all++; } public void incSuccess() { success++; } + public void incError() { + error++; + } + public void incInserted() { inserted++; } @@ -56,11 +66,16 @@ public class ImportSummary { updated++; } - public void incDeleted() { - inserted++; + public void incInsertedPlaylist() { + insertedPlaylist++; } - public void incAll() { - all++; + public void incUpdatedPlaylist() { + updatedPlaylist++; + } + + public void incDeletedPlaylist() { + deletedPlaylist++; } + } diff --git a/mc-vod-sync/mc-vod-sync-service/src/main/java/hu/user/mcvodsync/service/xls/VodXlsProcessor.java b/mc-vod-sync/mc-vod-sync-service/src/main/java/hu/user/mcvodsync/service/xls/VodXlsProcessor.java index 0d557c28..fe856c06 100644 --- a/mc-vod-sync/mc-vod-sync-service/src/main/java/hu/user/mcvodsync/service/xls/VodXlsProcessor.java +++ b/mc-vod-sync/mc-vod-sync-service/src/main/java/hu/user/mcvodsync/service/xls/VodXlsProcessor.java @@ -9,6 +9,7 @@ import org.dhatim.fastexcel.reader.Row; import org.dhatim.fastexcel.reader.Sheet; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.stereotype.Service; +import org.springframework.transaction.annotation.Transactional; import java.io.ByteArrayInputStream; import java.io.IOException; @@ -32,6 +33,7 @@ public class VodXlsProcessor { @Autowired private AssetImportService assetImportService; + @Transactional public ImportSummary process(String fileName, byte[] xlsData) { ImportSummary summary = ImportSummary.builder().started(Instant.now()).build(); try (InputStream is = new ByteArrayInputStream(xlsData); @@ -40,7 +42,7 @@ public class VodXlsProcessor { try (Stream rows = sheet.openStream()) { rows.forEach(row -> { if (row.getRowNum() == 1) { - assetImportService.prepare(); + assetImportService.prepare(summary); headers = getHeaders(row); } else { summary.incAll(); @@ -78,7 +80,7 @@ public class VodXlsProcessor { try { asset = assetMapper.toEntity(rowData); if (isAssetValid(asset)) { - assetImportService.processAsset(asset, summary); + assetImportService.processAsset(asset); } summary.incSuccess(); } catch (Exception e) {