From c015908ad43d6fa594bc07a4d4f413508d793edb Mon Sep 17 00:00:00 2001 From: =?utf8?q?V=C3=A1s=C3=A1ry=20D=C3=A1niel?= Date: Fri, 1 Dec 2023 14:41:09 +0100 Subject: [PATCH] Export completed (no statistics yet), import tests done --- .../src/main/resources/application-dev.yaml | 26 +-- .../user/mcvodsync/AssetImportExportBase.java | 89 ++++++++++ .../user/mcvodsync/AssetImportExportIT.java | 96 ++++++++++ .../java/hu/user/mcvodsync/AssetMapperIT.java | 2 +- .../hu/user/mcvodsync/BrightCoveClientIT.java | 87 +++++---- .../hu/user/mcvodsync/ExecutionLockIT.java | 45 +++++ .../java/hu/user/mcvodsync/RepositoryIT.java | 22 ++- .../hu/user/mcvodsync/VodXlsProcessorIT.java | 166 ------------------ .../brightcove/BrightCoveClient.java | 39 ++-- .../scripts/003_create_business_tables.sql | 3 +- .../hu/user/mcvodsync/db/PlaylistSync.java | 7 +- .../db/converter/StringListConverter.java | 23 +++ .../db/repository/AssetSyncRepository.java | 2 + .../db/repository/PlaylistSyncRepository.java | 5 +- .../service/config/ScheduledTasks.java | 38 ++++ .../service/config/TaskSchedulerConfig.java | 18 ++ .../{import => in}/AssetImportService.java | 32 ++-- .../service/{import => in}/AssetMapper.java | 2 +- .../service/{import => in}/ImportSummary.java | 2 +- .../{import => in}/VodXlsProcessor.java | 2 +- .../{export => out}/AssetExportService.java | 75 ++++++-- .../CustomPropertiesTranslator.java | 2 +- .../service/{export => out}/VideoMapper.java | 2 +- .../service/schedule/ScheduledExport.java | 30 ++++ .../service/schedule/ScheduledImport.java | 28 +++ .../schedule/ScheduledSunriseChacker.java | 29 +++ .../service/schedule/TaskSchedules.java | 14 -- 27 files changed, 601 insertions(+), 285 deletions(-) create mode 100644 mc-vod-sync/mc-vod-sync-app/src/test/java/hu/user/mcvodsync/AssetImportExportBase.java create mode 100644 mc-vod-sync/mc-vod-sync-app/src/test/java/hu/user/mcvodsync/AssetImportExportIT.java create mode 100644 mc-vod-sync/mc-vod-sync-app/src/test/java/hu/user/mcvodsync/ExecutionLockIT.java delete mode 100644 mc-vod-sync/mc-vod-sync-app/src/test/java/hu/user/mcvodsync/VodXlsProcessorIT.java create mode 100644 mc-vod-sync/mc-vod-sync-db/src/main/java/hu/user/mcvodsync/db/converter/StringListConverter.java create mode 100644 mc-vod-sync/mc-vod-sync-service/src/main/java/hu/user/mcvodsync/service/config/ScheduledTasks.java create mode 100644 mc-vod-sync/mc-vod-sync-service/src/main/java/hu/user/mcvodsync/service/config/TaskSchedulerConfig.java rename mc-vod-sync/mc-vod-sync-service/src/main/java/hu/user/mcvodsync/service/{import => in}/AssetImportService.java (86%) rename mc-vod-sync/mc-vod-sync-service/src/main/java/hu/user/mcvodsync/service/{import => in}/AssetMapper.java (98%) rename mc-vod-sync/mc-vod-sync-service/src/main/java/hu/user/mcvodsync/service/{import => in}/ImportSummary.java (97%) rename mc-vod-sync/mc-vod-sync-service/src/main/java/hu/user/mcvodsync/service/{import => in}/VodXlsProcessor.java (98%) rename mc-vod-sync/mc-vod-sync-service/src/main/java/hu/user/mcvodsync/service/{export => out}/AssetExportService.java (60%) rename mc-vod-sync/mc-vod-sync-service/src/main/java/hu/user/mcvodsync/service/{export => out}/CustomPropertiesTranslator.java (72%) rename mc-vod-sync/mc-vod-sync-service/src/main/java/hu/user/mcvodsync/service/{export => out}/VideoMapper.java (98%) create mode 100644 mc-vod-sync/mc-vod-sync-service/src/main/java/hu/user/mcvodsync/service/schedule/ScheduledExport.java create mode 100644 mc-vod-sync/mc-vod-sync-service/src/main/java/hu/user/mcvodsync/service/schedule/ScheduledImport.java create mode 100644 mc-vod-sync/mc-vod-sync-service/src/main/java/hu/user/mcvodsync/service/schedule/ScheduledSunriseChacker.java delete mode 100644 mc-vod-sync/mc-vod-sync-service/src/main/java/hu/user/mcvodsync/service/schedule/TaskSchedules.java diff --git a/mc-vod-sync/mc-vod-sync-app/src/main/resources/application-dev.yaml b/mc-vod-sync/mc-vod-sync-app/src/main/resources/application-dev.yaml index 167d87e6..7a5ee750 100644 --- a/mc-vod-sync/mc-vod-sync-app/src/main/resources/application-dev.yaml +++ b/mc-vod-sync/mc-vod-sync-app/src/main/resources/application-dev.yaml @@ -31,32 +31,14 @@ spring: # spring.mail.password= # spring.mail.properties.mail.smtp.auth=true # spring.mail.properties.mail.smtp.starttls.enable=true -camunda.bpm: - generic-properties.properties: - telemetry-reporter-activate: false - job-executor-acquire-by-priority: true - job-execution: - core-pool-size: 10 - #lock-time-in-millis: 600000 - database: - type: db2 - schema-update: false - table-prefix: CAMUNDA. - schema-name: CAMUNDA - webapp: - enabled: true - index-redirect-enabled: false - admin-user: - id: kermit - password: password - firstName: Kermit - filter: - create: All tasks - job-execution.enabled: true logging: level: org.hibernate.engine.jdbc.spi.SqlExceptionHelper: ERROR mc-vod-sync: + scheduler: + import: true + export: true + sunrise-checker: true application: user-name: user password: password diff --git a/mc-vod-sync/mc-vod-sync-app/src/test/java/hu/user/mcvodsync/AssetImportExportBase.java b/mc-vod-sync/mc-vod-sync-app/src/test/java/hu/user/mcvodsync/AssetImportExportBase.java new file mode 100644 index 00000000..7d50548a --- /dev/null +++ b/mc-vod-sync/mc-vod-sync-app/src/test/java/hu/user/mcvodsync/AssetImportExportBase.java @@ -0,0 +1,89 @@ +/* + * Copyright (c) $today.year-$today.month-24. + * By elGekko + */ + +package hu.user.mcvodsync; + +import hu.user.mcvodsync.db.PlaylistSync; +import hu.user.mcvodsync.db.SyncType; +import hu.user.mcvodsync.db.repository.AssetRepository; +import hu.user.mcvodsync.db.repository.AssetSyncRepository; +import hu.user.mcvodsync.db.repository.PlaylistSyncRepository; +import hu.user.mcvodsync.service.in.ImportSummary; +import hu.user.mcvodsync.service.in.VodXlsProcessor; +import lombok.extern.log4j.Log4j2; +import org.apache.commons.lang3.StringUtils; +import org.springframework.beans.factory.annotation.Autowired; + +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; +import java.util.List; + +import static org.junit.Assert.assertEquals; + + +@Log4j2 +public class AssetImportExportBase { + @Autowired + private VodXlsProcessor vodXlsProcessor; + + @Autowired + private AssetRepository assetRepository; + + @Autowired + private AssetSyncRepository assetSyncRepository; + + @Autowired + private PlaylistSyncRepository playlistSyncRepository; + + protected String getPlaylistSyncIds(String name) { + List playlists = playlistSyncRepository.findAllByName(name); + assertEquals(1, playlists.size()); + return StringUtils.join(playlists.get(0).getIds(), ","); + } + + protected void cleanup() { + cleanupSync(); + assetRepository.deleteAllInBatch(); + } + + protected void cleanupSync() { + playlistSyncRepository.deleteAllInBatch(); + assetSyncRepository.deleteAllInBatch(); + } + + protected ImportSummary process(String xlsFile) { + ImportSummary summary = null; + try { + Path input = Paths.get(xlsFile); + summary = vodXlsProcessor.process(input.getFileName().toString(), Files.readAllBytes(input)); + } catch (IOException e) { + log.error(e); + throw new RuntimeException(e); + } + return summary; + } + + protected void checkSummary(ImportSummary summary, long expectedAll, long expectedInsert, long expectedUpdate, long expectedDelete) { + assertEquals(expectedAll, summary.getAll()); + assertEquals(expectedAll, summary.getSuccess()); + assertEquals(expectedInsert, summary.getInserted()); + assertEquals(expectedUpdate, summary.getUpdated()); + assertEquals(expectedDelete, summary.getDeleted()); + } + + protected void checkAsset(long expected, SyncType syncType, long expectedAllCount) { + assertEquals(expected, assetSyncRepository.countBySyncType(syncType)); + assertEquals(expectedAllCount, assetSyncRepository.count()); + } + + protected void checkPlaylist(long expected, SyncType syncType, long expectedAllCount) { + assertEquals(expected, playlistSyncRepository.countBySyncType(syncType)); + assertEquals(expectedAllCount, playlistSyncRepository.count()); + } + + +} diff --git a/mc-vod-sync/mc-vod-sync-app/src/test/java/hu/user/mcvodsync/AssetImportExportIT.java b/mc-vod-sync/mc-vod-sync-app/src/test/java/hu/user/mcvodsync/AssetImportExportIT.java new file mode 100644 index 00000000..2f89a555 --- /dev/null +++ b/mc-vod-sync/mc-vod-sync-app/src/test/java/hu/user/mcvodsync/AssetImportExportIT.java @@ -0,0 +1,96 @@ +/* + * Copyright (c) $today.year-$today.month-24. + * By elGekko + */ + +package hu.user.mcvodsync; + +import hu.user.mcvodsync.db.SyncType; +import hu.user.mcvodsync.service.in.ImportSummary; +import lombok.extern.log4j.Log4j2; +import org.junit.AfterClass; +import org.junit.BeforeClass; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.springframework.boot.test.context.SpringBootTest; +import org.springframework.test.context.ActiveProfiles; +import org.springframework.test.context.TestPropertySource; +import org.springframework.test.context.junit4.SpringRunner; + +import javax.annotation.PostConstruct; +import java.io.IOException; + +import static org.junit.Assert.assertEquals; + + +@Log4j2 +@RunWith(SpringRunner.class) +@SpringBootTest +@ActiveProfiles("dev") +@TestPropertySource("classpath:application-dev.yaml") +public class AssetImportExportIT extends AssetImportExportBase { + + @BeforeClass + public static void beforeClass() { + } + + @AfterClass + public static void afterClass() { + } + + @PostConstruct + public void init() { + } + + @Test + public void initialLoadTest() throws IOException { +// Path xlsFile = Paths.get("src/test/resources/testdata-big.xlsx"); + log.info("initialLoadTest"); + cleanup(); + ImportSummary summary = process("src/test/resources/testdata.xlsx"); + checkSummary(summary, 9, 9, 0, 0); + checkAsset(9, SyncType.INSERT, 9); + checkPlaylist(3, SyncType.INSERT, 3); + assertEquals("CCEM110180,CCEM110183,CCEM110068", getPlaylistSyncIds("Playlist1")); + assertEquals("CCEM128963,CCEM128965,CCEM128967", getPlaylistSyncIds("Playlist2")); + assertEquals("CCEF008678,CCEM001148,CCEM072086", getPlaylistSyncIds("Playlist3")); + } + + @Test + public void changeVideoTest() throws IOException { + log.info("changeVideoTest"); + ImportSummary summary = process("src/test/resources/testdata.xlsx"); + cleanupSync(); + summary = process("src/test/resources/testdata-change-video.xlsx"); + checkSummary(summary, 9, 0, 1, 0); + checkAsset(1, SyncType.UPDATE, 1); + checkPlaylist(0, null, 0); + } + + @Test + public void deleteVideoTest() { + log.info("deleteVideoTest"); + ImportSummary summary = process("src/test/resources/testdata.xlsx"); + cleanupSync(); + summary = process("src/test/resources/testdata-delete-video.xlsx"); + checkSummary(summary, 9, 0, 1, 1); + checkAsset(1, SyncType.UPDATE, 2); + checkAsset(1, SyncType.DELETE, 2); + checkPlaylist(1, SyncType.UPDATE, 1); + assertEquals("CCEM110180,CCEM110183", getPlaylistSyncIds("Playlist1")); + } + + @Test + public void changePlaylistOrderTest() { + log.info("changePlaylistOrderTest"); + ImportSummary summary = process("src/test/resources/testdata.xlsx"); + cleanupSync(); + summary = process("src/test/resources/testdata-change-playlist-order.xlsx"); + checkSummary(summary, 8, 0, 1, 0); + checkAsset(1, SyncType.UPDATE, 1); + checkPlaylist(1, SyncType.UPDATE, 1); + assertEquals("CCEM001148,CCEM072086,CCEF008678", getPlaylistSyncIds("Playlist3")); + } + + +} diff --git a/mc-vod-sync/mc-vod-sync-app/src/test/java/hu/user/mcvodsync/AssetMapperIT.java b/mc-vod-sync/mc-vod-sync-app/src/test/java/hu/user/mcvodsync/AssetMapperIT.java index 44c4c855..b11b2f71 100644 --- a/mc-vod-sync/mc-vod-sync-app/src/test/java/hu/user/mcvodsync/AssetMapperIT.java +++ b/mc-vod-sync/mc-vod-sync-app/src/test/java/hu/user/mcvodsync/AssetMapperIT.java @@ -7,7 +7,7 @@ package hu.user.mcvodsync; import com.brightcove.cms.client.model.Video; import hu.user.mcvodsync.db.Asset; -import hu.user.mcvodsync.service.export.VideoMapper; +import hu.user.mcvodsync.service.out.VideoMapper; import lombok.extern.log4j.Log4j2; import org.junit.Test; import org.junit.runner.RunWith; diff --git a/mc-vod-sync/mc-vod-sync-app/src/test/java/hu/user/mcvodsync/BrightCoveClientIT.java b/mc-vod-sync/mc-vod-sync-app/src/test/java/hu/user/mcvodsync/BrightCoveClientIT.java index 54f58743..3d42a191 100644 --- a/mc-vod-sync/mc-vod-sync-app/src/test/java/hu/user/mcvodsync/BrightCoveClientIT.java +++ b/mc-vod-sync/mc-vod-sync-app/src/test/java/hu/user/mcvodsync/BrightCoveClientIT.java @@ -1,8 +1,6 @@ package hu.user.mcvodsync; -import com.brightcove.cms.client.model.CreateVideoRequestBodyFields; import com.brightcove.cms.client.model.Playlist; -import com.brightcove.cms.client.model.PlaylistInputFields; import com.brightcove.cms.client.model.Video; import com.google.common.collect.ImmutableMap; import hu.user.mcvodsync.brightcove.BrightCoveClient; @@ -13,6 +11,7 @@ import org.junit.runner.RunWith; import org.springframework.beans.factory.annotation.Autowired; import org.springframework.boot.test.context.SpringBootTest; import org.springframework.context.annotation.ComponentScan; +import org.springframework.http.HttpStatus; import org.springframework.test.context.ActiveProfiles; import org.springframework.test.context.TestPropertySource; import org.springframework.test.context.junit4.SpringRunner; @@ -24,6 +23,8 @@ import java.util.Collections; import java.util.List; import java.util.stream.Collectors; +import static org.junit.Assert.assertNotNull; + @Log4j2 @RunWith(SpringRunner.class) @ComponentScan("hu.user.mcvodsync") @@ -55,12 +56,11 @@ public class BrightCoveClientIT { List ids = getVideos(); for (int i = 1; i < 6; i++) { - PlaylistInputFields fields = new PlaylistInputFields(); - fields.setType(PlaylistInputFields.TypeEnum.EXPLICIT); - fields.setName("Playlist 0000" + i); - fields.setVideoIds(ids); - fields.setDescription("mc-vod-sync"); - Playlist playList = bcClient.createPlaylist(fields); + Playlist playlist = new Playlist(); + playlist.setName("Playlist 0000" + i); + playlist.setVideoIds(ids); + playlist.setDescription("mc-vod-sync"); + Playlist playList = bcClient.createPlaylist(playlist); logPlaylist(playList); } @@ -69,6 +69,31 @@ public class BrightCoveClientIT { } } + @Test + public void testDeleteTestPlayLists() { + try { + PagedSearch page = PagedSearch.builder() + .query("description:mc-vod-sync") + .sort("name") + .build(); + while (bcClient.getPlaylistsPaged(page)) { + page.getData().forEach(this::deletePlayList); + } + + } catch (Exception e) { + log.error(e); + } + } + + private void deletePlayList(Playlist playlist) { + try { + logPlaylist(playlist); + bcClient.deletePlaylist(playlist.getId()); + } catch (Exception e) { + log.error(e); + } + } + @Test public void updatePlayLists() { try { @@ -81,12 +106,12 @@ public class BrightCoveClientIT { Playlist playlist = page.getData().get(0); playlist.setVideoIds(getVideos()); -// List videoIds = playlist.getVideoIds(); -// assertNotNull(videoIds); -// String id1 = videoIds.get(0); -// String id2 = videoIds.get(videoIds.size() - 1); -// videoIds.set(0, id2); -// videoIds.set(videoIds.size() - 1, id1); + List videoIds = playlist.getVideoIds(); + assertNotNull(videoIds); + String id1 = videoIds.get(0); + String id2 = videoIds.get(videoIds.size() - 1); + videoIds.set(0, id2); + videoIds.set(videoIds.size() - 1, id1); Playlist playList = bcClient.updatePlaylist(playlist); @@ -153,16 +178,13 @@ public class BrightCoveClientIT { @Test public void testCreateVideo() { try { - CreateVideoRequestBodyFields fields = new CreateVideoRequestBodyFields(); - fields.setName("TEST video 00001"); - fields.setReferenceId("MCVODSYNC10001"); - fields.setState(CreateVideoRequestBodyFields.StateEnum.INACTIVE); - fields.setEconomics(CreateVideoRequestBodyFields.EconomicsEnum.FREE); - fields.setTags(Collections.singletonList("mc-vod-sync")); - fields.setCustomFields(ImmutableMap.of("Notes", "notes")); - Video video = bcClient.createVideo(fields); - logVideo(video); - + Video video = new Video(); + video.setName("TEST video 00001"); + video.setReferenceId("MCVODSYNC10001"); + video.setTags(Collections.singletonList("mc-vod-sync")); + video.setCustomFields(ImmutableMap.of("Notes", "notes")); + Video createdVideo = bcClient.createVideo(video); + logVideo(createdVideo); log.info("Search result"); List