From ae8026c165bff4e5056f1a8ae02520f30369c945 Mon Sep 17 00:00:00 2001 From: =?utf8?q?V=C3=A1s=C3=A1ry=20D=C3=A1niel?= Date: Wed, 10 Jan 2024 16:40:36 +0100 Subject: [PATCH] v0.0.3 --- mc-vod-sync/QUESTIONS.md | 3 + mc-vod-sync/mc-vod-sync-app/pom.xml | 2 +- .../src/main/resources/application-dev.yaml | 16 +- .../src/main/resources/application.yaml | 73 +++---- .../src/main/resources/logback-prod.xml | 33 ++++ .../src/main/resources/logback-spring.xml | 15 +- .../java/hu/user/mcvodsync/AssetExportIT.java | 74 +++++++ .../user/mcvodsync/AssetImportExportBase.java | 81 ++++++-- ...ImportExportIT.java => AssetImportIT.java} | 57 ++++-- .../java/hu/user/mcvodsync/AssetMapperIT.java | 7 +- .../hu/user/mcvodsync/BrightCoveClientIT.java | 38 +++- ...rviceTest.java => EmailSendServiceIT.java} | 19 +- .../hu/user/mcvodsync/ExecutionLockIT.java | 6 +- .../java/hu/user/mcvodsync/RepositoryIT.java | 13 +- .../resources/testdata-duplicate-video.xlsx | Bin 0 -> 16773 bytes .../mcvodsync/brightcove/ApiProperties.java | 2 + .../brightcove/BrightCoveClient.java | 108 +++++++++-- .../mc-vod-sync-db/migrations/bootstrap.sql | 6 +- .../scripts/003_create_business_tables.sql | 23 ++- .../main/java/hu/user/mcvodsync/db/Asset.java | 34 +++- .../java/hu/user/mcvodsync/db/AssetSync.java | 4 +- .../java/hu/user/mcvodsync/db/Associate.java | 10 +- .../hu/user/mcvodsync/db/PlaylistSync.java | 4 +- .../java/hu/user/mcvodsync/db/UploadFile.java | 10 +- .../db/converter/StringListConverter.java | 19 +- .../db/repository/AssetRepository.java | 24 ++- .../db/repository/AssetSyncRepository.java | 2 + .../db/repository/PlaylistSyncRepository.java | 2 + .../mcvodsync/service/ServiceProperties.java | 13 +- .../service/config/ScheduledTasks.java | 71 ++++++- .../service/data/CompositeSummary.java | 56 ++++++ .../user/mcvodsync/service/data/Summary.java | 53 +++++ .../service/event/ExportCompletedEvent.java | 6 +- .../service/event/ExportStartedEvent.java | 15 ++ .../service/event/ImportCompletedEvent.java | 6 +- .../service/in/AssetImportService.java | 78 ++++---- .../mcvodsync/service/in/AssetMapper.java | 7 + .../mcvodsync/service/in/ImportSummary.java | 81 -------- .../mcvodsync/service/in/VodXlsProcessor.java | 27 +-- .../service/mail/EmailSendService.java | 31 ++- ...ortService.java => SyncExportService.java} | 138 +++++++------ .../mcvodsync/service/out/SyncProcessor.java | 83 ++++++++ .../mcvodsync/service/out/VideoMapper.java | 65 +++++-- .../service/schedule/ScheduledExport.java | 16 +- .../service/schedule/ScheduledImport.java | 6 +- ...acker.java => ScheduledSunsetChacker.java} | 2 +- .../mcvodsync/service/time/TimeUtils.java | 14 ++ .../mcvodsync/ui/ApplicationProperties.java | 1 + .../java/hu/user/mcvodsync/ui/Constants.java | 3 +- .../ui/config/ResourceConfigurer.java | 2 +- .../ui/config/WebSecurityConfig.java | 2 +- .../ui/data/BrightCovePlaylistDataModel.java | 6 +- .../ui/data/BrightCoveVideoDataModel.java | 47 ++++- .../ui/view/ExportDataViewModel.java | 45 ----- .../mcvodsync/ui/view/ImportEventHandler.java | 2 - .../mcvodsync/ui/view/IndexViewModel.java | 18 +- ...tFileViewModel.java => MainViewModel.java} | 11 +- .../src/main/resources/metainfo/zk/zk.xml | 12 +- .../src/main/resources/web/associates.zul | 38 ++-- .../src/main/resources/web/browse-remote.zul | 45 ++++- .../main/resources/web/form/export-stats.zul | 181 ++++++++++++++++++ .../main/resources/web/form/import-stats.zul | 180 +++++++++++++++++ .../src/main/resources/web/form/test.zul | 3 + .../src/main/resources/web/import-file.zul | 131 ------------- .../src/main/resources/web/index.zul | 37 ++-- .../src/main/resources/web/main.zul | 62 ++++++ 66 files changed, 1630 insertions(+), 649 deletions(-) create mode 100644 mc-vod-sync/mc-vod-sync-app/src/main/resources/logback-prod.xml create mode 100644 mc-vod-sync/mc-vod-sync-app/src/test/java/hu/user/mcvodsync/AssetExportIT.java rename mc-vod-sync/mc-vod-sync-app/src/test/java/hu/user/mcvodsync/{AssetImportExportIT.java => AssetImportIT.java} (55%) rename mc-vod-sync/mc-vod-sync-app/src/test/java/hu/user/mcvodsync/{EmailSendServiceTest.java => EmailSendServiceIT.java} (53%) create mode 100644 mc-vod-sync/mc-vod-sync-app/src/test/resources/testdata-duplicate-video.xlsx create mode 100644 mc-vod-sync/mc-vod-sync-service/src/main/java/hu/user/mcvodsync/service/data/CompositeSummary.java create mode 100644 mc-vod-sync/mc-vod-sync-service/src/main/java/hu/user/mcvodsync/service/data/Summary.java create mode 100644 mc-vod-sync/mc-vod-sync-service/src/main/java/hu/user/mcvodsync/service/event/ExportStartedEvent.java delete mode 100644 mc-vod-sync/mc-vod-sync-service/src/main/java/hu/user/mcvodsync/service/in/ImportSummary.java rename mc-vod-sync/mc-vod-sync-service/src/main/java/hu/user/mcvodsync/service/out/{AssetExportService.java => SyncExportService.java} (62%) create mode 100644 mc-vod-sync/mc-vod-sync-service/src/main/java/hu/user/mcvodsync/service/out/SyncProcessor.java rename mc-vod-sync/mc-vod-sync-service/src/main/java/hu/user/mcvodsync/service/schedule/{ScheduledSunriseChacker.java => ScheduledSunsetChacker.java} (91%) create mode 100644 mc-vod-sync/mc-vod-sync-service/src/main/java/hu/user/mcvodsync/service/time/TimeUtils.java delete mode 100644 mc-vod-sync/mc-vod-sync-ui/src/main/java/hu/user/mcvodsync/ui/view/ExportDataViewModel.java rename mc-vod-sync/mc-vod-sync-ui/src/main/java/hu/user/mcvodsync/ui/view/{ImportFileViewModel.java => MainViewModel.java} (91%) create mode 100644 mc-vod-sync/mc-vod-sync-ui/src/main/resources/web/form/export-stats.zul create mode 100644 mc-vod-sync/mc-vod-sync-ui/src/main/resources/web/form/import-stats.zul create mode 100644 mc-vod-sync/mc-vod-sync-ui/src/main/resources/web/form/test.zul delete mode 100644 mc-vod-sync/mc-vod-sync-ui/src/main/resources/web/import-file.zul create mode 100644 mc-vod-sync/mc-vod-sync-ui/src/main/resources/web/main.zul diff --git a/mc-vod-sync/QUESTIONS.md b/mc-vod-sync/QUESTIONS.md index d1c780a2..c8db0b0f 100644 --- a/mc-vod-sync/QUESTIONS.md +++ b/mc-vod-sync/QUESTIONS.md @@ -1,3 +1,6 @@ +- Az extra notes hogyan értelmezendő? +- Egy video lehet e többször hozzáadva gyan ahhoz a listához? Ha nem ,kéne csekkolni betöltésnél... + actors_talent, Actors/Talent, , string, null required: false director, Director, , string, null required: false episode, Episode, , diff --git a/mc-vod-sync/mc-vod-sync-app/pom.xml b/mc-vod-sync/mc-vod-sync-app/pom.xml index 7bf9f2ce..b73d56eb 100644 --- a/mc-vod-sync/mc-vod-sync-app/pom.xml +++ b/mc-vod-sync/mc-vod-sync-app/pom.xml @@ -3,7 +3,7 @@ xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 https://maven.apache.org/xsd/maven-4.0.0.xsd"> 4.0.0 mc-vod-sync-app - 0.0.1-SNAPSHOT + 0.0.3 hu.user mc-vod-sync diff --git a/mc-vod-sync/mc-vod-sync-app/src/main/resources/application-dev.yaml b/mc-vod-sync/mc-vod-sync-app/src/main/resources/application-dev.yaml index e5fc5a9c..f2b12d1b 100644 --- a/mc-vod-sync/mc-vod-sync-app/src/main/resources/application-dev.yaml +++ b/mc-vod-sync/mc-vod-sync-app/src/main/resources/application-dev.yaml @@ -25,13 +25,14 @@ spring: username: db2admin password: password mail: - host: mx.in.useribm.hu +# host: mx.in.useribm.hu # port: 587 # spring.mail.username= # spring.mail.password= # spring.mail.properties.mail.smtp.auth=true # spring.mail.properties.mail.smtp.starttls.enable=true logging: + config: level: org.hibernate.engine.jdbc.spi.SqlExceptionHelper: ERROR # org.springframework.web.client.RestTemplate: DEBUG @@ -39,19 +40,24 @@ logging: # httpclient.wire: DEBUG mc-vod-sync: service: - target-video-insert-enabled: true - target-playlist-insert-enabled: true + target-video-insert-enabled: false + target-playlist-insert-enabled: false scheduler: import-enabled: false export-enabled: false - sunrise-checker-enabled: false + sunset-checker-enabled: false import-cron: 0 0 4 1/1 * ? * export-cron: 0 0 4 1/1 * ? * - sunrise-checker-cron: 0 0 4 1/1 * ? * + sunset-checker-cron: 0 0 4 1/1 * ? * + report: + recipient: vasary@elgekko.net + sender: noreply@mc-vod-sync application: user-name: user password: password + admin: true api: + base-url: https://cms.api.brightcove.com token-request-url: https://oauth.brightcove.com/v4/access_token client-id: 0337034f-bde3-44d4-81ba-6a4566ca8ba0 client-secret: 03Az2W6BSj4sTPSU7SVnRcJ6NDmeaFmB7EPbmuZFnt5slaC2bbscBOMvKR03CGVMxaVXUo9ao6e_gLvxUCjtgg diff --git a/mc-vod-sync/mc-vod-sync-app/src/main/resources/application.yaml b/mc-vod-sync/mc-vod-sync-app/src/main/resources/application.yaml index 39be7218..00b7dec1 100644 --- a/mc-vod-sync/mc-vod-sync-app/src/main/resources/application.yaml +++ b/mc-vod-sync/mc-vod-sync-app/src/main/resources/application.yaml @@ -1,5 +1,5 @@ server: - port: 8080 + port: 80 servlet: context-path: / zk: @@ -7,9 +7,7 @@ zk: zul-view-resolver-enabled: true spring: jpa: - # hibernate: - # use-new-id-generator-mappings: true - show-sql: true + show-sql: false properties: hibernate: format_sql: true @@ -20,37 +18,40 @@ spring: enabled: always datasource: type: com.zaxxer.hikari.HikariDataSource - url: jdbc:db2://dvdev.in.useribm.hu:50000/vodsync + url: jdbc:db2://localhost:25000/vodsync + retrieveMessagesFromServerOnGetMessage: true username: db2admin - password: password -camunda.bpm: - generic-properties.properties: - telemetry-reporter-activate: false - job-executor-acquire-by-priority: true - job-execution: - core-pool-size: 10 - #lock-time-in-millis: 600000 - database: - type: db2 - schema-update: false - table-prefix: CAMUNDA. - schema-name: CAMUNDA - webapp: - enabled: true - index-redirect-enabled: false - admin-user: - id: kermit - password: password - firstName: Kermit - filter: - create: All tasks - job-execution.enabled: true + password: Password01 + mail: + host: mx.in.useribm.hu +# port: 587 +# username= +# password= +# properties.mail.smtp.auth=true +# properties.mail.smtp.starttls.enable=true logging: - # config: classpath:log4j2.xml - level: - org.hibernate.engine.jdbc.spi.SqlExceptionHelper: ERROR -# org.springframework.security: DEBUG -# org.springframework.security.web: INFO -# pattern: -# console: "%d %-5level %logger : %msg%n" -# file: "%d %-5level [%thread] %logger : %msg%n" \ No newline at end of file + config: logback-prod.xml +mc-vod-sync: + service: + target-video-insert-enabled: false + target-playlist-insert-enabled: false + scheduler: + import-enabled: false + export-enabled: false + sunset-checker-enabled: false + import-cron: 0 0 4 1/1 * ? * + export-cron: 0 0 4 1/1 * ? * + sunset-checker-cron: 0 0 4 1/1 * ? * + report: + recipient: vasary@elgekko.net + sender: noreply@mc-vod-sync + application: + user-name: user + password: password + admin: true + api: + base-url: https://cms.api.brightcove.com + token-request-url: https://oauth.brightcove.com/v4/access_token + client-id: 0337034f-bde3-44d4-81ba-6a4566ca8ba0 + client-secret: 03Az2W6BSj4sTPSU7SVnRcJ6NDmeaFmB7EPbmuZFnt5slaC2bbscBOMvKR03CGVMxaVXUo9ao6e_gLvxUCjtgg + account-id: 6302605131001 diff --git a/mc-vod-sync/mc-vod-sync-app/src/main/resources/logback-prod.xml b/mc-vod-sync/mc-vod-sync-app/src/main/resources/logback-prod.xml new file mode 100644 index 00000000..049a51f7 --- /dev/null +++ b/mc-vod-sync/mc-vod-sync-app/src/main/resources/logback-prod.xml @@ -0,0 +1,33 @@ + + + + + + + + + %black(%d{ISO8601}) %highlight(%-5level) [%blue(%t)] %yellow(%C{1.}): %msg%n%throwable + + + + + + ${LOGS}/${LOGNAME}.log + + %d %p %C{1.} [%t] %m%n + + + ${LOGS}/archived/${LOGNAME}-%d{yyyy-MM-dd}.log.zip + + + + + + + + + + + + + \ No newline at end of file diff --git a/mc-vod-sync/mc-vod-sync-app/src/main/resources/logback-spring.xml b/mc-vod-sync/mc-vod-sync-app/src/main/resources/logback-spring.xml index 153ccc4f..d20bc1c5 100644 --- a/mc-vod-sync/mc-vod-sync-app/src/main/resources/logback-spring.xml +++ b/mc-vod-sync/mc-vod-sync-app/src/main/resources/logback-spring.xml @@ -1,12 +1,12 @@ - + - %black(%d{ISO8601}) %highlight(%-5level) [%blue(%t)] %yellow(%C{1.}): %msg%n%throwable + %black(%d{ISO8601}) %highlight(%-5level) [%blue(%t)] %yellow(%C{1.}): [%F:%L] %msg%n %throwable @@ -22,11 +22,14 @@ - - - + + + + + + - + diff --git a/mc-vod-sync/mc-vod-sync-app/src/test/java/hu/user/mcvodsync/AssetExportIT.java b/mc-vod-sync/mc-vod-sync-app/src/test/java/hu/user/mcvodsync/AssetExportIT.java new file mode 100644 index 00000000..b89cec87 --- /dev/null +++ b/mc-vod-sync/mc-vod-sync-app/src/test/java/hu/user/mcvodsync/AssetExportIT.java @@ -0,0 +1,74 @@ +/* + * Copyright (c) $today.year-$today.month-24. + * By elGekko + */ + +package hu.user.mcvodsync; + +import lombok.extern.log4j.Log4j2; +import org.junit.Test; +import org.junit.runner.RunWith; +import org.springframework.boot.test.context.SpringBootTest; +import org.springframework.test.context.ActiveProfiles; +import org.springframework.test.context.TestPropertySource; +import org.springframework.test.context.junit4.SpringRunner; + + +@Log4j2 +@RunWith(SpringRunner.class) +@SpringBootTest +@ActiveProfiles("dev") +@TestPropertySource("classpath:application-dev.yaml") +public class AssetExportIT extends AssetImportExportBase { + + @Test + public void initialLoadTest() throws Exception { + cleanupLocal(); + importFile("src/test/resources/testdata.xlsx"); + cleanupRemote(); + exportData(); + assertSynchronized(); + } + + @Test + public void changeVideoTest() throws Exception { + cleanupLocal(); + importFile("src/test/resources/testdata.xlsx"); + importFile("src/test/resources/testdata-change-video.xlsx"); + cleanupRemote(); + exportData(); + assertSynchronized(); + } + + @Test + public void deleteVideoTest() throws Exception { + cleanupLocal(); + importFile("src/test/resources/testdata.xlsx"); + importFile("src/test/resources/testdata-delete-video.xlsx"); + cleanupRemote(); + exportData(); + assertSynchronized(); + } + + @Test + public void changePlaylistOrderTest() throws Exception { + cleanupLocal(); + importFile("src/test/resources/testdata.xlsx"); + importFile("src/test/resources/testdata-change-playlist-order.xlsx"); + cleanupRemote(); + exportData(); + assertSynchronized(); + } + + @Test + public void duplicateVideoTest() throws Exception { + cleanupLocal(); + importFile("src/test/resources/testdata.xlsx"); + importFile("src/test/resources/testdata-duplicate-video.xlsx"); + cleanupRemote(); + exportData(); + assertSynchronized(); + } + + +} diff --git a/mc-vod-sync/mc-vod-sync-app/src/test/java/hu/user/mcvodsync/AssetImportExportBase.java b/mc-vod-sync/mc-vod-sync-app/src/test/java/hu/user/mcvodsync/AssetImportExportBase.java index 050b363e..43e93735 100644 --- a/mc-vod-sync/mc-vod-sync-app/src/test/java/hu/user/mcvodsync/AssetImportExportBase.java +++ b/mc-vod-sync/mc-vod-sync-app/src/test/java/hu/user/mcvodsync/AssetImportExportBase.java @@ -15,10 +15,11 @@ import hu.user.mcvodsync.db.SyncType; import hu.user.mcvodsync.db.repository.AssetRepository; import hu.user.mcvodsync.db.repository.AssetSyncRepository; import hu.user.mcvodsync.db.repository.PlaylistSyncRepository; +import hu.user.mcvodsync.service.data.CompositeSummary; +import hu.user.mcvodsync.service.data.Summary; import hu.user.mcvodsync.service.in.AssetImportService; -import hu.user.mcvodsync.service.in.ImportSummary; import hu.user.mcvodsync.service.in.VodXlsProcessor; -import hu.user.mcvodsync.service.out.AssetExportService; +import hu.user.mcvodsync.service.out.SyncProcessor; import hu.user.mcvodsync.service.out.VideoMapper; import lombok.extern.log4j.Log4j2; import org.apache.commons.lang3.StringUtils; @@ -55,7 +56,7 @@ public class AssetImportExportBase { private AssetImportService assetImportService; @Autowired - private AssetExportService assetExportService; + private SyncProcessor syncProcessor; @Autowired private BrightCoveClient brightCoveClient; @@ -66,22 +67,41 @@ public class AssetImportExportBase { protected String getPlaylistSyncIds(String name) { List playlists = playlistSyncRepository.findAllByName(name); assertEquals(1, playlists.size()); - return StringUtils.join(playlists.get(0).getIds(), ","); + List catalogIds = playlists.get(0).getIds().stream().map(assetRepository::findCatalogIdById).collect(Collectors.toList()); + return StringUtils.join(catalogIds, ","); } - protected void cleanup() { + protected void cleanup() throws Exception { cleanupLocal(); cleanupRemote(); assetRepository.deleteAllInBatch(); } - protected void cleanupRemote() { + protected void cleanupRemote() throws InterruptedException { log.info("cleanupRemote"); cleanupRemotePlaylists(); + waitForPlaylistCleanupCompleted(); + Thread.sleep(3000); cleanupRemoteVideos(); + waitForVideoCleanupCompleted(); log.info("cleanupRemote OK"); } + private void waitForVideoCleanupCompleted() { + while (true) { + int count = 0; + PagedSearch