+* A sunset kezelése is kell, tehát jöhet adat későbbi sunset-el?
+* A sunset és sunrise formátuma legyen egyforma (mindegy melyik) minden sorra!
+* Mik a kötelező mezők? Most: playlist, sunset, sunrise, title
+* A sunrise alapján lejárt video-kat a BrightCove API-n keresztül törölni kell, vagy csak a playlist-ből kell
+ eltávolítani?
+* Feltételeze, hogy a sunrise dátum inkluzív, tehát aznap még jogosult a kijátszás, a törlés az aktuális időponttal
+ egyező vagy régebbi sunrise dátumú videókat törli. A manuális működés miatt kellene figyelmeztetés, hogy futtatni kell
+ a szinkronzációt a törlés után (ha volt
+
+
* Ha hiba történik az import vagy a szinkronizálás során a teljes fájl tartalmat el kell vetni (rollback), vagy ami jó
- az frissüljön be?
+ az frissüljön be? ami jó, az frissüljön
* Ha hibás egy sor a táblázatban, akkor a teljes importot el kelll dobni, vagy csak a sort kell kihagyni? pl. üres
- hubinfo vagy episodetitle
-* Mi az Excel mező összerendelés?
-* Milyen fix értéket kapnak az Excelből nem jövő video mezők?
-* Mi a videoId, az API osztja? igen
-* Mi a playlist rendezési elve?
-* Playlist-en belül a sorrend változhat?
-* Egy video több playlist-en is szerepelhet?
-* Mi video asset metadata frissítés API-ja?
-* Video vs Media asset
-* Folder kezelés lesz?
+ hubinfo vagy episodetitle: sor kihagyása
+* Mi az Excel mező összerendelés? mapping tábla
+* Milyen fix értéket kapnak az Excelből nem jövő video mezők? mapping megmondja
+* Mi a videoId, az API osztja? nem, a catalogId az
+* Mi a playlist rendezési elve? seasonnr, epizodenr alapján
+* Playlist-en belül a sorrend változhat? igen
+* Egy video több playlist-en is szerepelhet? igen
-* Video törlése minden listáról:
-DEL https://cms.api.brightcove.com/v1/accounts/{{account_id}}/videos/{{video_id}}/references
+* Video törlése minden listáról:
+ DEL https://cms.api.brightcove.com/v1/accounts/{{account_id}}/videos/{{video_id}}/references
* Video törlése
-https://cms.api.brightcove.com/v1/accounts/{{account_id}}/videos{video_ids}}
+ https://cms.api.brightcove.com/v1/accounts/{{account_id}}/videos{video_ids}}
https://apis.support.brightcove.com/getting-started/getting-started-brightcove-apis.html
https://apis.support.brightcove.com/cms/index.html
package hu.user.mcvodsync;
+import hu.user.mcvodsync.db.repository.AssetRepository;
+import hu.user.mcvodsync.db.repository.AssetSyncRepository;
+import hu.user.mcvodsync.service.xls.AssetImportService;
+import hu.user.mcvodsync.service.xls.ImportSummary;
import lombok.extern.log4j.Log4j2;
import org.junit.Test;
import org.junit.runner.RunWith;
+import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.boot.test.context.SpringBootTest;
import org.springframework.context.annotation.ComponentScan;
import org.springframework.test.context.ActiveProfiles;
import org.springframework.test.context.TestPropertySource;
import org.springframework.test.context.junit4.SpringRunner;
+import java.sql.Date;
+import java.time.LocalDate;
+import java.time.format.DateTimeFormatter;
+import java.util.List;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNotNull;
+
@Log4j2
@RunWith(SpringRunner.class)
@TestPropertySource("classpath:application-dev.yaml")
//@AutoConfigureMockMvc
public class RepositoryIT {
-// @Autowired
-// ServiceRecordRepository serviceRecordRepository;
-// @Autowired
-// ProjectRepository projectRepository;
-// @Autowired
-// ProjectService projectService;
-// @Autowired
-// TreasuryService treasuryService;
-// @Autowired
-// private TreasuryRepository treasuryRepository;
+ @Autowired
+ private AssetRepository assetRepository;
+
+ @Autowired
+ private AssetSyncRepository assetSyncRepository;
+
+ @Autowired
+ private AssetImportService assetImportService;
@Test
- public void listProjects() {
-// List<Project> allItems = projectRepository.findAll();
-// log.info("Found {} items", allItems.size());
+ public void queryPlayLists() {
+ List<String> playlists = assetRepository.queryDistinctPlaylists();
+ assertNotNull(playlists);
+
+ List<String> assets = assetRepository.queryPlaylistsAssets(playlists.get(0));
+ assertNotNull(assets);
+ }
+
+ @Test
+ public void queryExpired() {
+ LocalDate currentDate = LocalDate.parse("2000-01-02", DateTimeFormatter.ISO_LOCAL_DATE);
+ List<String> ids = assetRepository.queryExpired(Date.valueOf(currentDate));
+ assertNotNull(ids);
+ assertEquals(1, ids.size());
+
+ ImportSummary summary = ImportSummary.builder().build();
+ assetImportService.removeExpired(currentDate);
+
+ ids = assetRepository.queryExpired(Date.valueOf(currentDate));
+ assertNotNull(ids);
+ assertEquals(0, ids.size());
+
}
}
package hu.user.mcvodsync;
+import hu.user.mcvodsync.db.SyncType;
import hu.user.mcvodsync.db.repository.AssetRepository;
+import hu.user.mcvodsync.db.repository.AssetSyncRepository;
+import hu.user.mcvodsync.db.repository.PlaylistSyncRepository;
+import hu.user.mcvodsync.service.xls.AssetImportService;
import hu.user.mcvodsync.service.xls.ImportSummary;
import hu.user.mcvodsync.service.xls.VodXlsProcessor;
import lombok.extern.log4j.Log4j2;
+import org.junit.AfterClass;
+import org.junit.BeforeClass;
import org.junit.Test;
+import org.junit.jupiter.api.Order;
import org.junit.runner.RunWith;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.boot.test.context.SpringBootTest;
import org.springframework.test.context.TestPropertySource;
import org.springframework.test.context.junit4.SpringRunner;
+import javax.annotation.PostConstruct;
import java.io.IOException;
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
-import java.util.List;
import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertNotNull;
@Log4j2
@Autowired
private VodXlsProcessor vodXlsProcessor;
+ @Autowired
+ private AssetImportService assetImportService;
+
@Autowired
private AssetRepository assetRepository;
+ @Autowired
+ private AssetSyncRepository assetSyncRepository;
+
+ @Autowired
+ private PlaylistSyncRepository playlistSyncRepository;
+
+ @BeforeClass
+ public static void beforeClass() {
+ }
+
+ @AfterClass
+ public static void afterClass() {
+ }
+
+ @PostConstruct
+ public void init() {
+ }
+
@Test
- public void queryPlayLists() {
- List<String> playlists = assetRepository.queryDistinctPlaylists();
- assertNotNull(playlists);
+ public void initialLoadTest() throws IOException {
+// Path xlsFile = Paths.get("src/test/resources/testdata-big.xlsx");
+ log.info("initialLoadTest");
+ cleanup();
+ ImportSummary summary = process("src/test/resources/testdata.xlsx");
+ checkSummary(summary, 9, 9, 0, 0);
+ checkAsset(9, SyncType.INSERT, 9);
+ checkPlaylist(3, SyncType.INSERT, 3);
+ }
- List<String> assets = assetRepository.queryPlaylistsAssets(playlists.get(0));
- assertNotNull(assets);
+ @Test
+ public void changeVideoTest() throws IOException {
+ log.info("changeVideoTest");
+ ImportSummary summary = process("src/test/resources/testdata.xlsx");
+ cleanupSync();
+ summary = process("src/test/resources/testdata-change-video.xlsx");
+ checkSummary(summary, 9, 0, 1, 0);
+ checkAsset(1, SyncType.UPDATE, 1);
}
@Test
- public void processXLS() throws IOException {
- Path xlsFile = Paths.get("src/test/resources/testdata.xlsx");
- ImportSummary process = vodXlsProcessor.process(xlsFile.getFileName().toString(), Files.readAllBytes(xlsFile));
- assertEquals(process.getAll(), process.getSuccess());
+ public void deleteVideoTest() {
+ log.info("deleteVideoTest");
+ ImportSummary summary = process("src/test/resources/testdata.xlsx");
+ cleanupSync();
+ summary = process("src/test/resources/testdata-delete-video.xlsx");
+ checkSummary(summary, 9, 0, 1, 1);
+ checkAsset(1, SyncType.UPDATE, 2);
+ checkAsset(1, SyncType.DELETE, 2);
+ checkPlaylist(1, SyncType.UPDATE, 1);
+ }
+
+ @Test
+ public void changePlaylistOrderTest() {
+ log.info("changePlaylistOrderTest");
+ ImportSummary summary = process("src/test/resources/testdata.xlsx");
+ cleanupSync();
+ summary = process("src/test/resources/testdata-change-playlist-order.xlsx");
+ checkSummary(summary, 8, 0, 1, 0);
+ checkAsset(1, SyncType.UPDATE, 1);
+ checkPlaylist(1, SyncType.UPDATE, 1);
+ }
+
+ @Test
+ @Order(5)
+ public void changePlaylistAddAssetTest() {
+ log.info("changePlaylistAddAssetTest");
+// "src/test/resources/testdata-playlist-add-asset.xlsx"
+ }
+
+ @Test
+ @Order(6)
+ public void changePlaylistRemoveAssetTest() {
+ log.info("changePlaylistRemoveAssetTest");
+// "src/test/resources/testdata-playlist-remove-asset.xlsx"
+ }
+
+ public void cleanup() {
+ cleanupSync();
+ assetRepository.deleteAllInBatch();
+ }
+
+ public void cleanupSync() {
+ playlistSyncRepository.deleteAllInBatch();
+ assetSyncRepository.deleteAllInBatch();
+ }
+
+ private ImportSummary process(String xlsFile) {
+ ImportSummary summary = null;
+ try {
+ Path input = Paths.get(xlsFile);
+ summary = vodXlsProcessor.process(input.getFileName().toString(), Files.readAllBytes(input));
+ } catch (IOException e) {
+ log.error(e);
+ throw new RuntimeException(e);
+ }
+ return summary;
+ }
+
+ private void checkSummary(ImportSummary summary, long expectedAll, long expectedInsert, long expectedUpdate, long expectedDelete) {
+ assertEquals(expectedAll, summary.getAll());
+ assertEquals(expectedAll, summary.getSuccess());
+ assertEquals(expectedInsert, summary.getInserted());
+ assertEquals(expectedUpdate, summary.getUpdated());
+ assertEquals(expectedDelete, summary.getDeleted());
+ }
+
+ private void checkAsset(long expected, SyncType syncType, long expectedAllCount) {
+ assertEquals(expected, assetSyncRepository.countBySyncType(syncType));
+ assertEquals(expectedAllCount, assetSyncRepository.count());
+ }
+
+ private void checkPlaylist(long expected, SyncType syncType, long expectedAllCount) {
+ assertEquals(expected, playlistSyncRepository.countBySyncType(syncType));
+ assertEquals(expectedAllCount, playlistSyncRepository.count());
}
}
-- // Bootstrap.sql
-DROP DATABASE IF EXISTS VODSYNC;
-CREATE DATABASE VODSYNC AUTOMATIC STORAGE YES USING CODESET UTF-8 TERRITORY hu COLLATE USING UCA500R1_S2 PAGESIZE 32 K;
+-- DROP DATABASE IF EXISTS VODSYNC;
+-- CREATE DATABASE VODSYNC AUTOMATIC STORAGE YES USING CODESET UTF-8 TERRITORY hu COLLATE USING UCA500R1_S2 PAGESIZE 32 K;
+
+TRUNCATE TABLE asset IMMEDIATE;
+TRUNCATE TABLE asset_sync IMMEDIATE;
+TRUNCATE TABLE playlist_sync IMMEDIATE;
CREATE TABLE asset_sync (
id BIGINT NOT NULL GENERATED BY DEFAULT AS IDENTITY,
catalog_id VARCHAR(20) NOT NULL,
- imported TIMESTAMP NOT NULL,
+ created TIMESTAMP NOT NULL,
sync_type VARCHAR(6) NOT NULL,
exported_success TIMESTAMP,
exported_error TIMESTAMP,
CREATE TABLE playlist_sync (
id BIGINT NOT NULL GENERATED BY DEFAULT AS IDENTITY,
playlist VARCHAR(100) NOT NULL,
- imported TIMESTAMP NOT NULL,
+ created TIMESTAMP NOT NULL,
sync_type VARCHAR(6) NOT NULL,
exported_success TIMESTAMP,
exported_error TIMESTAMP,
private String catalogId;
@Column(nullable = false)
- private Date imported;
+ private Date created;
@Enumerated(EnumType.STRING)
@Column(nullable = false)
private String playlist;
@Column(nullable = false)
- private Date imported;
+ private Date created;
@Enumerated(EnumType.STRING)
@Column(nullable = false)
import hu.user.mcvodsync.db.Asset;
import org.springframework.data.jpa.repository.JpaRepository;
+import org.springframework.data.jpa.repository.Modifying;
import org.springframework.data.jpa.repository.Query;
+import java.sql.Date;
import java.util.List;
public interface AssetRepository extends JpaRepository<Asset, String> {
@Query("SELECT a.catalogId FROM Asset a WHERE a.playlist = :playlist ORDER BY a.seasonNr, a.episodeNr")
List<String> queryPlaylistsAssets(String playlist);
+ @Query("SELECT a.catalogId FROM Asset a WHERE a.sunset <= :currentDate ORDER BY a.catalogId")
+ List<String> queryExpired(Date currentDate);
+
+ @Modifying
+ @Query("DELETE FROM Asset a WHERE a.sunset <= :currentDate")
+ void deleteExpired(Date currentDate);
}
package hu.user.mcvodsync.db.repository;
import hu.user.mcvodsync.db.AssetSync;
+import hu.user.mcvodsync.db.SyncType;
import org.springframework.data.jpa.repository.JpaRepository;
public interface AssetSyncRepository extends JpaRepository<AssetSync, String> {
+
+ long countBySyncType(SyncType syncType);
+
}
--- /dev/null
+package hu.user.mcvodsync.db.repository;
+
+import hu.user.mcvodsync.db.PlaylistSync;
+import hu.user.mcvodsync.db.SyncType;
+import org.springframework.data.jpa.repository.JpaRepository;
+
+public interface PlaylistSyncRepository extends JpaRepository<PlaylistSync, String> {
+
+ long countBySyncType(SyncType syncType);
+
+}
import hu.user.mcvodsync.db.Asset;
import hu.user.mcvodsync.db.AssetSync;
+import hu.user.mcvodsync.db.PlaylistSync;
import hu.user.mcvodsync.db.SyncType;
import hu.user.mcvodsync.db.repository.AssetRepository;
import hu.user.mcvodsync.db.repository.AssetSyncRepository;
+import hu.user.mcvodsync.db.repository.PlaylistSyncRepository;
import hu.user.mcvodsync.service.data.EntityDataService;
import lombok.extern.log4j.Log4j2;
+import org.apache.commons.collections4.CollectionUtils;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Service;
-import org.springframework.transaction.annotation.Transactional;
import java.time.Instant;
+import java.time.LocalDate;
import java.util.*;
+import java.util.stream.Collectors;
@Log4j2
@Service
@Autowired
private AssetSyncRepository assetSyncRepository;
+ @Autowired
+ private PlaylistSyncRepository playlistSyncRepository;
+
@Autowired
private EntityDataService<Asset> entityDataService;
private Map<String, List<String>> currentPlaylists;
- public void prepare() {
+ private ImportSummary summary;
+
+ public void prepare(ImportSummary summary) {
+ this.summary = summary;
currentPlaylists = getPlayLists();
}
return result;
}
- @Transactional
- public void processAsset(Asset asset, ImportSummary summary) {
+ public void processAsset(Asset asset) {
Optional<Asset> optionalEntity = assetRepository.findById(asset.getCatalogId());
AssetSync assetSync = AssetSync.builder()
- .imported(Date.from(Instant.now()))
+ .created(Date.from(Instant.now()))
.catalogId(asset.getCatalogId())
- .syncType(SyncType.INSERT)
.build();
if (optionalEntity.isPresent()) {
if (areDifferent) {
assetSync.setSyncType(SyncType.UPDATE);
}
+ } else {
+ assetSync.setSyncType(SyncType.INSERT);
}
- assetRepository.saveAndFlush(asset);
- assetSyncRepository.saveAndFlush(assetSync);
+ if (Objects.nonNull(assetSync.getSyncType())) {
+ assetRepository.save(asset);
+ assetSyncRepository.save(assetSync);
+ }
if (assetSync.getSyncType() == SyncType.UPDATE) {
summary.incUpdated();
}
}
+ public void removeExpired(LocalDate currentDate) {
+ List<String> ids = assetRepository.queryExpired(java.sql.Date.valueOf(currentDate));
+ List<AssetSync> syncs = ids.stream().map(id -> AssetSync.builder()
+ .catalogId(id)
+ .syncType(SyncType.DELETE)
+ .created(Date.from(Instant.now()))
+ .build()).collect(Collectors.toList());
+ assetSyncRepository.saveAll(syncs);
+ assetRepository.deleteExpired(java.sql.Date.valueOf(currentDate));
+ summary.setDeleted(summary.getDeleted() + ids.size());
+ }
+
public void finish() {
+ removeExpired(LocalDate.now());
+ processPlaylists();
+ }
+
+ public void processPlaylists() {
Map<String, List<String>> newPlaylists = getPlayLists();
+ Set<String> currentPlaylistNames = currentPlaylists.keySet();
+ Set<String> newPlaylistNames = newPlaylists.keySet();
+
+ Collection<String> playlistNamesDifference = CollectionUtils.subtract(currentPlaylistNames, newPlaylistNames);
+ if (!playlistNamesDifference.isEmpty()) {
+ playlistNamesDifference.forEach(playlist -> {
+ PlaylistSync sync = PlaylistSync.builder()
+ .playlist(playlist)
+ .syncType(SyncType.DELETE)
+ .created(Date.from(Instant.now()))
+ .build();
+ playlistSyncRepository.save(sync);
+ summary.incDeletedPlaylist();
+ });
+ }
- //CollectionUtils.isEqualCollection(Collection<?> a, Collection<?> b)
+ playlistNamesDifference = CollectionUtils.subtract(newPlaylistNames, currentPlaylistNames);
+ if (!playlistNamesDifference.isEmpty()) {
+ playlistNamesDifference.forEach(playlist -> {
+ PlaylistSync sync = PlaylistSync.builder()
+ .playlist(playlist)
+ .syncType(SyncType.INSERT)
+ .created(Date.from(Instant.now()))
+ .build();
+ playlistSyncRepository.save(sync);
+ summary.incInsertedPlaylist();
+ });
+ }
+
+ currentPlaylistNames.forEach(playlist -> {
+ List<String> currentContent = currentPlaylists.get(playlist);
+ List<String> newContent = newPlaylists.get(playlist);
+ if (isContentChanged(currentContent, newContent)) {
+ PlaylistSync sync = PlaylistSync.builder()
+ .playlist(playlist)
+ .syncType(SyncType.UPDATE)
+ .created(Date.from(Instant.now()))
+ .build();
+ playlistSyncRepository.save(sync);
+ summary.incUpdatedPlaylist();
+ }
+ });
+ }
+
+ private boolean isContentChanged(List<String> currentContent, List<String> newContent) {
+ if (currentContent.size() != newContent.size()) {
+ return true;
+ }
+ for (int i = 0; i < currentContent.size(); i++) {
+ if (!currentContent.get(i).equals(newContent.get(i))) {
+ return true;
+ }
+ }
+ return false;
}
}
}
@Named("mapSunset")
- static Date mapSunset(String dateValue) {
- Date result = null;
+ static Date mapSunset(String numberValue) {
+// Date result = null;
+// try {
+// if (StringUtils.isNotBlank(dateValue)) {
+// result = new Date(formatter.parse(dateValue).getTime());
+// }
+// } catch (Exception ignored) {
+// }
+// return result;
+ long longValue = 0;
try {
- if (StringUtils.isNotBlank(dateValue)) {
- result = new Date(formatter.parse(dateValue).getTime());
- }
+ longValue = Long.parseLong(numberValue);
} catch (Exception ignored) {
}
- return result;
+ long javaDateValue = (longValue - 25569) * 86400 * 1000;
+ return new Date(javaDateValue);
}
}
private long deleted;
+ private long insertedPlaylist;
+
+ private long updatedPlaylist;
+
+ private long deletedPlaylist;
+
public String toString() {
Duration executionDuration = Duration.between(started, finished);
String duration = DurationFormatUtils.formatDuration(executionDuration.toMillis(), "H:mm:ss", true);
return String.format("Execution started at %s, finished at %s.", started, finished) +
System.lineSeparator() +
- String.format("Process execution took %s on %d rows", duration, all) +
+ String.format("Process execution took %s on %d rows.", duration, all) +
System.lineSeparator() +
- String.format("Success count is %d, error count is %d", success, error) +
+ String.format("Success count: %d, error count: %d", success, error) +
System.lineSeparator() +
- String.format("Insert count is %d, update count is %d, delete count is %d", inserted, updated, deleted);
+ String.format("Insert count: %d, update count: %d, delete count: %d", inserted, updated, deleted);
}
- public void incError() {
- error++;
+ public void incAll() {
+ all++;
}
public void incSuccess() {
success++;
}
+ public void incError() {
+ error++;
+ }
+
public void incInserted() {
inserted++;
}
updated++;
}
- public void incDeleted() {
- inserted++;
+ public void incInsertedPlaylist() {
+ insertedPlaylist++;
}
- public void incAll() {
- all++;
+ public void incUpdatedPlaylist() {
+ updatedPlaylist++;
+ }
+
+ public void incDeletedPlaylist() {
+ deletedPlaylist++;
}
+
}
import org.dhatim.fastexcel.reader.Sheet;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Service;
+import org.springframework.transaction.annotation.Transactional;
import java.io.ByteArrayInputStream;
import java.io.IOException;
@Autowired
private AssetImportService assetImportService;
+ @Transactional
public ImportSummary process(String fileName, byte[] xlsData) {
ImportSummary summary = ImportSummary.builder().started(Instant.now()).build();
try (InputStream is = new ByteArrayInputStream(xlsData);
try (Stream<Row> rows = sheet.openStream()) {
rows.forEach(row -> {
if (row.getRowNum() == 1) {
- assetImportService.prepare();
+ assetImportService.prepare(summary);
headers = getHeaders(row);
} else {
summary.incAll();
try {
asset = assetMapper.toEntity(rowData);
if (isAssetValid(asset)) {
- assetImportService.processAsset(asset, summary);
+ assetImportService.processAsset(asset);
}
summary.incSuccess();
} catch (Exception e) {
https://git.useribm.hu / mediacube.git / commitdiff