xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 https://maven.apache.org/xsd/maven-4.0.0.xsd">
<modelVersion>4.0.0</modelVersion>
<artifactId>mc-vod-sync-app</artifactId>
- <version>0.0.3</version>
+ <version>0.0.4</version>
<parent>
<groupId>hu.user</groupId>
<artifactId>mc-vod-sync</artifactId>
mc-vod-sync:
service:
target-video-insert-enabled: false
+ target-video-update-enabled: false
target-playlist-insert-enabled: false
scheduler:
import-enabled: false
export-enabled: false
- sunset-checker-enabled: false
- import-cron: 0 0 4 1/1 * ? *
- export-cron: 0 0 4 1/1 * ? *
- sunset-checker-cron: 0 0 4 1/1 * ? *
+ schedule-checker-enabled: true
+ import-cron: 0 0 4 1/1 * ?
+ export-cron: 0 0 4 1/1 * ?
+ schedule-checker-cron: 0 38 14 1/1 * ?
report:
recipient: vasary@elgekko.net
sender: noreply@mc-vod-sync
mc-vod-sync:
service:
target-video-insert-enabled: false
+ target-video-update-enabled: false
target-playlist-insert-enabled: false
scheduler:
import-enabled: false
export-enabled: false
- sunset-checker-enabled: false
+ schedule-checker-enabled: false
import-cron: 0 0 4 1/1 * ? *
export-cron: 0 0 4 1/1 * ? *
- sunset-checker-cron: 0 0 4 1/1 * ? *
+ schedule-checker-cron: 0 0 4 1/1 * ? *
report:
recipient: vasary@elgekko.net
sender: noreply@mc-vod-sync
package hu.user.mcvodsync;
-import hu.user.mcvodsync.service.schedule.ScheduledSunsetChacker;
+import hu.user.mcvodsync.service.schedule.ScheduledDateChecker;
import lombok.extern.log4j.Log4j2;
import org.junit.Test;
import org.junit.runner.RunWith;
private TaskScheduler taskScheduler;
@Autowired
- private ScheduledSunsetChacker scheduledSunsetChacker;
+ private ScheduledDateChecker scheduledDateChecker;
@Test
public void testSemaphore() throws InterruptedException {
log.info("Test start");
Thread.sleep(1000);
- taskScheduler.schedule(scheduledSunsetChacker, Instant.now());
+ taskScheduler.schedule(scheduledDateChecker, Instant.now());
Thread.sleep(5000);
- log.info("Test finish");
+ log.info("Test processPlaylists");
}
}
\ No newline at end of file
import org.springframework.test.context.ActiveProfiles;
import org.springframework.test.context.junit4.SpringRunner;
-import java.sql.Date;
import java.time.Instant;
import java.time.LocalDate;
import java.time.format.DateTimeFormatter;
List<String> playlists = assetRepository.queryDistinctPlaylists();
assertNotNull(playlists);
- List<Long> assets = assetRepository.queryPlaylistsAssets(playlists.get(0));
+ List<Long> assets = assetRepository.queryPlaylistsAssetsByDate(playlists.get(0), java.sql.Date.valueOf(LocalDate.now()));
assertNotNull(assets);
}
@Test
public void queryExpired() {
LocalDate currentDate = LocalDate.parse("2000-01-02", DateTimeFormatter.ISO_LOCAL_DATE);
- List<Asset> ids = assetRepository.queryExpired(Date.valueOf(currentDate));
+ List<Asset> ids = assetRepository.queryExpired(java.sql.Date.valueOf(currentDate));
assertNotNull(ids);
assertEquals(1, ids.size());
CompositeSummary summary = CompositeSummary.builder().build();
- assetImportService.removeExpired(currentDate, summary);
+ assetImportService.removeExpired(summary);
- ids = assetRepository.queryExpired(Date.valueOf(currentDate));
+ ids = assetRepository.queryExpired(java.sql.Date.valueOf(currentDate));
assertNotNull(ids);
assertEquals(0, ids.size());
--- /dev/null
+package hu.user.mcvodsync;
+
+import hu.user.mcvodsync.db.Asset;
+import hu.user.mcvodsync.db.repository.AssetRepository;
+import hu.user.mcvodsync.service.schedule.ScheduledDateChecker;
+import lombok.extern.log4j.Log4j2;
+import org.junit.Test;
+import org.junit.runner.RunWith;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.boot.test.context.SpringBootTest;
+import org.springframework.test.context.ActiveProfiles;
+import org.springframework.test.context.TestPropertySource;
+import org.springframework.test.context.junit4.SpringRunner;
+
+import java.time.LocalDate;
+import java.util.Optional;
+
+import static org.junit.Assert.assertEquals;
+
+@Log4j2
+@RunWith(SpringRunner.class)
+@SpringBootTest
+@ActiveProfiles("dev")
+@TestPropertySource("classpath:application-dev.yaml")
+public class ScheduledDateCheckerIT extends AssetImportExportBase {
+
+ @Autowired
+ private ScheduledDateChecker scheduledDateChecker;
+
+ @Autowired
+ private AssetRepository assetRepository;
+
+ @Test
+ public void doNotPublishTest() {
+ cleanupLocal();
+ importFile("src/test/resources/testdata.xlsx");
+ cleanupLocalSync();
+
+ assetRepository.clearListed();
+ Optional<Asset> asset = assetRepository.findByCatalogId("CCEM110068");
+ asset.ifPresent(a -> {
+ a.setSunrise(java.sql.Date.valueOf(LocalDate.now().plusDays(2)));
+ assetRepository.save(a);
+ });
+
+ scheduledDateChecker.run();
+ assertEquals("CCEM110180,CCEM110183", getPlaylistSyncIds("Playlist1"));
+ assertEquals("CCEM128963,CCEM128965,CCEM128967", getPlaylistSyncIds("Playlist2"));
+ assertEquals("CCEF008678,CCEM001148,CCEM072086", getPlaylistSyncIds("Playlist3"));
+ }
+
+ @Test
+ public void publishTest() {
+ cleanupLocal();
+ importFile("src/test/resources/testdata.xlsx");
+ cleanupLocalSync();
+
+ Optional<Asset> asset = assetRepository.findByCatalogId("CCEM110068");
+ asset.ifPresent(a -> {
+ a.setSunrise(java.sql.Date.valueOf(LocalDate.now().plusDays(1)));
+ a.setListed(false);
+ assetRepository.save(a);
+ });
+
+ scheduledDateChecker.run();
+ assertEquals("CCEM110180,CCEM110183,CCEM110068", getPlaylistSyncIds("Playlist1"));
+ }
+
+ @Test
+ public void unPublishTest() {
+ cleanupLocal();
+ importFile("src/test/resources/testdata.xlsx");
+ cleanupLocalSync();
+
+ Optional<Asset> asset = assetRepository.findByCatalogId("CCEM110068");
+ asset.ifPresent(a -> {
+ a.setSunset(java.sql.Date.valueOf(LocalDate.now()));
+ assetRepository.save(a);
+ });
+
+ scheduledDateChecker.run();
+ assertEquals("CCEM110180,CCEM110183", getPlaylistSyncIds("Playlist1"));
+ }
+}
--- /dev/null
+-- // Modify Asset
+-- Migration SQL that makes the change goes here.
+ALTER TABLE asset
+ ADD COLUMN listed SMALLINT;
+
+CREATE INDEX idx_asset_playlist_listed ON asset(playlist, listed);
+CREATE INDEX idx_asset_playlist_sunrise ON asset(playlist, sunrise);
+CREATE INDEX idx_asset_sunset ON asset(sunset);
+
+-- //@UNDO
+-- SQL to undo the change goes here.
+DROP INDEX idx_asset_playlist_listed;
+DROP INDEX idx_asset_playlist_sunrise;
+DROP INDEX idx_asset_sunset;
+
+ALTER TABLE asset
+ DROP COLUMN listed;
+CALL SYSPROC.ADMIN_CMD('REORG TABLE asset');
@Column(name = "prog_genre3", length = 128)
private String progGenre3;
+ private boolean listed;
+
}
import org.springframework.data.jpa.repository.JpaRepository;
import org.springframework.data.jpa.repository.Modifying;
import org.springframework.data.jpa.repository.Query;
+import org.springframework.transaction.annotation.Transactional;
import java.sql.Date;
import java.util.List;
public interface AssetRepository extends JpaRepository<Asset, Long> {
+ Optional<Asset> findByCatalogId(String catalogId);
+
@Query("SELECT DISTINCT(a.playlist) FROM Asset a ORDER BY a.playlist")
List<String> queryDistinctPlaylists();
- @Query("SELECT a.id FROM Asset a WHERE a.playlist = :playlist ORDER BY a.seasonNr, a.episodeNr")
- List<Long> queryPlaylistsAssets(String playlist);
+ @Query("SELECT a.id FROM Asset a WHERE a.playlist = :playlist and a.sunrise <= :currentDate ORDER BY a.seasonNr, a.episodeNr")
+ List<Long> queryPlaylistsAssetsByDate(String playlist, Date currentDate);
+
+ @Query("SELECT a.id FROM Asset a WHERE a.playlist = :playlist AND a.listed = TRUE ORDER BY a.seasonNr, a.episodeNr")
+ List<Long> queryPlaylistsListedAssets(String playlist);
Optional<Asset> findByCatalogIdAndPlaylist(String catalogId, String playlist);
- @Query("SELECT a FROM Asset a WHERE a.sunset <= :currentDate ORDER BY a.catalogId")
+ @Query("SELECT a FROM Asset a WHERE a.sunset < :currentDate ORDER BY a.catalogId")
List<Asset> queryExpired(Date currentDate);
@Modifying
- @Query("DELETE FROM Asset a WHERE a.sunset <= :currentDate")
+ @Query("UPDATE Asset a SET a.listed = TRUE WHERE a.listed = FALSE AND a.id IN (:ids)")
+ void updateListed(List<Long> ids);
+
+ @Modifying
+ @Query("UPDATE Asset a SET a.listed = FALSE")
+ @Transactional
+ void clearListed();
+
+ @Modifying
+ @Query("DELETE FROM Asset a WHERE a.sunset < :currentDate")
+ @Transactional
void deleteExpired(Date currentDate);
@Query("SELECT a.referenceId FROM Asset a WHERE a.id = :id")
private boolean targetVideoInsertEnabled;
+ private boolean targetVideoUpdateEnabled;
+
private boolean targetPlaylistInsertEnabled;
private Scheduler scheduler;
private String exportCron;
- private boolean sunsetCheckerEnabled;
+ private boolean scheduleCheckerEnabled;
- private String sunsetCheckerCron;
+ private String scheduleCheckerCron;
}
@Getter
import hu.user.mcvodsync.db.repository.PlaylistSyncRepository;
import hu.user.mcvodsync.service.ServiceProperties;
import hu.user.mcvodsync.service.data.CompositeSummary;
-import hu.user.mcvodsync.service.event.ExportCompletedEvent;
-import hu.user.mcvodsync.service.event.ExportStartedEvent;
-import hu.user.mcvodsync.service.event.ImportCompletedEvent;
-import hu.user.mcvodsync.service.event.ImportStartedEvent;
+import hu.user.mcvodsync.service.event.*;
import hu.user.mcvodsync.service.mail.EmailSendService;
+import hu.user.mcvodsync.service.schedule.ScheduledDateChecker;
import hu.user.mcvodsync.service.schedule.ScheduledExport;
import hu.user.mcvodsync.service.schedule.ScheduledImport;
-import hu.user.mcvodsync.service.schedule.ScheduledSunsetChacker;
import hu.user.mcvodsync.service.time.TimeUtils;
import lombok.Getter;
import lombok.extern.log4j.Log4j2;
private ScheduledImport scheduledImport;
@Autowired
- private ScheduledSunsetChacker scheduledSunsetChacker;
+ private ScheduledDateChecker scheduledDateChecker;
@Autowired
private AssetSyncRepository assetSyncRepository;
private PlaylistSyncRepository playlistSyncRepository;
@Autowired
- EmailSendService emailSendService;
+ private EmailSendService emailSendService;
@Getter
private CompositeSummary importSummary;
if (scheduler.isImportEnabled()) {
taskScheduler.schedule(scheduledImport, new CronTrigger(scheduler.getExportCron()));
}
- if (scheduler.isSunsetCheckerEnabled()) {
- taskScheduler.schedule(scheduledSunsetChacker, new CronTrigger(scheduler.getSunsetCheckerCron()));
+ if (scheduler.isScheduleCheckerEnabled()) {
+ taskScheduler.schedule(scheduledDateChecker, new CronTrigger(scheduler.getScheduleCheckerCron()));
}
checkSyncStatus();
taskScheduler.schedule(scheduledExport, Instant.now());
}
+ public void startCheckDates() {
+ taskScheduler.schedule(scheduledDateChecker, Instant.now());
+ }
+
+ @Async
+ @EventListener
+ public void handleEvent(DateCheckerStartedEvent evt) {
+ log.info("DateCheckerStartedEvent handle");
+ importSummary = null;
+ status = String.format("Date checker is in progress, started as %s", TimeUtils.toString(evt.getTimestamp()));
+ }
+
+ @Async
+ @EventListener
+ public void handleEvent(DateCheckerCompletedEvent evt) {
+ log.info("DateCheckerCompletedEvent handle");
+ importSummary = evt.getSummary();
+ status = String.format("Date checker completed at %s", TimeUtils.toString(evt.getTimestamp()));
+ checkSyncStatus();
+ emailSendService.sendSimpleMessage("BrightCove date checker report", importSummary);
+ }
+
@Async
@EventListener
public void handleEvent(ImportStartedEvent evt) {
--- /dev/null
+package hu.user.mcvodsync.service.event;
+
+import hu.user.mcvodsync.service.data.CompositeSummary;
+import lombok.Getter;
+import lombok.Setter;
+import org.springframework.context.ApplicationEvent;
+
+@Getter
+@Setter
+public class DateCheckerCompletedEvent extends ApplicationEvent {
+
+ private final CompositeSummary summary;
+
+ public DateCheckerCompletedEvent(Object source, CompositeSummary summary) {
+ super(source);
+ this.summary = summary;
+ }
+}
--- /dev/null
+package hu.user.mcvodsync.service.event;
+
+import lombok.Getter;
+import lombok.Setter;
+import org.springframework.context.ApplicationEvent;
+
+@Getter
+@Setter
+public class DateCheckerStartedEvent extends ApplicationEvent {
+
+ public DateCheckerStartedEvent(Object source) {
+ super(source);
+ }
+
+}
import org.apache.commons.collections4.CollectionUtils;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Service;
+import org.springframework.transaction.annotation.Transactional;
import java.time.Instant;
import java.time.LocalDate;
private Map<String, List<Long>> currentPlaylists;
+ private java.sql.Date checkDate;
- public void prepare() {
- currentPlaylists = getPlayLists();
+ public void prepare(LocalDate currentData) {
+ checkDate = java.sql.Date.valueOf(currentData);
}
+
public Map<String, List<Long>> getPlayLists() {
Map<String, List<Long>> result = new HashMap<>();
List<String> playlistNames = assetRepository.queryDistinctPlaylists();
playlistNames.forEach(playlist -> {
- List<Long> playlistsAssets = assetRepository.queryPlaylistsAssets(playlist);
- result.put(playlist, playlistsAssets);
+ List<Long> playlistsAssets = assetRepository.queryPlaylistsListedAssets(playlist);
+ if (!playlistsAssets.isEmpty()) {
+ result.put(playlist, playlistsAssets);
+ }
+ });
+ return result;
+ }
+
+ private Map<String, List<Long>> getPlayListsByDate() {
+ Map<String, List<Long>> result = new HashMap<>();
+ List<String> playlistNames = assetRepository.queryDistinctPlaylists();
+ playlistNames.forEach(playlist -> {
+ List<Long> playlistsAssets = assetRepository.queryPlaylistsAssetsByDate(playlist, checkDate);
+ if (!playlistsAssets.isEmpty()) {
+ result.put(playlist, playlistsAssets);
+ }
});
return result;
}
}
}
- public void removeExpired(LocalDate currentDate, CompositeSummary summary) {
- List<Asset> expiredAssets = assetRepository.queryExpired(java.sql.Date.valueOf(currentDate));
- assetRepository.deleteExpired(java.sql.Date.valueOf(currentDate));
- List<AssetSync> syncs = expiredAssets.stream().map(asset -> AssetSync.builder()
- .assetId(asset.getId())
- .catalogId(asset.getCatalogId())
- .syncType(SyncType.DELETE)
- .build()).collect(Collectors.toList());
- assetSyncRepository.saveAllAndFlush(syncs);
- summary.getVideo().setDelete(summary.getVideo().getDelete() + expiredAssets.size());
+ public void removeExpired(CompositeSummary summary) {
+ List<Asset> expiredAssets = assetRepository.queryExpired(checkDate);
+ if (!expiredAssets.isEmpty()) {
+ assetRepository.deleteExpired(checkDate);
+ List<AssetSync> syncs = expiredAssets.stream().map(asset -> AssetSync.builder()
+ .assetId(asset.getId())
+ .catalogId(asset.getCatalogId())
+ .syncType(SyncType.DELETE)
+ .build()).collect(Collectors.toList());
+ assetSyncRepository.saveAllAndFlush(syncs);
+ summary.getVideo().setDelete(expiredAssets.size());
+ }
}
- public void finish(CompositeSummary summary) {
- removeExpired(LocalDate.now(), summary);
- processPlaylists(summary);
+ @Transactional
+ public void processPlaylists(CompositeSummary summary) {
+ currentPlaylists = getPlayLists();
+ removeExpired(summary);
+ checkPlaylists(summary);
}
- public void processPlaylists(CompositeSummary summary) {
- Map<String, List<Long>> newPlaylists = getPlayLists();
+ public void checkPlaylists(CompositeSummary summary) {
Set<String> currentPlaylistNames = currentPlaylists.keySet();
+
+ Map<String, List<Long>> newPlaylists = getPlayListsByDate();
Set<String> newPlaylistNames = newPlaylists.keySet();
Collection<String> playlistNamesDifference = CollectionUtils.subtract(currentPlaylistNames, newPlaylistNames);
.syncType(SyncType.DELETE)
.created(Date.from(Instant.now()))
.build();
- playlistSyncRepository.save(sync);
+ playlistSyncRepository.saveAndFlush(sync);
summary.getPlaylist().incDelete();
});
}
.created(Date.from(Instant.now()))
.ids(newPlaylists.get(name))
.build();
- playlistSyncRepository.save(sync);
+ playlistSyncRepository.saveAndFlush(sync);
+ assetRepository.updateListed(sync.getIds());
summary.getPlaylist().incInsert();
+
});
}
.ids(newPlaylists.get(name))
.created(Date.from(Instant.now()))
.build();
- playlistSyncRepository.save(sync);
+ playlistSyncRepository.saveAndFlush(sync);
+ assetRepository.updateListed(sync.getIds());
summary.getPlaylist().incUpdate();
}
});
import java.io.IOException;
import java.io.InputStream;
import java.time.Instant;
+import java.time.LocalDate;
import java.util.HashMap;
import java.util.List;
import java.util.Map;
try (Stream<Row> rows = sheet.openStream()) {
rows.forEach(row -> {
if (row.getRowNum() == 1) {
- assetImportService.prepare();
+ assetImportService.prepare(LocalDate.now());
headers = getHeaders(row);
} else {
log.info("Processing {}", row.getRowNum());
}
});
}
- assetImportService.finish(summary);
+ assetImportService.processPlaylists(summary);
} catch (IOException e) {
log.error("Excel file reading error from {}. System message: {}", fileName, e.getMessage());
import hu.user.mcvodsync.db.repository.AssetRepository;
import hu.user.mcvodsync.db.repository.AssetSyncRepository;
import hu.user.mcvodsync.db.repository.PlaylistSyncRepository;
+import hu.user.mcvodsync.service.ServiceProperties;
import hu.user.mcvodsync.service.data.CompositeSummary;
import lombok.extern.log4j.Log4j2;
import org.apache.commons.lang3.StringUtils;
@Autowired
private VideoMapper videoMapper;
+ @Autowired
+ private ServiceProperties serviceProperties;
+
private static final String MC_VOD_SYNC = "mc-vod-sync";
@Transactional
assetRepository.save(asset);
}
}
- if (StringUtils.isNotBlank(videoId)) {
- assetVideo.addTagsItem(MC_VOD_SYNC);
- bcClient.updateVideo(videoId, assetVideo);
- summary.getVideo().incUpdate();
+
+ if (serviceProperties.isTargetVideoUpdateEnabled()) {
+ if (StringUtils.isNotBlank(videoId)) {
+ assetVideo.addTagsItem(MC_VOD_SYNC);
+ bcClient.updateVideo(videoId, assetVideo);
+ summary.getVideo().incUpdate();
+ }
+ } else {
+ summary.getVideo().incSkip();
}
+
assetSync.setExportedSuccess(Date.from(Instant.now()));
}
summary.getVideo().incSuccess();
--- /dev/null
+package hu.user.mcvodsync.service.schedule;
+
+import hu.user.mcvodsync.service.data.CompositeSummary;
+import hu.user.mcvodsync.service.event.DateCheckerCompletedEvent;
+import hu.user.mcvodsync.service.event.DateCheckerStartedEvent;
+import hu.user.mcvodsync.service.in.AssetImportService;
+import lombok.SneakyThrows;
+import lombok.extern.log4j.Log4j2;
+import org.springframework.beans.factory.annotation.Autowired;
+import org.springframework.context.ApplicationEventPublisher;
+import org.springframework.stereotype.Component;
+
+import java.time.Instant;
+import java.time.LocalDate;
+
+@Log4j2
+@Component
+public class ScheduledDateChecker implements Runnable {
+ @Autowired
+ private ApplicationEventPublisher applicationEventPublisher;
+
+ @Autowired
+ private AssetImportService assetImportService;
+
+ @SneakyThrows
+ @Override
+ public void run() {
+ CompositeSummary summary = CompositeSummary.builder().started(Instant.now()).build();
+ try {
+ applicationEventPublisher.publishEvent(new DateCheckerStartedEvent(this));
+ log.info("ScheduledDateChecker started");
+ assetImportService.prepare(LocalDate.now().plusDays(1));
+ assetImportService.processPlaylists(summary);
+ } catch (Exception e) {
+ log.error(e);
+ } finally {
+ summary.setFinished(Instant.now());
+ applicationEventPublisher.publishEvent(new DateCheckerCompletedEvent(this, summary));
+ log.info("ScheduledDateChecker finished");
+ }
+
+ }
+}
+++ /dev/null
-package hu.user.mcvodsync.service.schedule;
-
-import lombok.SneakyThrows;
-import lombok.extern.log4j.Log4j2;
-import org.springframework.stereotype.Component;
-
-import java.util.concurrent.ThreadLocalRandom;
-
-@Log4j2
-@Component
-//@ConditionalOnExpression("'${mc-vod-sync.scheduler.sunrise-checker}'=='true'")
-public class ScheduledSunsetChacker implements Runnable {
-
- @SneakyThrows
- @Override
- public void run() {
- try {
- log.info("Start 3");
- log.info("Execute 3");
- int randomNum = ThreadLocalRandom.current().nextInt(100, 2000);
- Thread.sleep(randomNum);
- log.info("Finish 3");
-
- } catch (Exception e) {
- log.error(e);
- }
-
- }
-}
scheduledTasks.startExport();
}
+ @Command
+ public void onStartCheckDates() {
+ scheduledTasks.startCheckDates();
+ }
+
}
<separator orient="vertical"/>
<toolbarbutton label="Export" iconSclass="z-icon-cloud-upload"
onClick="@command('onStartExport')" style="font-size: 1.2em"/>
+ <space bar="true"/>
+ <toolbarbutton label="Check dates" iconSclass="z-icon-calendar-times-o"
+ onClick="@command('onStartCheckDates')" style="font-size: 1.2em"/>
</toolbar>
</vlayout>
</north>
https://git.useribm.hu / mediacube.git / commitdiff