From: Vásáry Dániel Date: Mon, 23 Sep 2019 11:00:58 +0000 (+0000) Subject: MediaCubeConfig bevezetese X-Git-Url: http://git.useribm.hu/?a=commitdiff_plain;h=6c52c18439126af606b1280884887b09ab3fef27;p=mediacube.git MediaCubeConfig bevezetese git-tfs-id: [http://tfs.userrendszerhaz.hu:8080/tfs/DefaultCollection]$/MediaCube;C31539 --- diff --git a/server/-configuration/mediacube.json b/server/-configuration/mediacube.json new file mode 100644 index 00000000..4c07e5ba --- /dev/null +++ b/server/-configuration/mediacube.json @@ -0,0 +1,51 @@ +{ + "authentication": { + "authEnabled": true, + "adHost": "10.10.254.11", + "adNonSecurePort": 3268, + "adBaseDn": "DC=intra,DC=echotv,DC=hu", + "adAdminMap": [ + "G_ECH_U_INFORMATIKUSOK", + "G_ECH_U_MUSZAKVEZETOK", + "ECH-ISILON-ADMINS" + ], + "adSubmitterMap": [ + "G_ECH_U_INFORMATIKUSOK", + "G_ECH_U_MUSZAKVEZETOK", + "ECH-ISILON-ADMINS" + ], + "adEditorMap": [ + "G_ECH_U_INFORMATIKUSOK", + "G_ECH_U_MUSZAKVEZETOK", + "ECH-ISILON-ADMINS" + ], + "localAccounts": [ + { + "user" : "user", + "password" : "password", + "email" : null + }, + { + "user" : "kuka", + "password" : "kuka", + "email" : null + }, + { + "user" : "lebony", + "password" : "lebony", + "email" : null + }, + { + "user" : "root", + "password" : "password", + "email" : "vasary@elgekko.net" + } + ], + "localAdmins": "root", + "localSubmitters": "lebony", + "localEditors": "editor" + } +} + + + diff --git a/server/-configuration/run-mediacube-server-user.launch b/server/-configuration/run-mediacube-server-user.launch index 7243a871..8fb202ba 100644 --- a/server/-configuration/run-mediacube-server-user.launch +++ b/server/-configuration/run-mediacube-server-user.launch @@ -6,7 +6,7 @@ - + @@ -19,7 +19,7 @@ - + diff --git a/server/hu.user.mediacube.indexer/src/main/java/hu/user/mediacube/indexer/BeeperControl.java b/server/hu.user.mediacube.indexer/src/main/java/hu/user/mediacube/indexer/BeeperControl.java deleted file mode 100644 index 9f7166fc..00000000 --- a/server/hu.user.mediacube.indexer/src/main/java/hu/user/mediacube/indexer/BeeperControl.java +++ /dev/null @@ -1,59 +0,0 @@ -package hu.user.mediacube.indexer; - -import static java.util.concurrent.TimeUnit.SECONDS; - -import java.util.concurrent.Executors; -import java.util.concurrent.ScheduledExecutorService; -import java.util.concurrent.ScheduledFuture; -import java.util.concurrent.TimeUnit; - -import javax.annotation.PostConstruct; -import javax.annotation.PreDestroy; - -import org.slf4j.Logger; -import org.slf4j.LoggerFactory; -import org.springframework.stereotype.Service; - -@Service -public class BeeperControl { - private static Logger logger = LoggerFactory.getLogger(BeeperControl.class); - private final ScheduledExecutorService scheduler = Executors.newScheduledThreadPool(1); - private ScheduledFuture beeperHandle; - - public void beepForAnHour() { - - final Runnable beeper = new Runnable() { - @Override - public void run() { - System.out.println("beep"); - } - }; - - beeperHandle = scheduler.scheduleAtFixedRate(beeper, 0, 5, SECONDS); - - scheduler.schedule(new Runnable() { - @Override - public void run() { - beeperHandle.cancel(true); - } - }, 60 * 60, SECONDS); - } - - @PostConstruct - public void start() { - logger.info("start"); - beepForAnHour(); - } - - @PreDestroy - public void stop() { - if (beeperHandle != null) - beeperHandle.cancel(true); - try { - scheduler.awaitTermination(2, TimeUnit.SECONDS); - } catch (InterruptedException e) { - logger.error("Scheduler termination error", e); - } - logger.info("stop"); - } -} diff --git a/server/hu.user.mediacube.indexer/src/main/java/hu/user/mediacube/indexer/IndexerService.java b/server/hu.user.mediacube.indexer/src/main/java/hu/user/mediacube/indexer/IndexerService.java new file mode 100644 index 00000000..6fdd9518 --- /dev/null +++ b/server/hu.user.mediacube.indexer/src/main/java/hu/user/mediacube/indexer/IndexerService.java @@ -0,0 +1,105 @@ +package hu.user.mediacube.indexer; + +import static java.util.concurrent.TimeUnit.SECONDS; + +import java.io.IOException; +import java.util.List; +import java.util.concurrent.Executors; +import java.util.concurrent.ScheduledExecutorService; +import java.util.concurrent.ScheduledFuture; + +import javax.annotation.PreDestroy; + +import org.apache.ibatis.logging.LogFactory; +import org.slf4j.Logger; +import org.slf4j.LoggerFactory; +import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.stereotype.Component; + +import hu.user.mediacube.indexer.db.IndexerDescription; +import hu.user.mediacube.indexer.db.IndexerViewMapper; +import hu.user.mediacube.indexer.db.MediadescriptionMapper; +import hu.user.mediacube.indexer.lucene.LuceneIndexer; + +//@Service +@Component +public class IndexerService implements Runnable { + private static Logger logger = LoggerFactory.getLogger(IndexerService.class); + private final ScheduledExecutorService scheduler = Executors.newScheduledThreadPool(1); + private ScheduledFuture beeperHandle; + + @Autowired + private IndexerViewMapper mapper; + @Autowired + private MediadescriptionMapper mmapper; + @Autowired + private LuceneIndexer indexer; + @Autowired + private JsonProperties config; + + @Override + //periodic task + public void run() { + logger.info("Reading index"); + List indexerDescriptions = mapper.selectAll(); + int i = 0; + LogFactory.useLog4J2Logging(); + for (IndexerDescription indexerDescription : indexerDescriptions) { + + String description = mapper.selectDescription(indexerDescription); + indexerDescription.setDescription(description); + //logger.info("{}", indexerDescription); + boolean processed = ++i % 10 == 0; + try { + indexer.index(indexerDescription); + if (i == indexerDescriptions.size()) + indexer.commit(); + } catch (IOException e) { + logger.error("Can not create index, caused by {} : {}", e.getClass().getSimpleName(), e.getMessage(), e); + break; + } + + if (processed) { + logger.info("Processed {} / {}", i, indexerDescriptions.size()); + } + + } + try { + indexer.commit(); + } catch (IOException e) { + logger.error("Commit error", e); + } + } + + public void schedule() { + // beeperHandle = scheduler.scheduleAtFixedRate(this, 0, 5, SECONDS); + beeperHandle = scheduler.schedule(this, 0, SECONDS); + + // scheduler.schedule(new Runnable() { + // @Override + // public void run() { + // beeperHandle.cancel(true); + // } + // }, 60 * 60, SECONDS); + } + + //@PostConstruct + public void start() { + logger.info("start"); + //schedule(); + if (config.isResetIndex()) + run(); + } + + @PreDestroy + public void stop() { + if (beeperHandle != null) + beeperHandle.cancel(true); + // try { + // scheduler.awaitTermination(2, TimeUnit.SECONDS); + // } catch (InterruptedException e) { + // logger.error("Scheduler termination error", e); + // } + logger.info("stop"); + } +} diff --git a/server/hu.user.mediacube.indexer/src/main/java/hu/user/mediacube/indexer/ObjectMapperConfiguration.java b/server/hu.user.mediacube.indexer/src/main/java/hu/user/mediacube/indexer/ObjectMapperConfiguration.java new file mode 100644 index 00000000..5a370df9 --- /dev/null +++ b/server/hu.user.mediacube.indexer/src/main/java/hu/user/mediacube/indexer/ObjectMapperConfiguration.java @@ -0,0 +1,19 @@ +package hu.user.mediacube.indexer; + +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; + +import com.fasterxml.jackson.annotation.JsonInclude.Include; +import com.fasterxml.jackson.databind.DeserializationFeature; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.SerializationFeature; + +@Configuration +public class ObjectMapperConfiguration { + + @Bean + public ObjectMapper objectMapper() { + return new ObjectMapper().disable(DeserializationFeature.FAIL_ON_UNKNOWN_PROPERTIES).disable(SerializationFeature.WRITE_DATES_AS_TIMESTAMPS) + .setSerializationInclusion(Include.NON_NULL).enable(SerializationFeature.INDENT_OUTPUT); + } +} diff --git a/server/hu.user.mediacube.indexer/src/main/java/hu/user/mediacube/indexer/RESTController.java b/server/hu.user.mediacube.indexer/src/main/java/hu/user/mediacube/indexer/RESTController.java index 58fcafcc..a5dcf215 100644 --- a/server/hu.user.mediacube.indexer/src/main/java/hu/user/mediacube/indexer/RESTController.java +++ b/server/hu.user.mediacube.indexer/src/main/java/hu/user/mediacube/indexer/RESTController.java @@ -1,54 +1,23 @@ package hu.user.mediacube.indexer; -import java.time.ZoneId; -import java.time.ZonedDateTime; -import java.util.Date; import java.util.List; -import org.apache.lucene.document.Document; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Autowired; +import org.springframework.http.HttpStatus; +import org.springframework.http.ResponseEntity; import org.springframework.web.bind.annotation.GetMapping; import org.springframework.web.bind.annotation.PathVariable; import org.springframework.web.bind.annotation.RequestMapping; import org.springframework.web.bind.annotation.RestController; +import hu.user.mediacube.indexer.db.IndexerDescription; import hu.user.mediacube.indexer.lucene.LuceneIndexer; @RestController @RequestMapping("rest") public class RESTController { - public class CustomResponse { - private int age; - private String name; - private ZonedDateTime date; - - public int getAge() { - return age; - } - - public ZonedDateTime getDate() { - return date; - } - - public String getName() { - return name; - } - - public void setAge(int age) { - this.age = age; - } - - public void setDate(ZonedDateTime date) { - this.date = date; - } - - public void setName(String name) { - this.name = name; - } - - } private static Logger logger = LoggerFactory.getLogger(RESTController.class); @@ -69,19 +38,29 @@ public class RESTController { GreetingService greetingService; @Autowired private LuceneIndexer indexer; + @Autowired + private IndexerService indexerService; - @GetMapping(path = "/{id}", produces = "application/json") - public CustomResponse getBook(@PathVariable int id) { - logger.info("getBook"); - CustomResponse result = new CustomResponse(); - result.setAge(id); - result.setName(greetingService.greet(id)); - result.setDate(ZonedDateTime.ofInstant(new Date().toInstant(), ZoneId.systemDefault())); - return result; + @GetMapping(path = "/create", produces = "application/json") + public ResponseEntity create() { + indexerService.run(); + return new ResponseEntity<>("Index complete", HttpStatus.OK); + } + + @GetMapping(path = "/reset", produces = "application/json") + public ResponseEntity reset() { + try { + indexer.reset(); + } catch (Exception e) { + logger.error("Indexer reset error", e); + return new ResponseEntity<>(e.getMessage(), HttpStatus.INTERNAL_SERVER_ERROR); + } + return new ResponseEntity<>("Reset complete", HttpStatus.OK); } @GetMapping(path = "/search/{queryString}/{skipRecords}/{takeRecords}", produces = "application/json") - public List search(@PathVariable String queryString, @PathVariable int skipRecords, @PathVariable int takeRecords) { + public List search(@PathVariable String queryString, @PathVariable int skipRecords, @PathVariable int takeRecords) { return indexer.search(queryString, skipRecords, takeRecords); } + } \ No newline at end of file diff --git a/server/hu.user.mediacube.indexer/src/main/java/hu/user/mediacube/indexer/db/IndexerDescription.java b/server/hu.user.mediacube.indexer/src/main/java/hu/user/mediacube/indexer/db/IndexerDescription.java new file mode 100644 index 00000000..8ee8591a --- /dev/null +++ b/server/hu.user.mediacube.indexer/src/main/java/hu/user/mediacube/indexer/db/IndexerDescription.java @@ -0,0 +1,185 @@ +package hu.user.mediacube.indexer.db; + +import java.sql.Timestamp; +import java.util.Iterator; + +import org.apache.lucene.document.Document; +import org.apache.lucene.index.IndexableField; + +public class IndexerDescription { + private Long itemId; + private Long mediaId; + private String itemTitle; + private String mediaTitle; + private String mediaHouseId; + private Timestamp archived; + private String description; + private Long length; + private Long mediaItemTypeId; + private String mediaItemTypeName; + private Long mediaFileId; + private String mediaFileHouseId; + private Integer mediaFileCount; + + public IndexerDescription() { + + } + + public IndexerDescription(Document doc, String highlight) { + description = highlight; + Iterator iterator = doc.iterator(); + while (iterator.hasNext()) { + IndexableField field = iterator.next(); + switch (field.name()) { + case "itemId": { + itemId = (Long) field.numericValue(); + break; + } + case "mediaId": { + mediaId = (Long) field.numericValue(); + break; + } + case "itemTitle": { + itemTitle = field.stringValue(); + break; + } + case "mediaTitle": { + mediaTitle = field.stringValue(); + break; + } + case "mediaHouseId": { + mediaHouseId = field.stringValue(); + break; + } + case "length": { + length = (Long) field.numericValue(); + break; + } + case "mediaItemTypeId": { + mediaItemTypeId = (Long) field.numericValue(); + break; + } + case "mediaItemTypeName": { + mediaItemTypeName = field.stringValue(); + break; + } + case "mediaFileId": { + mediaFileId = (Long) field.numericValue(); + break; + } + case "mediaFileHouseId": { + mediaFileHouseId = field.stringValue(); + break; + } + case "mediaFileCount": { + mediaFileCount = (Integer) field.numericValue(); + break; + } + } + } + } + + public Timestamp getArchived() { + return archived; + } + + public String getDescription() { + return description; + } + + public Long getItemId() { + return itemId; + } + + public String getItemTitle() { + return itemTitle; + } + + public Long getLength() { + return length; + } + + public Integer getMediaFileCount() { + return mediaFileCount; + } + + public String getMediaFileHouseId() { + return mediaFileHouseId; + } + + public Long getMediaFileId() { + return mediaFileId; + } + + public String getMediaHouseId() { + return mediaHouseId; + } + + public Long getMediaId() { + return mediaId; + } + + public Long getMediaItemTypeId() { + return mediaItemTypeId; + } + + public String getMediaItemTypeName() { + return mediaItemTypeName; + } + + public String getMediaTitle() { + return mediaTitle; + } + + public void setArchived(Timestamp archived) { + this.archived = archived; + } + + public void setDescription(String description) { + this.description = description; + } + + public void setItemId(Long itemId) { + this.itemId = itemId; + } + + public void setItemTitle(String itemTitle) { + this.itemTitle = itemTitle; + } + + public void setLength(Long length) { + this.length = length; + } + + public void setMediaFileCount(Integer mediaFileCount) { + this.mediaFileCount = mediaFileCount; + } + + public void setMediaFileHouseId(String mediaFileHouseId) { + this.mediaFileHouseId = mediaFileHouseId; + } + + public void setMediaFileId(Long mediaFileId) { + this.mediaFileId = mediaFileId; + } + + public void setMediaHouseId(String mediaHouseId) { + this.mediaHouseId = mediaHouseId; + } + + public void setMediaId(Long mediaId) { + this.mediaId = mediaId; + } + + public void setMediaItemTypeId(Long mediaItemTypeId) { + this.mediaItemTypeId = mediaItemTypeId; + } + + public void setMediaItemTypeName(String mediaItemTypeName) { + this.mediaItemTypeName = mediaItemTypeName; + } + + public void setMediaTitle(String mediaTitle) { + this.mediaTitle = mediaTitle; + } +} diff --git a/server/hu.user.mediacube.indexer/src/main/java/hu/user/mediacube/indexer/db/IndexerViewMapper.java b/server/hu.user.mediacube.indexer/src/main/java/hu/user/mediacube/indexer/db/IndexerViewMapper.java new file mode 100644 index 00000000..c98d08be --- /dev/null +++ b/server/hu.user.mediacube.indexer/src/main/java/hu/user/mediacube/indexer/db/IndexerViewMapper.java @@ -0,0 +1,34 @@ +package hu.user.mediacube.indexer.db; + +import java.util.List; + +import org.apache.ibatis.annotations.Mapper; +import org.apache.ibatis.annotations.Result; +import org.apache.ibatis.annotations.Results; +import org.apache.ibatis.annotations.SelectProvider; +import org.apache.ibatis.type.JdbcType; + +//SELECT * FROM VW_INDEXER FETCH FIRST 10000 ROWS ONLY + +@Mapper +public interface IndexerViewMapper { + + @SelectProvider(type = IndexerViewSqlProvider.class, method = "selectAll") + @Results({ @Result(column = "ITEMID", property = "itemId", jdbcType = JdbcType.BIGINT), + @Result(column = "ITEMTITLE", property = "itemTitle", jdbcType = JdbcType.VARCHAR), + @Result(column = "MEDIAID", property = "mediaId", jdbcType = JdbcType.BIGINT), + @Result(column = "MEDIATITLE", property = "mediaTitle", jdbcType = JdbcType.VARCHAR), + @Result(column = "MEDIAHOUSEID", property = "mediaHouseId", jdbcType = JdbcType.VARCHAR), + @Result(column = "ARCHIVED", property = "archived", jdbcType = JdbcType.TIMESTAMP), + @Result(column = "LENGTH", property = "length", jdbcType = JdbcType.BIGINT), + @Result(column = "MEDIAITEMTYPEID", property = "mediaItemTypeId", jdbcType = JdbcType.BIGINT), + @Result(column = "MEDIAITEMTYPENAME", property = "mediaItemTypeName", jdbcType = JdbcType.VARCHAR), + @Result(column = "MEDIAFILEID", property = "mediaFileId", jdbcType = JdbcType.BIGINT), + @Result(column = "MEDIAFILEHOUSEID", property = "mediaFileHouseId", jdbcType = JdbcType.VARCHAR), + @Result(column = "MEDIAFILECOUNT", property = "mediaFileCount", jdbcType = JdbcType.INTEGER), }) + List selectAll(); + + @SelectProvider(type = IndexerViewSqlProvider.class, method = "selectDescription") + String selectDescription(IndexerDescription description); + +} \ No newline at end of file diff --git a/server/hu.user.mediacube.indexer/src/main/java/hu/user/mediacube/indexer/db/IndexerViewSqlProvider.java b/server/hu.user.mediacube.indexer/src/main/java/hu/user/mediacube/indexer/db/IndexerViewSqlProvider.java new file mode 100644 index 00000000..4d455f60 --- /dev/null +++ b/server/hu.user.mediacube.indexer/src/main/java/hu/user/mediacube/indexer/db/IndexerViewSqlProvider.java @@ -0,0 +1,32 @@ +package hu.user.mediacube.indexer.db; + +import java.util.Map; + +import org.apache.ibatis.jdbc.SQL; + +public class IndexerViewSqlProvider { + + private Long getLongParam(Map parameter, String name) { + return parameter.containsKey(name) ? (Long) parameter.get(name) : null; + } + + public String selectAll() { + SQL sql = new SQL(); + sql.SELECT("*"); + sql.FROM("VW_INDEXER").FETCH_FIRST_ROWS_ONLY(100); + return sql.toString(); + } + + public String selectDescription(IndexerDescription description) { + // Long itemId = getLongParam(parameter, "itemId"); + // Long mediaId = getLongParam(parameter, "mediaId"); + // Long mediaFileId = getLongParam(parameter, "mediaFileId"); + + SQL sql = new SQL(); + sql.SELECT("FULLDESC(itemId, mediaId, mediaFileId)"); + sql.FROM("VW_ITEMS"); + sql.WHERE("itemId = #{itemId,jdbcType=BIGINT}").AND().WHERE("mediaId = #{mediaId,jdbcType=BIGINT}").AND() + .WHERE("mediaFileId = #{mediaFileId,jdbcType=BIGINT}"); + return sql.toString(); + } +} diff --git a/server/hu.user.mediacube.indexer/src/main/java/hu/user/mediacube/indexer/lucene/LuceneIndexer.java b/server/hu.user.mediacube.indexer/src/main/java/hu/user/mediacube/indexer/lucene/LuceneIndexer.java index d14d62ef..db21aece 100644 --- a/server/hu.user.mediacube.indexer/src/main/java/hu/user/mediacube/indexer/lucene/LuceneIndexer.java +++ b/server/hu.user.mediacube.indexer/src/main/java/hu/user/mediacube/indexer/lucene/LuceneIndexer.java @@ -12,8 +12,10 @@ import javax.annotation.PreDestroy; import org.apache.lucene.analysis.standard.StandardAnalyzer; import org.apache.lucene.document.Document; import org.apache.lucene.document.Field.Store; -import org.apache.lucene.document.LongPoint; +import org.apache.lucene.document.SortedDocValuesField; import org.apache.lucene.document.SortedNumericDocValuesField; +import org.apache.lucene.document.StoredField; +import org.apache.lucene.document.StringField; import org.apache.lucene.document.TextField; import org.apache.lucene.index.DirectoryReader; import org.apache.lucene.index.IndexReader; @@ -24,12 +26,13 @@ import org.apache.lucene.search.IndexSearcher; import org.apache.lucene.search.Query; import org.apache.lucene.search.ScoreDoc; import org.apache.lucene.search.Sort; +import org.apache.lucene.search.SortField; import org.apache.lucene.search.SortField.Type; -import org.apache.lucene.search.SortedNumericSortField; import org.apache.lucene.search.TopDocs; import org.apache.lucene.search.uhighlight.UnifiedHighlighter; import org.apache.lucene.store.Directory; import org.apache.lucene.store.SimpleFSDirectory; +import org.apache.lucene.util.BytesRef; import org.slf4j.Logger; import org.slf4j.LoggerFactory; import org.springframework.beans.factory.annotation.Autowired; @@ -37,33 +40,73 @@ import org.springframework.stereotype.Component; import org.springframework.util.FileSystemUtils; import hu.user.mediacube.indexer.JsonProperties; -import hu.user.mediacube.indexer.db.Mediadescription; +import hu.user.mediacube.indexer.db.IndexerDescription; @Component public class LuceneIndexer { private static Logger logger = LoggerFactory.getLogger(LuceneIndexer.class); private static final String DESCRIPTION = "description"; + // private static Field[] fields; + // static { + // FieldType TYPE = new FieldType(); + // TYPE.setDocValuesType(DocValuesType.SORTED_NUMERIC); + // TYPE.freeze(); + // + // fields={new Field("itemId","",TYPE)}; + // } @Autowired private JsonProperties config; private Directory directoryIndex; private IndexWriterConfig indexWriterConfig; private IndexWriter indexWritter; private StandardAnalyzer analyzer; + private Path indexPath; - public void index(Mediadescription mediadescription, boolean commit) throws IOException { + private void addInt(Document document, String name, Integer value) { + document.add(new StoredField(name, value)); + document.add(new SortedNumericDocValuesField(name, value)); + } + + private void addLong(Document document, String name, Long value) { + document.add(new StoredField(name, value)); + document.add(new SortedNumericDocValuesField(name, value)); + } + + private void addString(Document document, String name, String value) { + document.add(new StringField(name, value, Store.YES)); + document.add(new SortedDocValuesField(name, new BytesRef(value))); + } + + public void commit() throws IOException { + indexWritter.commit(); + } + + public void index(IndexerDescription indexerDescription) throws IOException { Document document = new Document(); - document.add(new SortedNumericDocValuesField("id", mediadescription.getId())); - document.add(new LongPoint("itemid", mediadescription.getItemid())); - document.add(new LongPoint("mediaid", mediadescription.getMediaid())); - document.add(new LongPoint("mediafileid", mediadescription.getMediafileid())); - document.add(new TextField(DESCRIPTION, mediadescription.getDescription(), Store.YES)); + addLong(document, "itemId", indexerDescription.getItemId()); + addString(document, "itemTitle", indexerDescription.getItemTitle()); + addLong(document, "mediaId", indexerDescription.getMediaId()); + addLong(document, "mediaFileId", indexerDescription.getMediaFileId()); + addString(document, "mediaTitle", indexerDescription.getMediaTitle()); + addString(document, "mediaHouseId", indexerDescription.getMediaHouseId()); + addString(document, "mediaItemTypeName", indexerDescription.getMediaItemTypeName()); + addLong(document, "mediaItemTypeId", indexerDescription.getMediaItemTypeId()); + addInt(document, "mediaFileCount", indexerDescription.getMediaFileCount()); + addString(document, "mediaFileHouseId", indexerDescription.getMediaFileHouseId()); + addString(document, "archived", indexerDescription.getArchived().toString()); + addLong(document, "length", indexerDescription.getLength()); + document.add(new TextField(DESCRIPTION, indexerDescription.getDescription(), Store.YES)); indexWritter.addDocument(document); - if (commit) - indexWritter.commit(); } - public List search(String queryString, int skipRecords, int takeRecords) { - List ret = new ArrayList(); + public void reset() throws Exception { + stop(); + FileSystemUtils.deleteRecursively(indexPath); + start(); + } + + public List search(String queryString, int skipRecords, int takeRecords) { + List ret = new ArrayList<>(); IndexReader indexReader = null; IndexSearcher searcher = null; try { @@ -71,21 +114,26 @@ public class LuceneIndexer { searcher = new IndexSearcher(indexReader); Query query = new QueryParser(DESCRIPTION, analyzer).parse(queryString); + // SortField sort = new SortedNumericSortField("id", Type.LONG, true); + SortField sort = new SortField("mediaTitle", Type.STRING, false); - TopDocs results = searcher.search(query, skipRecords + takeRecords, new Sort(new SortedNumericSortField("id", Type.LONG, true))); + //mediahouseid,mediatitle, mediafilehouseid,archived,hossz + TopDocs results = searcher.search(query, skipRecords + takeRecords, new Sort(sort)); //TopDocs results = searcher.search(query, skipRecords + takeRecords); ScoreDoc[] scoreDocs = results.scoreDocs; UnifiedHighlighter highlighter = new UnifiedHighlighter(searcher, analyzer); //highlighter.setFormatter(formatter); + logger.info("Total result: {}", results.totalHits); for (int i = skipRecords; i < results.totalHits.value; i++) { if (i > (skipRecords + takeRecords) - 1) { break; } Document doc = searcher.doc(scoreDocs[i].doc); - ret.add(doc); - String[] fragments = highlighter.highlight(DESCRIPTION, query, results); - logger.info("Item " + doc.getValues("id")[0] + " : " + fragments[0]); + String highlight = fragments[0]; + ret.add(new IndexerDescription(doc, highlight)); + + logger.info("Item " + doc.get("mediaTitle") + " : " + fragments[0]); } } catch (Exception e) { logger.error("Search", e); @@ -103,12 +151,11 @@ public class LuceneIndexer { @PostConstruct public void start() throws Exception { try { - Path path = Paths.get(config.getIndexLocation()); - if (config.isDropIndex()) - FileSystemUtils.deleteRecursively(path); - directoryIndex = new SimpleFSDirectory(path); analyzer = new StandardAnalyzer(); indexWriterConfig = new IndexWriterConfig(analyzer); + + indexPath = Paths.get(config.getIndexLocation()); + directoryIndex = new SimpleFSDirectory(indexPath); indexWritter = new IndexWriter(directoryIndex, indexWriterConfig); } catch (Exception e) { logger.error("Could not create indexer", e); diff --git a/server/hu.user.mediacube.indexer/src/main/resources/base.xml b/server/hu.user.mediacube.indexer/src/main/resources/base.xml index af37d27b..cc7cb51b 100644 --- a/server/hu.user.mediacube.indexer/src/main/resources/base.xml +++ b/server/hu.user.mediacube.indexer/src/main/resources/base.xml @@ -6,7 +6,7 @@ Base logback configuration provided for compatibility with Spring Boot 1.1 - + diff --git a/server/user.jobengine.osgi.server/src/user/jobengine/zk/model/MediaCubeAccount.java b/server/user.jobengine.osgi.server/src/user/jobengine/zk/model/MediaCubeAccount.java new file mode 100644 index 00000000..efdfea4a --- /dev/null +++ b/server/user.jobengine.osgi.server/src/user/jobengine/zk/model/MediaCubeAccount.java @@ -0,0 +1,32 @@ +package user.jobengine.zk.model; + +public class MediaCubeAccount { + private String user; + private String password; + private String email; + + public String getEmail() { + return email; + } + + public String getPassword() { + return password; + } + + public String getUser() { + return user; + } + + public void setEmail(String email) { + this.email = email; + } + + public void setPassword(String password) { + this.password = password; + } + + public void setUser(String user) { + this.user = user; + } + +} diff --git a/server/user.jobengine.osgi.server/src/user/jobengine/zk/model/MediaCubeAuthentication.java b/server/user.jobengine.osgi.server/src/user/jobengine/zk/model/MediaCubeAuthentication.java new file mode 100644 index 00000000..9802422d --- /dev/null +++ b/server/user.jobengine.osgi.server/src/user/jobengine/zk/model/MediaCubeAuthentication.java @@ -0,0 +1,106 @@ +package user.jobengine.zk.model; + +import java.util.List; + +public class MediaCubeAuthentication { + private boolean authEnabled; + private String adHost; + private int adNonSecurePort; + private String adBaseDn; + private List adAdminMap; + private List adSubmitterMap; + private List adEditorMap; + private List localAccounts; + private String localAdmins; + private String localSubmitters; + private String localEditors; + + public List getAdAdminMap() { + return adAdminMap; + } + + public String getAdBaseDn() { + return adBaseDn; + } + + public List getAdEditorMap() { + return adEditorMap; + } + + public String getAdHost() { + return adHost; + } + + public int getAdNonSecurePort() { + return adNonSecurePort; + } + + public List getAdSubmitterMap() { + return adSubmitterMap; + } + + public List getLocalAccounts() { + return localAccounts; + } + + public String getLocalAdmins() { + return localAdmins; + } + + public String getLocalEditors() { + return localEditors; + } + + public String getLocalSubmitters() { + return localSubmitters; + } + + public boolean isAuthEnabled() { + return authEnabled; + } + + public void setAdAdminMap(List adAdminMap) { + this.adAdminMap = adAdminMap; + } + + public void setAdBaseDn(String adBaseDn) { + this.adBaseDn = adBaseDn; + } + + public void setAdEditorMap(List adEditorMap) { + this.adEditorMap = adEditorMap; + } + + public void setAdHost(String adHost) { + this.adHost = adHost; + } + + public void setAdNonSecurePort(int adNonSecurePort) { + this.adNonSecurePort = adNonSecurePort; + } + + public void setAdSubmitterMap(List adSubmitterMap) { + this.adSubmitterMap = adSubmitterMap; + } + + public void setAuthEnabled(boolean authEnabled) { + this.authEnabled = authEnabled; + } + + public void setLocalAccounts(List localAccounts) { + this.localAccounts = localAccounts; + } + + public void setLocalAdmins(String localAdmins) { + this.localAdmins = localAdmins; + } + + public void setLocalEditors(String localEditors) { + this.localEditors = localEditors; + } + + public void setLocalSubmitters(String localSubmitters) { + this.localSubmitters = localSubmitters; + } + +} diff --git a/server/user.jobengine.osgi.server/src/user/jobengine/zk/model/MediaCubeConfig.java b/server/user.jobengine.osgi.server/src/user/jobengine/zk/model/MediaCubeConfig.java new file mode 100644 index 00000000..d56b5e7c --- /dev/null +++ b/server/user.jobengine.osgi.server/src/user/jobengine/zk/model/MediaCubeConfig.java @@ -0,0 +1,14 @@ +package user.jobengine.zk.model; + +public class MediaCubeConfig { + private MediaCubeAuthentication authentication; + + public MediaCubeAuthentication getAuthentication() { + return authentication; + } + + public void setAuthentication(MediaCubeAuthentication authentication) { + this.authentication = authentication; + } + +} diff --git a/server/user.jobengine.osgi.server/src/user/jobengine/zk/util/SessionUtil.java b/server/user.jobengine.osgi.server/src/user/jobengine/zk/util/SessionUtil.java index 32a96217..f1e58d64 100644 --- a/server/user.jobengine.osgi.server/src/user/jobengine/zk/util/SessionUtil.java +++ b/server/user.jobengine.osgi.server/src/user/jobengine/zk/util/SessionUtil.java @@ -22,12 +22,17 @@ import com.ibm.nosql.json.api.BasicDBList; import com.ibm.nosql.json.api.BasicDBObject; import com.ibm.nosql.json.api.QueryBuilder; +import user.commons.config.JSONConfig; import user.commons.nosql.NoSQLUtils; import user.jobengine.server.IJobChangedListener; import user.jobengine.server.JobEngine; import user.jobengine.zk.model.IndexModel; +import user.jobengine.zk.model.MaestroConfig; +import user.jobengine.zk.model.MediaCubeConfig; public class SessionUtil { + private static final String JOBENGINE_MEDIACUBE_CONFIG = "jobengine.mediacube.config"; + private static final String JOBENGINE_MAESTRO_CONFIG = "jobengine.maestro.config"; private static final String SEARCH_FILTER = "searchFilter"; private static final String COLUMN_ORDERS = "columnOrders"; private static final String PAGE_SIZE = "pageSize"; @@ -85,15 +90,42 @@ public class SessionUtil { } private static Properties getConfiguration() { + + MediaCubeConfig mediaCubeConfig = getMediaCubeConfig(); Properties cfg = new Properties(); try (InputStream is = new FileInputStream(System.getProperty(AUTH_LOCATION))) { cfg.load(is); } catch (Exception e) { logger.catching(e); } + return cfg; } + public static MaestroConfig getMaestroConfig() { + MaestroConfig result = null; + try { + result = (MaestroConfig) getAttribute(JOBENGINE_MAESTRO_CONFIG); + if (result == null) { + result = JSONConfig.read(JOBENGINE_MAESTRO_CONFIG, MaestroConfig.class); + setAttribute(JOBENGINE_MAESTRO_CONFIG, result); + } + } catch (Exception e) { + logger.catching(e); + } + return result; + } + + public static MediaCubeConfig getMediaCubeConfig() { + MediaCubeConfig result = null; + try { + result = JSONConfig.read(JOBENGINE_MEDIACUBE_CONFIG, MediaCubeConfig.class); + } catch (Exception e) { + logger.catching(e); + } + return result; + } + static public String getQueryParameter(String key) { String result = null; HttpServletRequest request = (HttpServletRequest) Executions.getCurrent().getNativeRequest();