From: Vásáry Dániel Date: Wed, 2 Oct 2019 14:13:06 +0000 (+0000) Subject: git-tfs-id: [http://tfs.userrendszerhaz.hu:8080/tfs/DefaultCollection]$/MediaCube... X-Git-Url: http://git.useribm.hu/?a=commitdiff_plain;h=dd21a9c8236ff156d47a1b8ced6705f11c6f13b6;p=mediacube.git git-tfs-id: [tfs.userrendszerhaz.hu:8080/tfs/DefaultCollection]$/MediaCube;C31555 --- diff --git a/server/-configuration/run-mediacube-server-user.launch b/server/-configuration/run-mediacube-server-user.launch index 5192ce74..72ca10dd 100644 --- a/server/-configuration/run-mediacube-server-user.launch +++ b/server/-configuration/run-mediacube-server-user.launch @@ -19,7 +19,7 @@ - + diff --git a/server/hu.user.mediacube.indexer/src/main/java/hu/user/mediacube/indexer/IndexerService.java b/server/hu.user.mediacube.indexer/src/main/java/hu/user/mediacube/indexer/IndexerService.java index 6fdd9518..23e2b063 100644 --- a/server/hu.user.mediacube.indexer/src/main/java/hu/user/mediacube/indexer/IndexerService.java +++ b/server/hu.user.mediacube.indexer/src/main/java/hu/user/mediacube/indexer/IndexerService.java @@ -44,30 +44,31 @@ public class IndexerService implements Runnable { List indexerDescriptions = mapper.selectAll(); int i = 0; LogFactory.useLog4J2Logging(); + + int logStep = indexerDescriptions.size() / 10; + for (IndexerDescription indexerDescription : indexerDescriptions) { String description = mapper.selectDescription(indexerDescription); indexerDescription.setDescription(description); //logger.info("{}", indexerDescription); - boolean processed = ++i % 10 == 0; + boolean putLog = ++i % logStep == 0; + boolean lastRow = i == indexerDescriptions.size(); try { indexer.index(indexerDescription); - if (i == indexerDescriptions.size()) + + if (putLog || lastRow) + logger.info("Processed {} / {}", i, indexerDescriptions.size()); + + if (lastRow) { indexer.commit(); + logger.error("Index committed"); + } } catch (IOException e) { logger.error("Can not create index, caused by {} : {}", e.getClass().getSimpleName(), e.getMessage(), e); break; } - if (processed) { - logger.info("Processed {} / {}", i, indexerDescriptions.size()); - } - - } - try { - indexer.commit(); - } catch (IOException e) { - logger.error("Commit error", e); } } diff --git a/server/hu.user.mediacube.indexer/src/main/java/hu/user/mediacube/indexer/RESTController.java b/server/hu.user.mediacube.indexer/src/main/java/hu/user/mediacube/indexer/RESTController.java index d2bf1662..1bddf96c 100644 --- a/server/hu.user.mediacube.indexer/src/main/java/hu/user/mediacube/indexer/RESTController.java +++ b/server/hu.user.mediacube.indexer/src/main/java/hu/user/mediacube/indexer/RESTController.java @@ -16,7 +16,7 @@ import hu.user.mediacube.indexer.db.IndexerDescription; import hu.user.mediacube.indexer.lucene.LuceneIndexer; @RestController -@RequestMapping("rest") +@RequestMapping("indexer") public class RESTController { private static Logger logger = LoggerFactory.getLogger(RESTController.class); diff --git a/server/hu.user.mediacube.indexer/src/main/resources/application.properties b/server/hu.user.mediacube.indexer/src/main/resources/application.properties index 9b0951ed..ccc948e9 100644 --- a/server/hu.user.mediacube.indexer/src/main/resources/application.properties +++ b/server/hu.user.mediacube.indexer/src/main/resources/application.properties @@ -1,3 +1,4 @@ +server.port=8081 spring.main.banner-mode=off spring.output.ansi.enabled=always spring.datasource.driver-class-name=com.ibm.db2.jcc.DB2Driver diff --git a/server/user.jobengine.executors/src/user/jobengine/server/steps/TSMBackupStep.java b/server/user.jobengine.executors/src/user/jobengine/server/steps/TSMBackupStep.java index 71ad952e..b00e23e9 100644 --- a/server/user.jobengine.executors/src/user/jobengine/server/steps/TSMBackupStep.java +++ b/server/user.jobengine.executors/src/user/jobengine/server/steps/TSMBackupStep.java @@ -93,6 +93,7 @@ public class TSMBackupStep extends JobStep { logger.error(marker, "Az '{}' állomány .catched jelző állománya nem törölhető. Az újabb archiválási kísérlethez annak kézi eltávolítása szükséges!", sourceMediaFile.getName()); + mediaCubeMedia.remove(); } else { saveMetadata(mediaCubeMedia, sourceMediaFile, existingMediaId); logger.info(marker, "Az '{}' archiválása sikeres.", sourceFileName); diff --git a/server/user.jobengine.osgi.db/META-INF/MANIFEST.MF b/server/user.jobengine.osgi.db/META-INF/MANIFEST.MF index 11d166a0..b982f35a 100644 --- a/server/user.jobengine.osgi.db/META-INF/MANIFEST.MF +++ b/server/user.jobengine.osgi.db/META-INF/MANIFEST.MF @@ -14,6 +14,9 @@ Require-Bundle: user.jobengine.osgi.commons;bundle-version="1.0.0", Service-Component: OSGI-INF/component.xml Bundle-ActivationPolicy: lazy Import-Package: com.fasterxml.jackson.annotation;version="2.4.5", - org.apache.logging.log4j;version="2.8.2" + javax.ws.rs.client, + javax.ws.rs.core, + org.apache.logging.log4j;version="2.8.2", + org.jboss.resteasy.client.jaxrs Export-Package: user.jobengine.db diff --git a/server/user.jobengine.osgi.db/src/user/jobengine/db/DynamicAttributes.java b/server/user.jobengine.osgi.db/src/user/jobengine/db/DynamicAttributes.java index f8d1b2be..bf17346b 100644 --- a/server/user.jobengine.osgi.db/src/user/jobengine/db/DynamicAttributes.java +++ b/server/user.jobengine.osgi.db/src/user/jobengine/db/DynamicAttributes.java @@ -54,7 +54,7 @@ public class DynamicAttributes extends JSONBase implements Serializable { private void initalize() { ItemManager.getInstance().traceIn(); if (attributes == null) { - if (getItemType() != null) { + if (getItemType() != null && !getItemType().getIsStatic()) { attributes = new HashMap(); List metadatas = getItemType().getMetadatas(); if (metadatas != null) { diff --git a/server/user.jobengine.osgi.db/src/user/jobengine/db/DynamicAttributesDAO.java b/server/user.jobengine.osgi.db/src/user/jobengine/db/DynamicAttributesDAO.java index 9e76a5ed..cb5e2b92 100644 --- a/server/user.jobengine.osgi.db/src/user/jobengine/db/DynamicAttributesDAO.java +++ b/server/user.jobengine.osgi.db/src/user/jobengine/db/DynamicAttributesDAO.java @@ -13,8 +13,6 @@ import user.commons.BaseType; import user.commons.IEntityBase; public class DynamicAttributesDAO extends EntityBaseDAO { - private Converter converter; - public class BooleanConverter implements IValueConverter { @Override @@ -28,6 +26,8 @@ public class DynamicAttributesDAO extends EntityBaseDAO { } } + private Converter converter; + public DynamicAttributesDAO(ItemManager manager) { super(manager); HashMap converters = new HashMap(); @@ -37,25 +37,6 @@ public class DynamicAttributesDAO extends EntityBaseDAO { } - void addAsDomain(DefaultContext context, IEntityBase entity, Metadata metadata, List domains, boolean modify) throws SQLException { - DomainIndexDAO indexdao = (DomainIndexDAO) manager.getBaseDAO(DomainIndex.class); - - if (domains != null && domains.size() > 0) { - logger.debug("Adding domain attribute " + metadata.getMetadataElement().getName() + " , value count: " + domains.size()); - long domainCategoryId = metadata.getMetadataElement().getDomainCategoryId(); - DomainCategory dc = (DomainCategory) manager.get(DomainCategory.class, domainCategoryId); - dc.setDomains(domains); - DomainDAO dao = (DomainDAO) manager.getBaseDAO(Domain.class); - dao.addMissing(context, dc); - if (modify) - indexdao.removeAll(context, entity, metadata); - indexdao.addAll(context, entity, metadata, dc); - } else { - if (modify) - indexdao.removeAll(context, entity, metadata); - } - } - @Override public void add(DefaultContext context, IEntityBase entity) { manager.traceIn(); @@ -63,7 +44,7 @@ public class DynamicAttributesDAO extends EntityBaseDAO { da.checkParameter("ID", da.getId(), false); Map attributes = da.getAttributes(); ItemType itemType = (ItemType) manager.get(ItemType.class, da.getItemTypeId()); - if (itemType != null) { + if (itemType != null && !itemType.getIsStatic()) { List metadatas = itemType.getMetadatas(); if (metadatas != null) { for (Metadata metadata : metadatas) { @@ -118,15 +99,23 @@ public class DynamicAttributesDAO extends EntityBaseDAO { manager.traceOut(); } - @SuppressWarnings("unchecked") - Domains getAsDomain(DefaultContext context, IEntityBase entity, Metadata metadata) throws SQLException { - logger.debug("Reading domain attribute " + metadata.getMetadataElement().getName()); - DomainDAO dao = (DomainDAO) manager.getBaseDAO(Domain.class); - List domains = (List) (List) dao.getAll(context, entity, metadata); - Domains result = null; - if (domains != null) - result = new Domains(domains); - return result; + void addAsDomain(DefaultContext context, IEntityBase entity, Metadata metadata, List domains, boolean modify) throws SQLException { + DomainIndexDAO indexdao = (DomainIndexDAO) manager.getBaseDAO(DomainIndex.class); + + if (domains != null && domains.size() > 0) { + logger.debug("Adding domain attribute " + metadata.getMetadataElement().getName() + " , value count: " + domains.size()); + long domainCategoryId = metadata.getMetadataElement().getDomainCategoryId(); + DomainCategory dc = (DomainCategory) manager.get(DomainCategory.class, domainCategoryId); + dc.setDomains(domains); + DomainDAO dao = (DomainDAO) manager.getBaseDAO(Domain.class); + dao.addMissing(context, dc); + if (modify) + indexdao.removeAll(context, entity, metadata); + indexdao.addAll(context, entity, metadata, dc); + } else { + if (modify) + indexdao.removeAll(context, entity, metadata); + } } void get(IEntityBase entity) { @@ -184,6 +173,17 @@ public class DynamicAttributesDAO extends EntityBaseDAO { manager.traceOut(); } + @SuppressWarnings("unchecked") + Domains getAsDomain(DefaultContext context, IEntityBase entity, Metadata metadata) throws SQLException { + logger.debug("Reading domain attribute " + metadata.getMetadataElement().getName()); + DomainDAO dao = (DomainDAO) manager.getBaseDAO(Domain.class); + List domains = (List) (List) dao.getAll(context, entity, metadata); + Domains result = null; + if (domains != null) + result = new Domains(domains); + return result; + } + @Override public void modify(DefaultContext context, IEntityBase entity) { manager.traceIn(); @@ -250,12 +250,6 @@ public class DynamicAttributesDAO extends EntityBaseDAO { manager.traceOut(); } - void removeAsDomain(DefaultContext context, IEntityBase entity, Metadata metadata) throws SQLException { - logger.debug("Removing domain attribute " + metadata.getMetadataElement().getName()); - DomainIndexDAO indexdao = (DomainIndexDAO) manager.getBaseDAO(DomainIndex.class); - indexdao.removeAll(context, entity, metadata); - } - @Override public void remove(DefaultContext context, IEntityBase entity) { manager.traceIn(); @@ -291,4 +285,10 @@ public class DynamicAttributesDAO extends EntityBaseDAO { } manager.traceOut(); } + + void removeAsDomain(DefaultContext context, IEntityBase entity, Metadata metadata) throws SQLException { + logger.debug("Removing domain attribute " + metadata.getMetadataElement().getName()); + DomainIndexDAO indexdao = (DomainIndexDAO) manager.getBaseDAO(DomainIndex.class); + indexdao.removeAll(context, entity, metadata); + } } diff --git a/server/user.jobengine.osgi.db/src/user/jobengine/db/ItemManager.java b/server/user.jobengine.osgi.db/src/user/jobengine/db/ItemManager.java index 320029c2..9dcdcaf4 100644 --- a/server/user.jobengine.osgi.db/src/user/jobengine/db/ItemManager.java +++ b/server/user.jobengine.osgi.db/src/user/jobengine/db/ItemManager.java @@ -47,6 +47,7 @@ import user.jobengine.db.ItemManagerData.UserRight; * JobEngine adatbázis modell kezelő API. */ public class ItemManager extends MemoryCache implements IItemManager { + private static final String JOBENGINE_MEDIAFINDER_EXTERNAL = "jobengine.mediafinder.external"; private static final Logger logger = LogManager.getLogger(); public static String DBURL = "jobengine.db.url"; public static String DBUSERNAME = "jobengine.db.user"; @@ -70,8 +71,7 @@ public class ItemManager extends MemoryCache implements IItemManager { String dbPassword = null; private SignalServer signalServer; - private MediaFinder finder; - private LuceneMediaFinder luceneFinder; + private IMediaFinder finder; public ItemManager() { initializeItemManager(signalServer); @@ -81,10 +81,17 @@ public class ItemManager extends MemoryCache implements IItemManager { // System.getProperty("jobengine.mediafinder.external") == true // System.getProperty("jobengine.mediafinder.external.url") - if (System.getProperty("jobengine.mediafinder.external") == "false") { - finder = new MediaFinder(this); + boolean useLucene = false; + try { + useLucene = Boolean.parseBoolean(System.getProperty(JOBENGINE_MEDIAFINDER_EXTERNAL)); + } catch (Exception e) { + logger.info("Configuration property {} mising", JOBENGINE_MEDIAFINDER_EXTERNAL); + } + + if (useLucene) { + finder = new LuceneMediaFinder(); } else { - luceneFinder = new LuceneMediaFinder(); + finder = new MediaFinder(this); } } @@ -993,12 +1000,7 @@ public class ItemManager extends MemoryCache implements IItemManager { public SearchResult search(String criteria, List searchFilters, String orderBy, boolean orderAscending, int fromResult, int toResult) { traceIn(); - SearchResult result = null; - if (System.getProperty("jobengine.mediafinder.external") == "false") { - result = finder.search(criteria, searchFilters, orderBy, orderAscending, fromResult, toResult); - } else { - result = luceneFinder.search(criteria, searchFilters, orderBy, orderAscending, fromResult, toResult); - } + SearchResult result = finder.search(criteria, searchFilters, orderBy, orderAscending, fromResult, toResult); traceOut(); return result; } diff --git a/server/user.jobengine.osgi.db/src/user/jobengine/db/LuceneMediaFinder.java b/server/user.jobengine.osgi.db/src/user/jobengine/db/LuceneMediaFinder.java index 5b76f96b..5ed90c95 100644 --- a/server/user.jobengine.osgi.db/src/user/jobengine/db/LuceneMediaFinder.java +++ b/server/user.jobengine.osgi.db/src/user/jobengine/db/LuceneMediaFinder.java @@ -2,19 +2,26 @@ package user.jobengine.db; import java.util.List; -import org.jboss.resteasy.client.jaxrs.BasicAuthentication; +import javax.ws.rs.client.WebTarget; +import javax.ws.rs.core.Response; + import org.jboss.resteasy.client.jaxrs.ResteasyClient; import org.jboss.resteasy.client.jaxrs.ResteasyClientBuilder; public class LuceneMediaFinder implements IMediaFinder { + private WebTarget webTarget; + private String apiAddress = "http://localhost:8081/indexer"; + private String apiPath = "/search/*.*/0/10/1/0"; + @Override public SearchResult search(String criteria, List searchFilters, String orderBy, boolean orderAscending, int fromResult, int toResult) { // TODO Auto-generated method stub ResteasyClient client = new ResteasyClientBuilder().build(); - webTarget = client.target(apiAddress).register(new BasicAuthentication(apiUser, apiPwd)); - + webTarget = client.target(apiAddress); + WebTarget target = webTarget.path(apiPath); + Response response = target.request().get(); return null; } diff --git a/server/user.jobengine.osgi.db/src/user/jobengine/db/Metadata.java b/server/user.jobengine.osgi.db/src/user/jobengine/db/Metadata.java index 67b1b1d4..86b52516 100644 --- a/server/user.jobengine.osgi.db/src/user/jobengine/db/Metadata.java +++ b/server/user.jobengine.osgi.db/src/user/jobengine/db/Metadata.java @@ -33,10 +33,9 @@ public class Metadata extends EntityBase implements Serializable { private String parameter = null; /** - * Serializációs technikával másolatot készít egy Metadata objektumról. A - * DynamicAttributes osztály használja. - * - * @return Metadata objektum másolata. + * Serializ�ci�s technik�val m�solatot k�sz�t egy Metadata objektumr�l. A DynamicAttributes oszt�ly haszn�lja. + * + * @return Metadata objektum m�solata. */ Metadata copy() { Metadata result = null; @@ -96,6 +95,15 @@ public class Metadata extends EntityBase implements Serializable { return result; } + public String getFieldName() { + String fieldName = null; + if (staticTable != null) + fieldName = String.format(getPrefix() + getSqlField()); + else + fieldName = String.format("%1$s.%2$s", getTableName(), getColumnName()); + return fieldName; + } + public GuiType getGuiType() { return guiType; } @@ -176,6 +184,17 @@ public class Metadata extends EntityBase implements Serializable { return pojoField; } + public String getPrefix() { + String result = null; + if (staticTable == StaticTables.MEDIA) + result = String.format("mm."); + if (staticTable == StaticTables.ITEM) + result = String.format("ii."); + if (staticTable == StaticTables.FOLDER) + result = String.format("f."); + return result; + } + public String getRemoveSQL() { ItemType itemType = (ItemType) ItemManager.getInstance().retrieveCached(ItemType.class, getItemTypeId()); String tableName = itemType.getTableName(); @@ -195,26 +214,6 @@ public class Metadata extends EntityBase implements Serializable { return getStaticTable() != null ? null : "METADATA_" + getId(); } - public String getPrefix() { - String result = null; - if (staticTable == StaticTables.MEDIA) - result = String.format("mm."); - if (staticTable == StaticTables.ITEM) - result = String.format("ii."); - if (staticTable == StaticTables.FOLDER) - result = String.format("f."); - return result; - } - - public String getFieldName() { - String fieldName = null; - if (staticTable != null) - fieldName = String.format(getPrefix() + getSqlField()); - else - fieldName = String.format("%1$s.%2$s", getTableName(), getColumnName()); - return fieldName; - } - // public Object getValue() { // return value; // } diff --git a/server/user.jobengine.osgi.server/META-INF/MANIFEST.MF b/server/user.jobengine.osgi.server/META-INF/MANIFEST.MF index eb87cb54..c0fcecad 100644 --- a/server/user.jobengine.osgi.server/META-INF/MANIFEST.MF +++ b/server/user.jobengine.osgi.server/META-INF/MANIFEST.MF @@ -4,25 +4,12 @@ Bundle-Name: Server Bundle-SymbolicName: user.jobengine.osgi.server;singleton:=true Bundle-Version: 1.0.0 Service-Component: OSGI-INF/component.xml, OSGI-INF/componentBinder.xml -Import-Package: com.fasterxml.jackson.annotation;version="2.4.5", - com.fasterxml.jackson.core;version="2.4.5", - com.fasterxml.jackson.databind;version="2.4.5", - com.fasterxml.jackson.databind.util;version="2.4.5", - com.fasterxml.jackson.datatype.joda;version="2.4.5", - com.fasterxml.jackson.jaxrs.json;version="2.4.5", - javax.servlet;version="3.1.0", +Import-Package: javax.servlet;version="3.1.0", javax.servlet.http;version="3.1.0", - javax.ws.rs.client, - javax.ws.rs.core;version="2.0.1", org.apache.logging.log4j;version="2.8.2", org.apache.logging.log4j.message;version="2.8.2", org.eclipse.core.runtime.adaptor, org.eclipse.osgi.framework.console;version="1.1.0", - org.jboss.resteasy.client.jaxrs, - org.jboss.resteasy.plugins.providers, - org.jboss.resteasy.spi, - org.joda.time;version="2.2.0", - org.joda.time.base;version="2.2.0", org.osgi.framework;version="1.5.0", org.osgi.framework.wiring;version="1.2.0", org.osgi.util.tracker;version="1.4.0", diff --git a/server/user.jobengine.osgi.server/test/user/jobengine/server/IT/SupportTest.java b/server/user.jobengine.osgi.server/test/user/jobengine/server/IT/SupportTest.java new file mode 100644 index 00000000..62255305 --- /dev/null +++ b/server/user.jobengine.osgi.server/test/user/jobengine/server/IT/SupportTest.java @@ -0,0 +1,65 @@ +package user.jobengine.server.IT; + +import java.util.Date; + +import org.junit.AfterClass; +import org.junit.BeforeClass; +import org.junit.Test; + +import user.commons.logging.LogUtils; +import user.jobengine.db.IItemManager; +import user.jobengine.db.Item; +import user.jobengine.db.ItemManager; +import user.jobengine.db.Media; +import user.jobengine.db.MediaFile; + +public class SupportTest { + public class BD { + Date fBD; + Date nBD; + } + + protected static IItemManager manager = null; + + private static final String LOWRES_ROOT = "//10.10.1.27/mclowres/www/video"; + + private static final long[] duplicateLength = { 0 }; + + @BeforeClass + static public void setUpConnection() { + System.setProperty("jobengine.octopus.rundowns.name", "rundowns180620"); + System.setProperty("jobengine.octopus.stories.name", "stories180620"); + System.setProperty("jobengine.octopus.storyfolders.name", "storyfolders180620"); + + System.setProperty("jobengine.nosql.db.url", "jdbc:db2://10.228.198.1:50000/mccache:retrieveMessagesFromServerOnGetMessage=true;"); + System.setProperty("jobengine.nosql.db.user", "db2admin"); + System.setProperty("jobengine.nosql.db.password", "password"); + System.setProperty(ItemManager.DBURL, "jdbc:db2://10.228.198.1:50000/mediaarc:retrieveMessagesFromServerOnGetMessage=true;"); + System.setProperty(ItemManager.DBUSERNAME, "db2admin"); + System.setProperty(ItemManager.DBPASSWORD, "password"); + System.setProperty(LogUtils.LOGLEVEL, "INFO"); + // System.setProperty(ItemManager.DBURL, + + manager = new ItemManager(); + manager.connect(); + } + + @AfterClass + static public void tearDownConnection() { + manager.disconnect(); + } + + @Test + public void mergeItemStructure() throws Exception { + Item item = manager.createItem("Default", "Item1", "Itemdesc1", "Itemhouseid1"); + Media media = manager.createMedia("Default", "Media1", "Mediadesc1", "Mediahouseid1"); + item.appendMedia(media); + MediaFile mf1 = manager.createMediaFile("/xxx", manager.getFileType("Lowres"), manager.getSystemStore(true), media); + MediaFile mf2 = manager.createMediaFile("/yyy", manager.getFileType("Lowres"), manager.getSystemStore(true), media); + + item.add(); + + //manager.mergeItemStructure(item); + } + +}