From 341bad53a75541de76775e417243e0672cd38ffe Mon Sep 17 00:00:00 2001 From: =?utf8?q?V=C3=A1s=C3=A1ry=20D=C3=A1niel?= Date: Fri, 29 Sep 2017 14:46:32 +0000 Subject: [PATCH] git-tfs-id: [http://tfs.userrendszerhaz.hu:8080/tfs/DefaultCollection]$/MediaCube;C30489 --- .../debug-mediacube-remote.launch | 4 +- server/-configuration/dsm.opt | 3 + server/-configuration/log4j2.xml | 24 +++- server/-configuration/scheduledjobs.json | 3 +- server/-configuration/start-mediacube.sh | 11 ++ server/-product/DEPLOY-README.txt | 9 +- server/-product/pom.xml | 40 ++++++ .../jobtemplates/archive-item.xml | 7 ++ .../jobtemplates/archive-ondemand.xml | 5 +- .../server/steps/ArchiveItemSubmitStep.java | 6 +- .../server/steps/ArchiveListBuilderStep.java | 4 +- .../user/jobengine/server/steps/FakeStep.java | 11 +- .../steps/SelenioLowresTranscodeStep.java | 30 +++-- .../user/commons/IRemoteFileListCallback.java | 5 + .../src/user/commons/StoreUri.java | 96 ++++++++------- .../remotestore/FtpDirectoryLister.java | 11 ++ .../commons/remotestore/IDirectoryLister.java | 16 ++- .../remotestore/LocalDirectoryLister.java | 18 ++- .../remotestore/SambaDirectoryLister.java | 35 ++++-- .../user/commons/remotestore/TSMLister.java | 51 ++++---- .../src/user/jobengine/db/MemoryCache.java | 116 +++++++++--------- .../pages/search_items.zul | 10 +- .../user/jobengine/server/steps/IJobStep.java | 5 - 23 files changed, 339 insertions(+), 181 deletions(-) create mode 100644 server/-configuration/dsm.opt create mode 100644 server/-configuration/start-mediacube.sh create mode 100644 server/user.jobengine.osgi.commons/src/user/commons/IRemoteFileListCallback.java diff --git a/server/-configuration/debug-mediacube-remote.launch b/server/-configuration/debug-mediacube-remote.launch index f534e3bf..86198ac0 100644 --- a/server/-configuration/debug-mediacube-remote.launch +++ b/server/-configuration/debug-mediacube-remote.launch @@ -8,9 +8,9 @@ - + - + diff --git a/server/-configuration/dsm.opt b/server/-configuration/dsm.opt new file mode 100644 index 00000000..cdd39f70 --- /dev/null +++ b/server/-configuration/dsm.opt @@ -0,0 +1,3 @@ +SErvername MEDIACUBE +*traceflags api +*tracefile /opt/mediacube/tsm.trace \ No newline at end of file diff --git a/server/-configuration/log4j2.xml b/server/-configuration/log4j2.xml index 8debc3cb..3a675331 100644 --- a/server/-configuration/log4j2.xml +++ b/server/-configuration/log4j2.xml @@ -2,17 +2,28 @@ log/mediacube.log - log/$${date:yyyy-MM}/chronos-%d{MM-dd-yyyy}-%i.log.gz - log/chronos-err.log - log/$${date:yyyy-MM}/chronos-err-%d{MM-dd-yyyy}-%i.log.gz + log/$${date:yyyy-MM}/mediacube-%d{MM-dd-yyyy}-%i.log.gz + log/mediacube-err.log + log/$${date:yyyy-MM}/mediacube-err-%d{MM-dd-yyyy}-%i.log.gz + + + + + + - - + + + + + + + @@ -39,7 +50,8 @@ - + + diff --git a/server/-configuration/scheduledjobs.json b/server/-configuration/scheduledjobs.json index 86fb8243..28603a41 100644 --- a/server/-configuration/scheduledjobs.json +++ b/server/-configuration/scheduledjobs.json @@ -6,7 +6,8 @@ "executeimmediate": true, "cronexpression": "0 0 0/1 1/1 * ? *", "parameters": [ - {"name": "inputFolder", "value": "/mnt/ISILON/ARCHIVE", "type": "java.lang.String"} + {"name": "inputFolder", "value": "/mnt/ISILON/ARCHIVE", "type": "java.lang.String"}, + {"name": "globalInputFolder", "value": "\\\\10.10.1.100\\BRAAVOS\\ARCHIVE", "type": "java.lang.String"} ] }, { diff --git a/server/-configuration/start-mediacube.sh b/server/-configuration/start-mediacube.sh new file mode 100644 index 00000000..dbad0af3 --- /dev/null +++ b/server/-configuration/start-mediacube.sh @@ -0,0 +1,11 @@ +export DSMI_CONFIG=/opt/mediacube/configuration/dsm.opt +export DSMI_DIR=/opt/tivoli/tsm/client/api/bin64 +export DSMI_LOG=/opt/mediacube/log +export DSM_DIR=/opt/tivoli/tsm/client/api/bin64 +# export LANG=hu_HU.ISO8859-2 +export LD_LIBRARY_PATH=/opt/tivoli/tsm/client/ba/bin:/opt/tivoli/tsm/client/api/bin64 +export LIBPATH=/opt/tivoli/tsm/client/ba/bin:/opt/tivoli/tsm/client/api/bin64 +export SHLIB_PATH=/opt/tivoli/tsm/client/ba/bin:/opt/tivoli/tsm/client/api/bin64 +export TZ=NFT-1DFT,M3.5.0,M10.5.0 +export PATH=/usr/lib/jvm/java-1.8.0-openjdk-1.8.0.102-4.b14.el7.x86_64/bin:$PATH +./mediacube \ No newline at end of file diff --git a/server/-product/DEPLOY-README.txt b/server/-product/DEPLOY-README.txt index 046da097..2d12684e 100644 --- a/server/-product/DEPLOY-README.txt +++ b/server/-product/DEPLOY-README.txt @@ -1,3 +1,9 @@ +Install +--------------- +yum install nginx +yum install samba + + Services global --------------- @@ -31,8 +37,9 @@ Add Samba user Serving \\10.10.1.26\mclowres\ -Windows mapping +Mapping net use m: \\10.10.1.26\mclowres /user:mclowres Broadca5T /persistent:no + mount -t cifs -o username=mediacube,password=Broadca5T //10.10.1.100/BRAAVOS /mnt/ISILON diff --git a/server/-product/pom.xml b/server/-product/pom.xml index bdb789e7..df1e9a93 100644 --- a/server/-product/pom.xml +++ b/server/-product/pom.xml @@ -60,6 +60,7 @@ log4j2.xml scheduledjobs.json + dsm.opt @@ -79,6 +80,7 @@ log4j2.xml scheduledjobs.json + dsm.opt @@ -204,6 +206,43 @@ + + copy-resources-6 + install + + copy-resources + + + ${project.build.directory}/products/MediaCube/win32/win32/x86_64 + + + ${basedir}/../-configuration + + start-mediacube.sh + + + + + + + copy-resources-6-lnx + install + + copy-resources + + + ${project.build.directory}/products/MediaCube/linux/gtk/x86_64 + + + ${basedir}/../-configuration + + start-mediacube.sh + + + + + + @@ -227,6 +266,7 @@ "cd ${remote.location}" "synchronize remote" "call chmod +x mediacube" + "call chmod +x start-mediacube.sh" "exit" diff --git a/server/user.jobengine.executors/jobtemplates/archive-item.xml b/server/user.jobengine.executors/jobtemplates/archive-item.xml index 70d22926..bc716cea 100644 --- a/server/user.jobengine.executors/jobtemplates/archive-item.xml +++ b/server/user.jobengine.executors/jobtemplates/archive-item.xml @@ -3,6 +3,7 @@ + @@ -33,6 +34,12 @@ + + + + + + diff --git a/server/user.jobengine.executors/jobtemplates/archive-ondemand.xml b/server/user.jobengine.executors/jobtemplates/archive-ondemand.xml index 4265ba2a..c06878ee 100644 --- a/server/user.jobengine.executors/jobtemplates/archive-ondemand.xml +++ b/server/user.jobengine.executors/jobtemplates/archive-ondemand.xml @@ -2,8 +2,8 @@ - + @@ -25,6 +25,9 @@ + + + diff --git a/server/user.jobengine.executors/src/user/jobengine/server/steps/ArchiveItemSubmitStep.java b/server/user.jobengine.executors/src/user/jobengine/server/steps/ArchiveItemSubmitStep.java index da19d1ef..afb60e80 100644 --- a/server/user.jobengine.executors/src/user/jobengine/server/steps/ArchiveItemSubmitStep.java +++ b/server/user.jobengine.executors/src/user/jobengine/server/steps/ArchiveItemSubmitStep.java @@ -17,17 +17,19 @@ import user.jobengine.server.IJobRuntime; * @author robi */ public class ArchiveItemSubmitStep extends JobStep { + private static final String GLOBAL_INPUT_FOLDER = "globalInputFolder"; private static final Logger logger = LogManager.getLogger(); private static final String JOBTEMPLATE = "archive-item.xml"; private static final String ARCHIVE_ITEM = "archiveItem"; @StepEntry - public Object[] execute(ArrayList archiveList, IJobEngine jobEngine, IJobRuntime jobRuntime) { + public Object[] execute(String globalInputFolder, ArrayList archiveList, IJobEngine jobEngine, IJobRuntime jobRuntime) { try { for (ArchiveItem archiveItem : archiveList) { String title = "Archiválás: " + archiveItem.getMediaFile().toFile().getName(); try { - jobEngine.submit(JOBTEMPLATE, title, ListUtils.asMap(ARCHIVE_ITEM, archiveItem)); + IJobRuntime submit = jobEngine.submit(JOBTEMPLATE, title, + ListUtils.asMap(ARCHIVE_ITEM, archiveItem, GLOBAL_INPUT_FOLDER, globalInputFolder)); } catch (Exception e) { if (archiveItem != null) removeCatchedFile(archiveItem.getCatchedFile()); diff --git a/server/user.jobengine.executors/src/user/jobengine/server/steps/ArchiveListBuilderStep.java b/server/user.jobengine.executors/src/user/jobengine/server/steps/ArchiveListBuilderStep.java index 9278fe02..4df08f34 100644 --- a/server/user.jobengine.executors/src/user/jobengine/server/steps/ArchiveListBuilderStep.java +++ b/server/user.jobengine.executors/src/user/jobengine/server/steps/ArchiveListBuilderStep.java @@ -6,7 +6,6 @@ import java.nio.file.DirectoryStream; import java.nio.file.Files; import java.nio.file.Path; import java.nio.file.Paths; -import java.nio.file.StandardOpenOption; import java.util.ArrayList; import java.util.List; @@ -70,7 +69,8 @@ public class ArchiveListBuilderStep extends JobStep { private void createCatchedFile(Path catchedFilePath) { try { - Files.write(catchedFilePath, CATCHED.getBytes(UTF8), StandardOpenOption.CREATE); + Files.createFile(catchedFilePath); + //Files.write(catchedFilePath, CATCHED.getBytes(UTF8), StandardOpenOption.CREATE); } catch (Exception e) { logger.catching(e); } diff --git a/server/user.jobengine.executors/src/user/jobengine/server/steps/FakeStep.java b/server/user.jobengine.executors/src/user/jobengine/server/steps/FakeStep.java index 62aff45c..39ab264a 100644 --- a/server/user.jobengine.executors/src/user/jobengine/server/steps/FakeStep.java +++ b/server/user.jobengine.executors/src/user/jobengine/server/steps/FakeStep.java @@ -2,6 +2,7 @@ package user.jobengine.server.steps; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; +import org.apache.logging.log4j.MarkerManager; import com.ibm.nosql.json.JSONUtil; import com.ibm.nosql.json.api.BasicDBObject; @@ -16,7 +17,14 @@ public class FakeStep extends JobStep { @StepEntry public Object[] execute(long itemID, IJobEngine jobEngine, IJobRuntime jobRuntime) { try { - logger.info(IJobStep.INFO_MARKER, "Starting Fake step, parameter: {}", itemID); + // IItemManager manager = jobEngine.getItemManager(); + // StoreUri sourceUri = manager.createStoreUri(RemoteStoreProtocol.SMB, "\\\\10.10.1.100\\BRAAVOS\\ARCHIVE"); + // sourceUri.setUserName("mediacube"); + // sourceUri.setPassword("Broadca5T"); + // List remoteFiles = sourceUri.getRemoteFiles(); + // for (RemoteFile remoteFile : remoteFiles) + // logger.info(remoteFile.getName()); + String json = "{ houseID: '111222' }"; BasicDBObject o = (BasicDBObject) JSONUtil.jsonToDbObject(json); for (int i = 0; i < count; i++) { @@ -26,6 +34,7 @@ public class FakeStep extends JobStep { jobRuntime.incrementProgress((i + 1) * count); // logger.info("Progress {}", jobRuntime.getProgress()); } + logger.warn(MarkerManager.getMarker("teszt2"), "Starting Fake step, parameter: {}", itemID); } catch (Exception e) { logger.error(e.getMessage()); } diff --git a/server/user.jobengine.executors/src/user/jobengine/server/steps/SelenioLowresTranscodeStep.java b/server/user.jobengine.executors/src/user/jobengine/server/steps/SelenioLowresTranscodeStep.java index 49b3eb8e..a4835318 100644 --- a/server/user.jobengine.executors/src/user/jobengine/server/steps/SelenioLowresTranscodeStep.java +++ b/server/user.jobengine.executors/src/user/jobengine/server/steps/SelenioLowresTranscodeStep.java @@ -2,6 +2,8 @@ package user.jobengine.server.steps; import java.io.File; import java.net.URL; +import java.nio.file.Path; +import java.nio.file.Paths; import java.util.Arrays; import java.util.List; @@ -46,7 +48,6 @@ public class SelenioLowresTranscodeStep extends JobStep { private IItemManager manager; private Store store; private FileType fileType; - private String outputPath; public SelenioLowresTranscodeStep() { try { @@ -65,7 +66,7 @@ public class SelenioLowresTranscodeStep extends JobStep { } } - private TranscodeRequest buildTranscodeRequest(String projectFilePath, String sourceFilePath) throws java.lang.Exception { + private TranscodeRequest buildTranscodeRequest(String projectFilePath, String sourceFilePath, String outputPath) throws java.lang.Exception { Clip clip = new Clip(); MediaFile mediaFile = new MediaFile(); mediaFile.setFile(sourceFilePath); @@ -97,29 +98,34 @@ public class SelenioLowresTranscodeStep extends JobStep { } @StepEntry - public Object[] execute(ArchiveItem archiveItem, Media mediaCubeMedia, IJobEngine jobEngine, IJobRuntime jobRuntime) throws Exception { + public Object[] execute(String globalInputFolder, ArchiveItem archiveItem, Media mediaCubeMedia, IJobEngine jobEngine, IJobRuntime jobRuntime) + throws Exception { + if (StringUtils.isBlank(globalInputFolder)) + throw new NullPointerException("System is not configured properly, 'globalInputFolder' parameter missing."); manager = jobEngine.getItemManager(); + store = manager.getSystemStore(true); if (store == null) throw new NullPointerException("System is not configured properly, low-res system store definition missing."); - StoreUri sourceStoreUri = store.getSourceStoreUri(RemoteStoreProtocol.SMB); - if (sourceStoreUri == null) - throw new NullPointerException("System is not configured properly, low-res system store SMB store URI definition missing."); - outputPath = sourceStoreUri.getRootPath(); - fileType = manager.getFileType(LOWRES_FILETYPE); if (fileType == null) throw new NullPointerException("System is not configured properly, low-res file type definition missing."); - String sourceFile = archiveItem.getMediaFile().toString(); - TranscodeRequest transcodeRequest = buildTranscodeRequest(projectFilePath, sourceFile); + StoreUri sourceStoreUri = store.getSourceStoreUri(RemoteStoreProtocol.SMB); + if (sourceStoreUri == null) + throw new NullPointerException("System is not configured properly, low-res system store SMB store URI definition missing."); + + Path inputPath = Paths.get(globalInputFolder, archiveItem.getMediaFile().toFile().getName()); + String sourceFilePath = inputPath.toString(); + String outputPath = sourceStoreUri.getRootPath(); + TranscodeRequest transcodeRequest = buildTranscodeRequest(projectFilePath, sourceFilePath, outputPath); TranscodeTask transcodeTask = transcoder.submitTranscodeTask(transcodeRequest); if (transcodeTask == null) - throw new NullPointerException("Unable to submit transcode task, server response is empty for transcode input: " + sourceFile); + throw new NullPointerException("Unable to submit transcode task, server response is empty for transcode input: " + sourceFilePath); - transcodeTask = monitor(jobRuntime, sourceFile, transcodeTask); + transcodeTask = monitor(jobRuntime, sourceFilePath, transcodeTask); jobRuntime.incrementProgress(100); diff --git a/server/user.jobengine.osgi.commons/src/user/commons/IRemoteFileListCallback.java b/server/user.jobengine.osgi.commons/src/user/commons/IRemoteFileListCallback.java new file mode 100644 index 00000000..854f0e1b --- /dev/null +++ b/server/user.jobengine.osgi.commons/src/user/commons/IRemoteFileListCallback.java @@ -0,0 +1,5 @@ +package user.commons; + +public interface IRemoteFileListCallback { + boolean onRemoteFile(RemoteFile remoteFile); +} diff --git a/server/user.jobengine.osgi.commons/src/user/commons/StoreUri.java b/server/user.jobengine.osgi.commons/src/user/commons/StoreUri.java index d6870457..5f6d93cd 100644 --- a/server/user.jobengine.osgi.commons/src/user/commons/StoreUri.java +++ b/server/user.jobengine.osgi.commons/src/user/commons/StoreUri.java @@ -64,8 +64,7 @@ public class StoreUri extends EntityBase implements Serializable { this.statusListenerList = new EventListenerList(); } - public RemoteFile copy(IDirectoryLister inputLister, IDirectoryLister outputLister, RemoteFile source, - String targetName) throws Exception { + public RemoteFile copy(IDirectoryLister inputLister, IDirectoryLister outputLister, RemoteFile source, String targetName) throws Exception { RemoteFile result = null; long uploadedBytes = 0; int byteCount = 0; @@ -153,8 +152,8 @@ public class StoreUri extends EntityBase implements Serializable { return currentPath; } - public String getFilter() { - return filter; + public DeliveryMethod getDelivery() { + return delivery; } // public String getFullPath() { @@ -172,6 +171,10 @@ public class StoreUri extends EntityBase implements Serializable { // return result.replace("\\", "/"); // } + public String getFilter() { + return filter; + } + public String getPassword() { return password; } @@ -180,6 +183,23 @@ public class StoreUri extends EntityBase implements Serializable { return portNumber; } + public RemoteStoreProtocol getProtocol() { + return protocol; + } + + public RemoteFile getRemoteFile(String fileName) { + RemoteFile result = null; + try { + IDirectoryLister inputLister = RemoteFileHandler.createLister(this); + if (inputLister == null) + throw new Exception("InputLister is null."); + result = inputLister.get(fileName); + } catch (Exception e) { + logger.error(e); + } + return result; + } + public List getRemoteFiles() throws Exception { List result = null; IDirectoryLister lister = RemoteFileHandler.createLister(this); @@ -188,6 +208,13 @@ public class StoreUri extends EntityBase implements Serializable { return result; } + public void getRemoteFiles(IRemoteFileListCallback callback) throws Exception { + List result = null; + IDirectoryLister lister = RemoteFileHandler.createLister(this); + if (lister != null) + lister.list(callback); + } + public String getRootPath() { return rootPath; } @@ -228,6 +255,10 @@ public class StoreUri extends EntityBase implements Serializable { this.currentPath = currentPath; } + public void setDelivery(DeliveryMethod delivery) { + this.delivery = delivery; + } + public void setFilter(String filter) { this.filter = filter; } @@ -240,6 +271,10 @@ public class StoreUri extends EntityBase implements Serializable { this.portNumber = portNumber; } + public void setProtocol(RemoteStoreProtocol protocol) { + this.protocol = protocol; + } + public void setRootPath(String rootPath) { this.rootPath = rootPath; } @@ -268,19 +303,24 @@ public class StoreUri extends EntityBase implements Serializable { this.userName = userName; } - public RemoteFile getRemoteFile(String fileName) { - RemoteFile result = null; + @Override + public String toString() { + String result = null; try { - IDirectoryLister inputLister = RemoteFileHandler.createLister(this); - if (inputLister == null) - throw new Exception("InputLister is null."); - result = inputLister.get(fileName); + result = toString(false); } catch (Exception e) { - logger.error(e); + e.printStackTrace(); } return result; } + public String toString(boolean skipProtocol) throws Exception { + String result = skipProtocol ? uri : protocol.getValue() + uri; + result = DirectoryUtils.combine(result, rootPath); + result = DirectoryUtils.combine(result, currentPath); + return result; + } + public RemoteFile transferFrom(StoreUri targetStoreUri, RemoteFile remoteFile, String targetName) throws Exception { IDirectoryLister inputLister = RemoteFileHandler.createLister(this); if (inputLister == null) @@ -309,38 +349,4 @@ public class StoreUri extends EntityBase implements Serializable { return copy(inputLister, outputLister, remoteFile, targetName); } - - @Override - public String toString() { - String result = null; - try { - result = toString(false); - } catch (Exception e) { - e.printStackTrace(); - } - return result; - } - - public String toString(boolean skipProtocol) throws Exception { - String result = skipProtocol ? uri : protocol.getValue() + uri; - result = DirectoryUtils.combine(result, rootPath); - result = DirectoryUtils.combine(result, currentPath); - return result; - } - - public RemoteStoreProtocol getProtocol() { - return protocol; - } - - public void setProtocol(RemoteStoreProtocol protocol) { - this.protocol = protocol; - } - - public DeliveryMethod getDelivery() { - return delivery; - } - - public void setDelivery(DeliveryMethod delivery) { - this.delivery = delivery; - } } \ No newline at end of file diff --git a/server/user.jobengine.osgi.commons/src/user/commons/remotestore/FtpDirectoryLister.java b/server/user.jobengine.osgi.commons/src/user/commons/remotestore/FtpDirectoryLister.java index 00407cbe..6fa2d189 100644 --- a/server/user.jobengine.osgi.commons/src/user/commons/remotestore/FtpDirectoryLister.java +++ b/server/user.jobengine.osgi.commons/src/user/commons/remotestore/FtpDirectoryLister.java @@ -6,11 +6,13 @@ import java.sql.Timestamp; import java.util.ArrayList; import java.util.List; +import org.apache.commons.lang.NotImplementedException; import org.apache.commons.lang.StringUtils; import org.apache.commons.net.ftp.FTP; import org.apache.commons.net.ftp.FTPClient; import org.apache.commons.net.ftp.FTPFile; +import user.commons.IRemoteFileListCallback; import user.commons.RemoteFile; import user.commons.StoreUri; @@ -23,6 +25,7 @@ public class FtpDirectoryLister implements IDirectoryLister { this.storeUri = storeUri; } + @Override public boolean checkAvailability(RemoteFile remoteFile) throws Exception { boolean result = false; InputStream input = null; @@ -47,6 +50,7 @@ public class FtpDirectoryLister implements IDirectoryLister { return result; } + @Override public void cleanUp() throws Exception { if (ftpClient != null) { if (ftpClient.isConnected()) { @@ -108,6 +112,7 @@ public class FtpDirectoryLister implements IDirectoryLister { return result; } + @Override public InputStream getInputStream(RemoteFile remoteFile) throws Exception { InputStream input = null; connect(); @@ -118,6 +123,7 @@ public class FtpDirectoryLister implements IDirectoryLister { return input; } + @Override public OutputStream getOutputStream(RemoteFile remoteFile) throws Exception { OutputStream output = null; connect(); @@ -149,6 +155,11 @@ public class FtpDirectoryLister implements IDirectoryLister { return result; } + @Override + public void list(IRemoteFileListCallback callback) throws Exception { + throw new NotImplementedException(); + } + private RemoteFile toRemoteFile(FTPFile file) { RemoteFile result = new RemoteFile(); result.setName(file.getName()); diff --git a/server/user.jobengine.osgi.commons/src/user/commons/remotestore/IDirectoryLister.java b/server/user.jobengine.osgi.commons/src/user/commons/remotestore/IDirectoryLister.java index 49310841..02f716ec 100644 --- a/server/user.jobengine.osgi.commons/src/user/commons/remotestore/IDirectoryLister.java +++ b/server/user.jobengine.osgi.commons/src/user/commons/remotestore/IDirectoryLister.java @@ -4,15 +4,23 @@ import java.io.InputStream; import java.io.OutputStream; import java.util.List; +import user.commons.IRemoteFileListCallback; import user.commons.RemoteFile; public interface IDirectoryLister { - List list() throws Exception; - InputStream getInputStream(RemoteFile remoteFile) throws Exception; - OutputStream getOutputStream(RemoteFile remoteFile) throws Exception; boolean checkAvailability(RemoteFile remoteFile) throws Exception; + void cleanUp() throws Exception; + RemoteFile get(String fileName) throws Exception; - + + InputStream getInputStream(RemoteFile remoteFile) throws Exception; + + OutputStream getOutputStream(RemoteFile remoteFile) throws Exception; + + List list() throws Exception; + + void list(IRemoteFileListCallback callback) throws Exception; + } diff --git a/server/user.jobengine.osgi.commons/src/user/commons/remotestore/LocalDirectoryLister.java b/server/user.jobengine.osgi.commons/src/user/commons/remotestore/LocalDirectoryLister.java index 9f872654..3e2ff7b5 100644 --- a/server/user.jobengine.osgi.commons/src/user/commons/remotestore/LocalDirectoryLister.java +++ b/server/user.jobengine.osgi.commons/src/user/commons/remotestore/LocalDirectoryLister.java @@ -11,6 +11,9 @@ import java.util.Arrays; import java.util.Comparator; import java.util.List; +import org.apache.commons.lang.NotImplementedException; + +import user.commons.IRemoteFileListCallback; import user.commons.RemoteFile; import user.commons.StoreUri; @@ -42,12 +45,14 @@ public class LocalDirectoryLister implements IDirectoryLister { return toRemoteFile(file); } + @Override public InputStream getInputStream(RemoteFile remoteFile) throws Exception { String path = storeUri.toString(true); File file = new File(path + remoteFile.getName()); return new FileInputStream(file); } + @Override public OutputStream getOutputStream(RemoteFile remoteFile) throws Exception { String path = storeUri.toString(true); File file = new File(path + remoteFile.getName()); @@ -59,18 +64,18 @@ public class LocalDirectoryLister implements IDirectoryLister { List result = new ArrayList(); String path = storeUri.toString(true); File dir = new File(path); - File[] dirs = dir.listFiles(new DirectoryFileFilter(storeUri - .getFilter(), true)); - File[] files = dir.listFiles(new DirectoryFileFilter(storeUri - .getFilter(), false)); + File[] dirs = dir.listFiles(new DirectoryFileFilter(storeUri.getFilter(), true)); + File[] files = dir.listFiles(new DirectoryFileFilter(storeUri.getFilter(), false)); Arrays.sort(dirs, new Comparator() { + @Override public int compare(File f1, File f2) { return f1.getName().compareToIgnoreCase(f2.getName()); } }); Arrays.sort(files, new Comparator() { + @Override public int compare(File f1, File f2) { return f1.getName().compareToIgnoreCase(f2.getName()); } @@ -88,6 +93,11 @@ public class LocalDirectoryLister implements IDirectoryLister { return result; } + @Override + public void list(IRemoteFileListCallback callback) throws Exception { + throw new NotImplementedException(); + } + private RemoteFile toRemoteFile(File file) { RemoteFile result = new RemoteFile(); result.setName(file.getName()); diff --git a/server/user.jobengine.osgi.commons/src/user/commons/remotestore/SambaDirectoryLister.java b/server/user.jobengine.osgi.commons/src/user/commons/remotestore/SambaDirectoryLister.java index 7ac55e1f..b594c197 100644 --- a/server/user.jobengine.osgi.commons/src/user/commons/remotestore/SambaDirectoryLister.java +++ b/server/user.jobengine.osgi.commons/src/user/commons/remotestore/SambaDirectoryLister.java @@ -8,6 +8,7 @@ import java.util.List; import jcifs.smb.NtlmPasswordAuthentication; import jcifs.smb.SmbFile; +import user.commons.IRemoteFileListCallback; import user.commons.RemoteFile; import user.commons.StoreUri; @@ -21,8 +22,7 @@ public class SambaDirectoryLister implements IDirectoryLister { @Override public boolean checkAvailability(RemoteFile remoteFile) throws Exception { boolean result = false; - NtlmPasswordAuthentication auth = new NtlmPasswordAuthentication(null, - storeUri.getUserName(), storeUri.getPassword()); + NtlmPasswordAuthentication auth = new NtlmPasswordAuthentication(null, storeUri.getUserName(), storeUri.getPassword()); long size = remoteFile.getSize(); String path = storeUri.toString(true) + remoteFile.getName(); SmbFile smbFile = new SmbFile(path, auth); @@ -38,8 +38,7 @@ public class SambaDirectoryLister implements IDirectoryLister { @Override public RemoteFile get(String fileName) throws Exception { RemoteFile result = null; - NtlmPasswordAuthentication auth = new NtlmPasswordAuthentication(null, - storeUri.getUserName(), storeUri.getPassword()); + NtlmPasswordAuthentication auth = new NtlmPasswordAuthentication(null, storeUri.getUserName(), storeUri.getPassword()); String path = storeUri.toString(true) + fileName; SmbFile smbFile = new SmbFile(path, auth); if (smbFile != null) @@ -47,17 +46,17 @@ public class SambaDirectoryLister implements IDirectoryLister { return result; } + @Override public InputStream getInputStream(RemoteFile remoteFile) throws Exception { - NtlmPasswordAuthentication auth = new NtlmPasswordAuthentication(null, - storeUri.getUserName(), storeUri.getPassword()); + NtlmPasswordAuthentication auth = new NtlmPasswordAuthentication(null, storeUri.getUserName(), storeUri.getPassword()); String path = storeUri.toString(true) + remoteFile.getName(); SmbFile smbFile = new SmbFile(path, auth); return smbFile.getInputStream(); } + @Override public OutputStream getOutputStream(RemoteFile remoteFile) throws Exception { - NtlmPasswordAuthentication auth = new NtlmPasswordAuthentication(null, - storeUri.getUserName(), storeUri.getPassword()); + NtlmPasswordAuthentication auth = new NtlmPasswordAuthentication(null, storeUri.getUserName(), storeUri.getPassword()); String path = storeUri.toString(true) + remoteFile.getName(); SmbFile smbFile = new SmbFile(path, auth); return smbFile.getOutputStream(); @@ -66,9 +65,8 @@ public class SambaDirectoryLister implements IDirectoryLister { @Override public List list() throws Exception { List result = new ArrayList(); - NtlmPasswordAuthentication auth = new NtlmPasswordAuthentication(null, - storeUri.getUserName(), storeUri.getPassword()); - String path = storeUri.toString(true); + NtlmPasswordAuthentication auth = new NtlmPasswordAuthentication(null, storeUri.getUserName(), storeUri.getPassword()); + String path = "smb:" + storeUri.toString(true).replace("\\", "/"); SmbFile smbFile = new SmbFile(path, auth); SmbFile[] list = smbFile.listFiles(); for (SmbFile smb : list) { @@ -78,6 +76,21 @@ public class SambaDirectoryLister implements IDirectoryLister { return result; } + @Override + public void list(IRemoteFileListCallback callback) throws Exception { + NtlmPasswordAuthentication auth = new NtlmPasswordAuthentication(null, storeUri.getUserName(), storeUri.getPassword()); + String path = "smb:" + storeUri.toString(true).replace("\\", "/"); + SmbFile smbFile = new SmbFile(path, auth); + SmbFile[] list = smbFile.listFiles(); + for (SmbFile smb : list) { + RemoteFile file = toRemoteFile(smb); + if (callback != null) { + if (!callback.onRemoteFile(file)) + break; + } + } + } + private RemoteFile toRemoteFile(SmbFile file) throws Exception { RemoteFile result = new RemoteFile(); result.setName(file.getName()); diff --git a/server/user.jobengine.osgi.commons/src/user/commons/remotestore/TSMLister.java b/server/user.jobengine.osgi.commons/src/user/commons/remotestore/TSMLister.java index 3af0cee1..6b7a9b0e 100644 --- a/server/user.jobengine.osgi.commons/src/user/commons/remotestore/TSMLister.java +++ b/server/user.jobengine.osgi.commons/src/user/commons/remotestore/TSMLister.java @@ -5,9 +5,11 @@ import java.io.InputStream; import java.io.OutputStream; import java.util.List; +import org.apache.commons.lang.NotImplementedException; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; +import user.commons.IRemoteFileListCallback; import user.commons.RemoteFile; import user.commons.StoreUri; import user.tsm.client.TSMBackupFileObject; @@ -15,10 +17,10 @@ import user.tsm.client.TSMBufferedClient; public class TSMLister implements IDirectoryLister { - private TSMBufferedClient client; private static final String FSNAME = File.separator + "JOBENGINE"; private static final String HLNAME = File.separator + "JOBENGINE"; private static final Logger logger = LogManager.getLogger(); + private TSMBufferedClient client; public TSMLister(StoreUri storeUri) throws Exception { client = new TSMBufferedClient(); @@ -28,8 +30,27 @@ public class TSMLister implements IDirectoryLister { } @Override - public List list() throws Exception { - return null; + public boolean checkAvailability(RemoteFile remoteFile) throws Exception { + // TODO Auto-generated method stub + return false; + } + + @Override + public void cleanUp() throws Exception { + client.disconnect(); + } + + @Override + public RemoteFile get(String fileName) throws Exception { + RemoteFile result = null; + TSMBackupFileObject backupFileObject = client.getActiveBackupFileObject(FSNAME, HLNAME, File.separator + fileName); + logger.info("TSMLister get"); + if (backupFileObject != null) { + result = toRemoteFile(backupFileObject); + logger.info("TSMLister backupFileObject-et elk�rte a k�vetkez� n�vvel: " + backupFileObject.getLowLevelName()); + } else + cleanUp(); + return result; } @Override @@ -54,29 +75,13 @@ public class TSMLister implements IDirectoryLister { } @Override - public boolean checkAvailability(RemoteFile remoteFile) throws Exception { - // TODO Auto-generated method stub - return false; - } - - @Override - public void cleanUp() throws Exception { - client.disconnect(); + public List list() throws Exception { + return null; } @Override - public RemoteFile get(String fileName) throws Exception { - RemoteFile result = null; - TSMBackupFileObject backupFileObject = client.getActiveBackupFileObject(FSNAME, HLNAME, - File.separator + fileName); - logger.info("TSMLister get"); - if (backupFileObject != null) { - result = toRemoteFile(backupFileObject); - logger.info( - "TSMLister backupFileObject-et elk�rte a k�vetkez� n�vvel: " + backupFileObject.getLowLevelName()); - } else - cleanUp(); - return result; + public void list(IRemoteFileListCallback callback) throws Exception { + throw new NotImplementedException(); } private RemoteFile toRemoteFile(TSMBackupFileObject file) { diff --git a/server/user.jobengine.osgi.db/src/user/jobengine/db/MemoryCache.java b/server/user.jobengine.osgi.db/src/user/jobengine/db/MemoryCache.java index d76952e4..3b335fe8 100644 --- a/server/user.jobengine.osgi.db/src/user/jobengine/db/MemoryCache.java +++ b/server/user.jobengine.osgi.db/src/user/jobengine/db/MemoryCache.java @@ -11,15 +11,17 @@ import org.apache.logging.log4j.Logger; import user.commons.IEntityBase; class MemoryCache { + private static final Logger logger = LogManager.getLogger(); boolean enabled; Map, Map>> cacheMap = new ConcurrentHashMap, Map>>(); Object lock = new Object(); private IItemManager manager; - private static final Logger logger = LogManager.getLogger(); - public void setItemManager(IItemManager manager) { - this.manager = manager; - enableCache(); + void clearAllCached() { + manager.traceIn(); + for (Class baseClass : cacheMap.keySet()) + clearCached(baseClass); + manager.traceOut(); } void clearCached(Class baseClass) { @@ -30,13 +32,6 @@ class MemoryCache { manager.traceOut(); } - void clearAllCached() { - manager.traceIn(); - for (Class baseClass : cacheMap.keySet()) - clearCached(baseClass); - manager.traceOut(); - } - public void disableCache() { enabled = false; clearAllCached(); @@ -46,34 +41,28 @@ class MemoryCache { public void enableCache() { enabled = true; logger.info("Cache is enabled"); + // logger.info(MarkerManager.getMarker("teszt1"), "Starting1"); + // logger.warn(MarkerManager.getMarker("teszt2"), "Starting2"); + // logger.warn(MarkerManager.getMarker("teszt3"), "Starting3"); + // logger.error(MarkerManager.getMarker("teszt4"), "Starting4"); } - Map> getCache(Class baseClass) { - Map> cache = null; - if (enabled) { - if (cacheMap.containsKey(baseClass)) { - cache = cacheMap.get(baseClass); - } else { - cache = new ConcurrentHashMap>(); - cacheMap.put(baseClass, cache); - } - } - return cache; - } - - boolean setAllCached(Class baseClass, List entities) { + void garbageCached(Class baseClass, Object key) { manager.traceIn(); - boolean result = true; - if (entities != null) { - logger.debug(String.format("Setting all %s, count %d", baseClass, entities.size())); - for (IEntityBase entity : entities) { - result = storeCached(entity.getId(), entity); - if (!result) - break; + Map> cache = getCache(baseClass); + if (cache != null) { + try { + if (cache.containsKey(key)) { + SoftReference r = cache.get(key); + if (r != null) + r.clear(); + } + + } catch (Exception ex) { + logger.error(ex.getMessage()); } } manager.traceOut(); - return result; } public List getAllCached(Class baseClass) { @@ -103,23 +92,17 @@ class MemoryCache { return result; } - public boolean storeCached(Object key, IEntityBase entity) { - manager.traceIn(); - if (entity == null) - manager.throwError(new Exception("Can not store null value into cache")); - Map> cache = getCache(entity.getClass()); - boolean result = false; - if (cache != null) { - try { - cache.put(key, new SoftReference(entity)); - logger.debug("Store " + entity.getClass() + ", key: " + key); - result = true; - } catch (Exception ex) { - logger.error(ex.getMessage()); + Map> getCache(Class baseClass) { + Map> cache = null; + if (enabled) { + if (cacheMap.containsKey(baseClass)) { + cache = cacheMap.get(baseClass); + } else { + cache = new ConcurrentHashMap>(); + cacheMap.put(baseClass, cache); } } - manager.traceOut(); - return result; + return cache; } public void removeCached(Class baseClass, Object key) { @@ -165,21 +148,42 @@ class MemoryCache { return result; } - void garbageCached(Class baseClass, Object key) { + boolean setAllCached(Class baseClass, List entities) { manager.traceIn(); - Map> cache = getCache(baseClass); + boolean result = true; + if (entities != null) { + logger.debug(String.format("Setting all %s, count %d", baseClass, entities.size())); + for (IEntityBase entity : entities) { + result = storeCached(entity.getId(), entity); + if (!result) + break; + } + } + manager.traceOut(); + return result; + } + + public void setItemManager(IItemManager manager) { + this.manager = manager; + enableCache(); + } + + public boolean storeCached(Object key, IEntityBase entity) { + manager.traceIn(); + if (entity == null) + manager.throwError(new Exception("Can not store null value into cache")); + Map> cache = getCache(entity.getClass()); + boolean result = false; if (cache != null) { try { - if (cache.containsKey(key)) { - SoftReference r = cache.get(key); - if (r != null) - r.clear(); - } - + cache.put(key, new SoftReference(entity)); + logger.debug("Store " + entity.getClass() + ", key: " + key); + result = true; } catch (Exception ex) { logger.error(ex.getMessage()); } } manager.traceOut(); + return result; } } diff --git a/server/user.jobengine.osgi.server/pages/search_items.zul b/server/user.jobengine.osgi.server/pages/search_items.zul index d89a3c38..f50c30ba 100644 --- a/server/user.jobengine.osgi.server/pages/search_items.zul +++ b/server/user.jobengine.osgi.server/pages/search_items.zul @@ -16,7 +16,7 @@
- +