From 93d124d921b88fb2c6342ba95ff5a23f9bbf3015 Mon Sep 17 00:00:00 2001 From: =?utf8?q?V=C3=A1s=C3=A1ry=20D=C3=A1niel?= Date: Mon, 29 Jan 2024 20:59:19 +0100 Subject: [PATCH] Project config changes --- server/-product/production/HIRTV/.gitignore | 1 + .../production/MEDIAVIVANTIS/.gitignore | 1 + server/hu.user.theme.userdark/.gitignore | 0 .../.settings/org.eclipse.jdt.core.prefs | 2 + .../.settings/org.eclipse.m2e.core.prefs | 4 + .../org.eclipse.core.resources.prefs | 2 + .../.settings/org.eclipse.m2e.core.prefs | 4 + .../steps/TranscodeMediaSamuraiStep.java | 157 ++++++++++ server/user.jobengine.log4j/.gitignore | 1 + .../build-commons-module.launch | 20 ++ .../src/user/commons/MediaSamuraiAPI.java | 284 ++++++++++++++++++ .../org.eclipse.core.resources.prefs | 12 + 12 files changed, 488 insertions(+) create mode 100644 server/-product/production/HIRTV/.gitignore create mode 100644 server/-product/production/MEDIAVIVANTIS/.gitignore create mode 100644 server/hu.user.theme.userdark/.gitignore create mode 100644 server/hu.user.theme.userdark/.settings/org.eclipse.jdt.core.prefs create mode 100644 server/hu.user.theme.userdark/.settings/org.eclipse.m2e.core.prefs create mode 100644 server/user.commons.zk/.settings/org.eclipse.core.resources.prefs create mode 100644 server/user.commons.zk/.settings/org.eclipse.m2e.core.prefs create mode 100644 server/user.jobengine.executors/src/user/jobengine/server/steps/TranscodeMediaSamuraiStep.java create mode 100644 server/user.jobengine.log4j/.gitignore create mode 100644 server/user.jobengine.osgi.commons/build-commons-module.launch create mode 100644 server/user.jobengine.osgi.commons/src/user/commons/MediaSamuraiAPI.java create mode 100644 server/user.mediacube.gui/.settings/org.eclipse.core.resources.prefs diff --git a/server/-product/production/HIRTV/.gitignore b/server/-product/production/HIRTV/.gitignore new file mode 100644 index 00000000..82520ca1 --- /dev/null +++ b/server/-product/production/HIRTV/.gitignore @@ -0,0 +1 @@ +/tmp/ diff --git a/server/-product/production/MEDIAVIVANTIS/.gitignore b/server/-product/production/MEDIAVIVANTIS/.gitignore new file mode 100644 index 00000000..82520ca1 --- /dev/null +++ b/server/-product/production/MEDIAVIVANTIS/.gitignore @@ -0,0 +1 @@ +/tmp/ diff --git a/server/hu.user.theme.userdark/.gitignore b/server/hu.user.theme.userdark/.gitignore new file mode 100644 index 00000000..e69de29b diff --git a/server/hu.user.theme.userdark/.settings/org.eclipse.jdt.core.prefs b/server/hu.user.theme.userdark/.settings/org.eclipse.jdt.core.prefs new file mode 100644 index 00000000..4ede96d8 --- /dev/null +++ b/server/hu.user.theme.userdark/.settings/org.eclipse.jdt.core.prefs @@ -0,0 +1,2 @@ +eclipse.preferences.version=1 +org.eclipse.jdt.core.compiler.problem.forbiddenReference=warning diff --git a/server/hu.user.theme.userdark/.settings/org.eclipse.m2e.core.prefs b/server/hu.user.theme.userdark/.settings/org.eclipse.m2e.core.prefs new file mode 100644 index 00000000..f897a7f1 --- /dev/null +++ b/server/hu.user.theme.userdark/.settings/org.eclipse.m2e.core.prefs @@ -0,0 +1,4 @@ +activeProfiles= +eclipse.preferences.version=1 +resolveWorkspaceProjects=true +version=1 diff --git a/server/user.commons.zk/.settings/org.eclipse.core.resources.prefs b/server/user.commons.zk/.settings/org.eclipse.core.resources.prefs new file mode 100644 index 00000000..99f26c02 --- /dev/null +++ b/server/user.commons.zk/.settings/org.eclipse.core.resources.prefs @@ -0,0 +1,2 @@ +eclipse.preferences.version=1 +encoding/=UTF-8 diff --git a/server/user.commons.zk/.settings/org.eclipse.m2e.core.prefs b/server/user.commons.zk/.settings/org.eclipse.m2e.core.prefs new file mode 100644 index 00000000..f897a7f1 --- /dev/null +++ b/server/user.commons.zk/.settings/org.eclipse.m2e.core.prefs @@ -0,0 +1,4 @@ +activeProfiles= +eclipse.preferences.version=1 +resolveWorkspaceProjects=true +version=1 diff --git a/server/user.jobengine.executors/src/user/jobengine/server/steps/TranscodeMediaSamuraiStep.java b/server/user.jobengine.executors/src/user/jobengine/server/steps/TranscodeMediaSamuraiStep.java new file mode 100644 index 00000000..70dbeffe --- /dev/null +++ b/server/user.jobengine.executors/src/user/jobengine/server/steps/TranscodeMediaSamuraiStep.java @@ -0,0 +1,157 @@ +package user.jobengine.server.steps; + +import java.io.File; +import java.io.IOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.nio.file.Paths; + +import org.apache.commons.io.FilenameUtils; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.apache.logging.log4j.Marker; +import org.apache.logging.log4j.message.Message; +import org.apache.logging.log4j.message.ParameterizedMessage; + +import user.commons.FFAStransAPI; +import user.commons.IFFAStransAPI; +import user.commons.StoreUri; +import user.commons.mediatool.Timecode; +import user.commons.mediatool.Timecode.Type; +import user.commons.remotestore.RemoteStoreProtocol; +import user.jobengine.db.FileType; +import user.jobengine.db.IItemManager; +import user.jobengine.db.Media; +import user.jobengine.db.Store; +import user.jobengine.server.IJobEngine; +import user.jobengine.server.IJobRuntime; +import user.jobengine.server.steps.shared.EscortFiles; + +public class TranscodeMediaSamuraiStep extends JobStep { + private static final int POLL_INTERVALL = 3000; + private static final String MP4EXT = ".MP4"; + private static final String MXFEXT = ".MXF"; + private static final String LOWRES_FILETYPE = "Low-res"; + private static final Logger logger = LogManager.getLogger("TranscodeFFAStranStep"); + private IItemManager manager; + private Store store; + private FileType fileType; + private Media mediaCubeMedia; + private Marker marker; + + @StepEntry + public Object[] execute(ArchiveItem archiveItem, Media mediaCubeMedia, String transcoderAddress, + String transcoderTemplateName, String globalHiresSourcePath, String localLowresTargetPath, + boolean deleteSource, IJobEngine jobEngine, IJobRuntime jobRuntime) throws Exception { + + this.marker = jobRuntime.getSessionMarker(); + this.manager = jobEngine.getItemManager(); + this.store = check(manager.getCurrentLowresStore(), "lowres Store"); + this.fileType = check(manager.getFileType(LOWRES_FILETYPE), "lowres FileType"); + this.mediaCubeMedia = check(mediaCubeMedia, "mediaCubeMedia"); + check(archiveItem, "archiveItem"); + check(transcoderAddress, "transcoderAddress"); + check(transcoderTemplateName, "transcoderTemplateName"); + check(globalHiresSourcePath, "globalHiresSourcePath"); + check(localLowresTargetPath, "localLowresTargetPath"); + + File sourceMediaFile = new File(archiveItem.getMediaFile()); + logger.info("Transcoding {}", archiveItem.getMediaFile()); + String sourceFileName = sourceMediaFile.getName(); + Timecode timecode = new Timecode(mediaCubeMedia.getLength(), Type.PAL); + + String details = String.format("%s (%s, %d bytes)", sourceFileName, timecode.toString(), + sourceMediaFile.length()); + + StoreUri storeUri = store.getTargetStoreUri(RemoteStoreProtocol.LOCAL); + if (storeUri == null) + throw new Exception("Can not detect proxy folder."); + + String webPath = storeUri.toString(true); + + Path targetPath = null; + try { + String targetFileName = FilenameUtils.removeExtension(sourceFileName) + MP4EXT; + targetPath = Paths.get(localLowresTargetPath, targetFileName); + if (!targetPath.toFile().exists()) { + // jobRuntime.setDescription(String.format("%s: %s", + // jobRuntime.getDescription(), details)); + jobRuntime.setDescription(String.format("%s transzkódolása", details)); + String sourceFile = Paths.get(globalHiresSourcePath, sourceFileName).toString(); + IFFAStransAPI api = new FFAStransAPI(transcoderAddress, p -> { + if (p <= 100) + jobRuntime.incrementProgress(p); + }); + + api.submit(transcoderTemplateName, sourceFile); + api.monitor(POLL_INTERVALL); + } + + // a sikeres transzkod utan nem mindig van ott egybol a fajl + long started = System.currentTimeMillis(); + while (!targetPath.toFile().exists()) { + long current = System.currentTimeMillis(); + // max 5 perc varakozas + if (current - started > 5 * 60 * 1000) + throw new Exception("Transcode job target file access timed out"); + Thread.sleep(POLL_INTERVALL); + } + + postprocess(targetPath, webPath); + + } catch (Exception e) { + logger.catching(e); + Message m = new ParameterizedMessage("{} átkódolás hiba: {}", sourceFileName, e.getMessage()); + logger.error(marker, m); + throw new Exception(m.getFormattedMessage()); + } finally { + try { + if (deleteSource && sourceMediaFile != null && sourceMediaFile.exists()) + sourceMediaFile.delete(); + } catch (Exception e) { + logger.catching(e); + } + try { + if (deleteSource && targetPath != null && targetPath.toFile().exists()) + Files.delete(targetPath); + } catch (Exception e) { + logger.catching(e); + } + } + return null; + } + + private void postprocess(Path transcodedFilePath, String webPath) throws IOException { + Path lowresPath = null; + try { + String transcodedFileName = transcodedFilePath.getFileName().toString(); + String targetPath = null; + if (transcodedFileName.indexOf(".") > 2) { + Path subdir = Paths.get(transcodedFileName.substring(0, 1), transcodedFileName.substring(1, 2), + transcodedFileName.substring(2, 3)); + EscortFiles.ensureUNCFolder(webPath, subdir.toString()); + targetPath = Paths.get(subdir.toString(), transcodedFileName).toString(); + } else { + targetPath = transcodedFileName; + } + lowresPath = Paths.get(webPath, targetPath); + int version = 1; + while (lowresPath.toFile().exists()) { + String fileName = transcodedFileName + version + MP4EXT; + lowresPath = Paths.get(lowresPath.toString().replace(transcodedFileName, fileName)); + targetPath = targetPath.replace(transcodedFileName, fileName); + transcodedFileName = fileName; + version++; + } + + Files.move(transcodedFilePath, lowresPath); + manager.createMediaFile(targetPath, fileType, store, mediaCubeMedia).add(); + } catch (IOException e) { + logger.catching(e); + logger.error(marker, "A(z) '{}' állomány mozgatása a '{}' helyre nem sikerült.", transcodedFilePath, + lowresPath); + throw e; + } + } + +} diff --git a/server/user.jobengine.log4j/.gitignore b/server/user.jobengine.log4j/.gitignore new file mode 100644 index 00000000..ae3c1726 --- /dev/null +++ b/server/user.jobengine.log4j/.gitignore @@ -0,0 +1 @@ +/bin/ diff --git a/server/user.jobengine.osgi.commons/build-commons-module.launch b/server/user.jobengine.osgi.commons/build-commons-module.launch new file mode 100644 index 00000000..fa575a6a --- /dev/null +++ b/server/user.jobengine.osgi.commons/build-commons-module.launch @@ -0,0 +1,20 @@ + + + + + + + + + + + + + + + + + + + + diff --git a/server/user.jobengine.osgi.commons/src/user/commons/MediaSamuraiAPI.java b/server/user.jobengine.osgi.commons/src/user/commons/MediaSamuraiAPI.java new file mode 100644 index 00000000..9dc7acc7 --- /dev/null +++ b/server/user.jobengine.osgi.commons/src/user/commons/MediaSamuraiAPI.java @@ -0,0 +1,284 @@ +package user.commons; + +import java.io.File; +import java.nio.file.Files; +import java.nio.file.Paths; +import java.util.List; +import java.util.concurrent.Executors; +import java.util.concurrent.ThreadPoolExecutor; +import java.util.concurrent.TimeUnit; + +import javax.ws.rs.client.Entity; +import javax.ws.rs.core.MediaType; +import javax.ws.rs.core.Response; + +import org.apache.commons.io.FilenameUtils; +import org.apache.commons.lang.StringUtils; +import org.apache.logging.log4j.LogManager; +import org.apache.logging.log4j.Logger; +import org.jboss.resteasy.client.jaxrs.ResteasyClientBuilder; +import org.jboss.resteasy.client.jaxrs.ResteasyWebTarget; + +import com.ibm.nosql.json.JSONUtil; +import com.ibm.nosql.json.api.BasicDBObject; + +import user.commons.nosql.NoSQLUtils; + +public class MediaSamuraiAPI implements IFFAStransAPI { + + private static final Logger logger = LogManager.getLogger(); + + public static void main(String[] args) throws Exception { + final ThreadPoolExecutor executor = (ThreadPoolExecutor) Executors.newFixedThreadPool(10); + + String inDir = "\\\\10.11.1.90\\data\\"; + String outDir = "\\\\10.11.1.100\\Promise\\TRANSCODER\\FFASTRANSCODER\\Out\\"; + + String inputs[] = { "13-02009-0000-1.mov" }; + for (int i = 0; i < inputs.length; i++) { + final String inputi = inDir + inputs[i]; + final String outputi = outDir + FilenameUtils.removeExtension(inputs[i]) + ".MP4"; + // Task task = new Task(inDir + input, outDir + output); + + Runnable task = new Runnable() { + private String input = inputi; + private String output = outputi; + + @Override + public void run() { + try { + IFFAStransAPI api = new MediaSamuraiAPI("http://10.11.1.111:65445/api/json/v1/", p -> { + // System.out.println(output + " progress: " + p); + }); + + api.submit("MP4", input); + api.monitor(1000); + Thread.sleep(2000); + if (!Files.exists(Paths.get(output))) + throw new Exception("Missing " + output); + } catch (Exception e) { + e.printStackTrace(); + } + } + + }; + + executor.execute(task); + } + executor.shutdown(); + executor.awaitTermination(1, TimeUnit.HOURS); + } + + private ResteasyWebTarget webTarget; + private IProgressChangedListener listener; + private String jobId; + private String inputFile; + private String workflowName; + private String inputFileName; + + public MediaSamuraiAPI(String apiAddress, IProgressChangedListener listener) { + this.listener = listener; + webTarget = new ResteasyClientBuilder().build().target(apiAddress); + } + + private void doSubmit(BasicDBObject jobToSubmit) throws Exception { + ResteasyWebTarget target = webTarget.path("jobs"); + Response apiResponse = target.request().post(Entity.entity(jobToSubmit.toString(), MediaType.APPLICATION_JSON)); + if (apiResponse.getStatus() != 202) + throw new Exception("Can not submit, response status is: " + apiResponse.getStatus()); + String json = apiResponse.readEntity(String.class); + // logger.info("Transoder response: {}", json); + if (StringUtils.isBlank(json)) + throw new Exception("Can not submit, response JSON is empty"); + BasicDBObject resultObject = (BasicDBObject) JSONUtil.jsonToDbObject(json); + if (resultObject == null) + throw new Exception("Can not submit, response object is null"); + jobId = resultObject.getString("job_id"); + logger.info("Job for {} is submitted, ID: {}", inputFile, jobId); + } + + @Override + public BasicDBObject getHistory() { + ResteasyWebTarget target = webTarget.path("history"); + BasicDBObject result = null; + try { + Response apiResponse = target.request().get(); + if (apiResponse.getStatus() != 200) { + logger.info("{} | Invalid response {}", inputFileName, apiResponse.getStatus()); + return null; + } + String json = apiResponse.readEntity(String.class); + BasicDBObject resultObject = (BasicDBObject) JSONUtil.jsonToDbObject(json); + List jobs = NoSQLUtils.asList(resultObject, "history"); + + if (jobs == null || jobs.size() == 0) { + logger.info("{} | No jobs in response", inputFileName); + return null; + } + + File f = new File(inputFile); + + for (BasicDBObject job : jobs) { + if (job == null) + continue; + + String file = NoSQLUtils.asString(job, "file"); + if (file == null) + continue; + + String wf_name = NoSQLUtils.asString(job, "wf_name"); + if (wf_name == null || !workflowName.equals(wf_name)) + continue; + + if (file.equals(f.getName())) { + // logger.info("Found success history {}", f.getName()); + result = job; + break; + } + if (file.equals(inputFile)) { + // logger.info("Found error history {}", inputFile); + result = job; + break; + } + } + } catch (Exception e) { + logger.error(e.getClass() + " " + e.getMessage()); + } + + return result; + } + + @Override + public BasicDBObject getStatus(String jobID) { + ResteasyWebTarget target = webTarget.path("jobs"); + + BasicDBObject result = null; + try { + Response apiResponse = target.request().get(); + // logger.info("Transoder response code: {}", apiResponse.getStatus()); + + if (apiResponse.getStatus() != 200) { + logger.info("{} | Invalid response {}", inputFileName, apiResponse.getStatus()); + return null; + } + String json = apiResponse.readEntity(String.class); + // logger.info("Transoder response: {}", json); + // System.out.println(json); + BasicDBObject resultObject = (BasicDBObject) JSONUtil.jsonToDbObject(json); + List jobs = NoSQLUtils.asList(resultObject, "jobs"); + + if (jobs == null || jobs.size() == 0) { + logger.info("{} | No jobs in response", inputFileName); + return null; + } + for (BasicDBObject job : jobs) { + if (jobID.equals(job.getString("job_id"))) { + // logger.info("Found job {}", jobID); + result = job; + break; + } + } + + } catch (Exception e) { + logger.error(e.getClass() + " " + e.getMessage()); + } + + return result; + } + + private long getWorkflowId(String workflowName, List workflows, long wfID) { + for (BasicDBObject workflow : workflows) { + if (!workflowName.equals(workflow.getString("wf_name"))) + continue; + wfID = workflow.getLong("wf_id"); + } + return wfID; + } + + @Override + public List getWorkflows() { + ResteasyWebTarget target = webTarget.path("workflows"); + Response apiResponse = target.request().get(); + if (apiResponse.getStatus() != 200) + return null; + String json = apiResponse.readEntity(String.class); + BasicDBObject resultObject = (BasicDBObject) JSONUtil.jsonToDbObject(json); + return NoSQLUtils.asList(resultObject, "workflows"); + } + + @Override + public void monitor(int pollIntervall) throws InterruptedException, Exception { + int progress = 0; + + long started = System.currentTimeMillis(); + + BasicDBObject status = null; + boolean hasAnyResponse = false; + while (true) { + long queryTime = System.currentTimeMillis(); + + status = getStatus(jobId); + if (status != null) { + hasAnyResponse = true; + List splits = NoSQLUtils.asList(status, "splits"); + + int current = 0; + if (splits != null && splits.size() > 0) { + for (BasicDBObject split : splits) { + String prg = NoSQLUtils.asString(split, "progress"); + current += (int) Float.parseFloat(prg); + } + current = current / splits.size(); + } + if (current != progress) { + progress = current; + listener.onProgressChanged(progress); + logger.info("{} | {}%", inputFileName, progress); + } + + } else { + logger.info("{} status does not exist, checking history", inputFileName); + listener.onProgressChanged(100); + BasicDBObject history = getHistory(); + if (history != null) { + hasAnyResponse = true; + long state = NoSQLUtils.asLong(history, "state"); + String jobEnd = history.getString("job_end"); + if (history == null || state != 1) { + String error = NoSQLUtils.asString(history, "outcome"); + if (error == null) + error = "Not specified error occured"; + throw new Exception("State: " + state + ", message: " + error); + } else { + logger.info("{} | completed", inputFileName); + break; + } + + } + } + + // 5 percig nincs valasz + if (!hasAnyResponse && (queryTime - started > 10 * 60 * 1000)) + throw new Exception("Transcoder timeout"); + Thread.sleep(pollIntervall); + } + } + + @Override + public void submit(String workflowName, String inputFile) throws Exception { + this.workflowName = workflowName; + this.inputFile = inputFile; + this.inputFileName = Paths.get(inputFile).getFileName().toString(); + List workflows = getWorkflows(); + if (workflows == null) + throw new Exception("No workflows"); + + long wfID = -1; + wfID = getWorkflowId(workflowName, workflows, wfID); + if (wfID < 0) + throw new Exception("Workflow does not exist: " + workflowName); + + BasicDBObject jobToSubmit = new BasicDBObject("wf_id", wfID).append("inputfile", inputFile); + doSubmit(jobToSubmit); + } +} diff --git a/server/user.mediacube.gui/.settings/org.eclipse.core.resources.prefs b/server/user.mediacube.gui/.settings/org.eclipse.core.resources.prefs new file mode 100644 index 00000000..803674ea --- /dev/null +++ b/server/user.mediacube.gui/.settings/org.eclipse.core.resources.prefs @@ -0,0 +1,12 @@ +eclipse.preferences.version=1 +encoding//pages/jobselector.zul=UTF-8 +encoding//pages/login.zul=UTF-8 +encoding//resources/i3-label_hu.properties=UTF-8 +encoding//src/user/jobengine/zk/model/MaestroJobListModel.java=UTF-8 +encoding//src/user/jobengine/zk/model/MissingMaterialsModel.java=UTF-8 +encoding//src/user/jobengine/zk/model/RetrieveBatchSelectorModel.java=UTF-8 +encoding//src/user/jobengine/zk/model/RetrieveSelectorModel.java=UTF-8 +encoding//src/user/jobengine/zk/model/TargetsListModel.java=UTF-8 +encoding//src/user/jobengine/zk/util/ADHandler.java=UTF-8 +encoding//src/user/jobengine/zk/util/SessionUtil.java=UTF-8 +encoding/=UTF-8 -- 2.54.0