/**
* Create converted ffmpeg payload
*
* @param object DigitalObject to store the payload
* @param file File to be stored as payload
* @return Payload new payload
* @throws StorageException if there is a problem trying to store
* @throws FileNotFoundException if the file provided does not exist
*/
public Payload createFfmpegPayload(DigitalObject object, File file)
throws StorageException, FileNotFoundException {
String name = file.getName();
Payload payload = StorageUtils.createOrUpdatePayload(object, name, new FileInputStream(file));
payload.setContentType(MimeTypeUtil.getMimeType(name));
payload.setLabel(name);
return payload;
}
/**
* Compile all available metadata and add to object.
*
* <p>Note that a False response indicates data has been written into the 'errors' map and should
* be followed by an 'errorAndClose()' call.
*
* @return: boolean: False if any errors occurred, True otherwise
*/
private boolean compileMetadata(DigitalObject object) {
// Nothing to do
if (info == null) {
return true;
}
if (!info.isSupported()) {
return true;
}
// Get base metadata of source
JsonConfigHelper fullMetadata = null;
try {
fullMetadata = new JsonConfigHelper(info.toString());
} catch (IOException ex) {
addError("metadata", "Error parsing metadata output", ex);
return false;
}
// Add individual conversion(s) metadata
if (!metadata.isEmpty()) {
fullMetadata.setJsonMap("outputs", metadata);
}
// Write the file to disk
File metaFile;
try {
// log.debug("\nMetadata:\n{}", fullMetadata.toString());
metaFile = writeMetadata(fullMetadata.toString());
if (metaFile == null) {
addError("metadata", "Unknown error extracting metadata");
return false;
}
} catch (TransformerException ex) {
addError("metadata", "Error writing metadata to disk", ex);
return false;
}
// Store metadata
try {
Payload payload = createFfmpegPayload(object, metaFile);
payload.setType(PayloadType.Enrichment);
payload.close();
} catch (Exception ex) {
addError("metadata", "Error storing metadata payload", ex);
return false;
} finally {
metaFile.delete();
}
// Everything should be fine if we got here
return true;
}
/**
* Create ffmpeg error payload
*
* @param object : DigitalObject to store the payload
* @return Payload the error payload
* @throws FileNotFoundException if the file provided does not exist
* @throws UnsupportedEncodingException for encoding errors in the message
*/
public Payload createFfmpegErrorPayload(DigitalObject object)
throws StorageException, FileNotFoundException, UnsupportedEncodingException {
// Compile our error data
JsonConfigHelper content = new JsonConfigHelper();
content.setJsonMap("/", errors);
log.debug("\nErrors:\n{}", content.toString());
InputStream data = new ByteArrayInputStream(content.toString().getBytes("UTF-8"));
// Write to the object
Payload payload = StorageUtils.createOrUpdatePayload(object, ERROR_PAYLOAD, data);
payload.setType(PayloadType.Error);
payload.setContentType("application/json");
payload.setLabel("FFMPEG conversion errors");
return payload;
}
/**
* Read FFMPEG metadata from the object if it exists
*
* @param object: The object to extract data from
*/
private void readMetadata(DigitalObject object) {
Set<String> pids = object.getPayloadIdList();
if (pids.contains(METADATA_PAYLOAD)) {
try {
Payload payload = object.getPayload(METADATA_PAYLOAD);
JsonConfigHelper data = new JsonConfigHelper(payload.open());
payload.close();
oldMetadata = data.getJsonMap("outputs");
// for (String k : oldMetadata.keySet()) {
// log.debug("\n====\n{}\n===\n{}", k, oldMetadata.get(k));
// }
} catch (IOException ex) {
log.error("Error parsing metadata JSON: ", ex);
} catch (StorageException ex) {
log.error("Error accessing metadata payload: ", ex);
}
}
}
@Test
public void addAndGetObject1() throws Exception {
fs.addObject(newObject);
FileSystemDigitalObject addedObject =
(FileSystemDigitalObject) fs.getObject("oai:eprints.usq.edu.au:318");
Assert.assertEquals(
FilenameUtils.normalize(
tmpDir
+ "/_fs_test/d0b1c5bd0660ad67a16b7111aafc9389/"
+ "e2/92/e292378c5b38b0d5a4aba11fd40e7151"),
addedObject.getPath().getAbsolutePath());
List<Payload> payloads = addedObject.getPayloadList();
Assert.assertEquals(1, payloads.size());
Payload payload = payloads.get(0);
Assert.assertEquals("Dublin Core Metadata", payload.getLabel());
}
/**
* Stream data from specified payload into a file in our temp cache.
*
* @param object: The digital object to use
* @param pid: The payload ID to extract
* @return File: The cached File
* @throws FileNotFoundException: If accessing the cache fails
* @throws StorageException: If accessing the object in storage fails
* @throws IOException: If the data copy fails
*/
private File cacheFile(DigitalObject object, String pid)
throws FileNotFoundException, StorageException, IOException {
// Get our cache location
File file = new File(outputDir, pid);
FileOutputStream tempFileOut = new FileOutputStream(file);
// Get payload from storage
Payload payload = object.getPayload(pid);
try {
// Copy to cache
IOUtils.copy(payload.open(), tempFileOut);
} catch (IOException ex) {
payload.close();
throw ex;
}
// Close and return
payload.close();
tempFileOut.close();
return file;
}
@Test
public void addAndGetObject2() throws Exception {
fs.addObject(fileObject);
FileSystemDigitalObject addedObject =
(FileSystemDigitalObject) fs.getObject("/Users/fascinator/Documents/sample.odt");
Assert.assertEquals(
FilenameUtils.normalize(
tmpDir
+ "/_fs_test/d0b1c5bd0660ad67a16b7111aafc9389/"
+ "11/b4/11b498d057256a0b602fa0e7c4073fc3"),
addedObject.getPath().getAbsolutePath());
List<Payload> payloads = addedObject.getPayloadList();
Assert.assertEquals(2, payloads.size());
Payload payload1 = addedObject.getPayload("sample.odt");
Assert.assertEquals("ICE Sample Document", payload1.getLabel());
Payload payload2 = addedObject.getPayload("images/ice-services.png");
Assert.assertEquals("ICE Services Diagram", payload2.getLabel());
}
@Test
public void transformAudio() throws Exception {
Transformer t = new FfmpegTransformer(ffmpeg);
t.init(new File(getClass().getResource("/ffmpeg-config.json").toURI()));
DigitalObject outputObject = t.transform(sourceObject);
// should have 2 payloads
Assert.assertEquals("There should be 2 Payloads", 2, outputObject.getPayloadIdList().size());
String preview = null;
for (String i : outputObject.getPayloadIdList()) {
Payload p = outputObject.getPayload(i);
if (p.getType() == PayloadType.Preview) {
preview = i;
}
}
// should have a preview payload
Assert.assertNotNull("Should have a Preview", preview);
outputObject.close();
}
public void addPayload(String oid, Payload payload) {
log.debug("Adding payload {} to {}", payload.getId(), oid);
FileSystemDigitalObject fileObject = (FileSystemDigitalObject) getObject(oid);
FileSystemPayload filePayload = new FileSystemPayload(fileObject.getPath(), payload);
File payloadFile = filePayload.getFile();
File parentDir = payloadFile.getParentFile();
parentDir.mkdirs();
try {
FileOutputStream out = new FileOutputStream(payloadFile);
IOUtils.copy(filePayload.getInputStream(), out);
out.close();
} catch (IOException ioe) {
log.error("Failed to add payload", ioe);
}
}
/**
* Check the object for a multi-segment source and merge them. Such sources must come from a
* harvester specifically designed to match this transformer. As such we can make certain
* assumptions, and if they are not met we just fail silently with a log entry.
*
* @param object: The digital object to modify
*/
private void mergeSegments(DigitalObject object) {
try {
// Retrieve (optional) segment information from metadata
Properties props = object.getMetadata();
String segs = props.getProperty("mediaSegments");
if (segs == null) {
return;
}
int segments = Integer.parseInt(segs);
if (segments <= 1) {
return;
}
// We need to do some merging, lets validate IDs first
log.info("Found {} source segments! Merging...", segments);
List<String> segmentIds = new ArrayList();
Set<String> payloadIds = object.getPayloadIdList();
// The first segment
String sourceId = object.getSourceId();
if (sourceId == null || !payloadIds.contains(sourceId)) {
log.error("Cannot find source payload.");
return;
}
segmentIds.add(sourceId);
// Find the other segments
for (int i = 1; i < segments; i++) {
// We won't know the extension though
String segmentId = "segment" + i + ".";
for (String pid : payloadIds) {
if (pid.startsWith(segmentId)) {
segmentIds.add(pid);
}
}
}
// Did we find every segment?
if (segmentIds.size() != segments) {
log.error("Unable to find all segments in payload list.");
return;
}
// Transcode all the files to neutral MPEGs first
Map<String, File> files = new HashMap();
for (String segment : segmentIds) {
try {
File file = basicMpeg(object, segment);
if (file != null) {
files.put(segment, file);
}
} catch (Exception ex) {
log.error("Error transcoding segment to MPEG: ", ex);
// Cleanup
for (File f : files.values()) {
if (f.exists()) {
f.delete();
}
}
return;
}
}
// Did every transcoding succeed?
if (files.size() != segments) {
log.error("At least one segment transcoding failed.");
// Cleanup
for (File f : files.values()) {
if (f.exists()) {
f.delete();
}
}
return;
}
// Now to try merging all the segments. In MPEG format
// they can just be concatenated.
try {
// Create our output file
String filename = "temp_" + MERGED_PAYLOAD + "mpg";
File merged = new File(outputDir, filename);
if (merged.exists()) {
merged.delete();
}
FileOutputStream out = new FileOutputStream(merged);
// Merge each segment in order
for (String sId : segmentIds) {
try {
mergeSegment(out, files.get(sId));
} catch (IOException ex) {
log.error("Failed to stream to merged file: ", ex);
out.close();
// Cleanup
for (File f : files.values()) {
if (f.exists()) {
f.delete();
}
}
merged.delete();
return;
}
}
out.close();
// Final step, run the output file through a transcoding to
// write the correct metadata (eg. duration)
filename = MERGED_PAYLOAD + finalFormat;
File transcoded = new File(outputDir, filename);
if (transcoded.exists()) {
transcoded.delete();
}
// Render
String stderr = mergeRender(merged, transcoded);
log.debug("=====\n{}", stderr);
if (transcoded.exists()) {
// Now we need to 'fix' the object, add the new source
FileInputStream fis = new FileInputStream(transcoded);
String pid = transcoded.getName();
Payload p = StorageUtils.createOrUpdatePayload(object, pid, fis);
fis.close();
p.setType(PayloadType.Source);
object.setSourceId(pid);
// Remove all the old segments
for (String sId : segmentIds) {
object.removePayload(sId);
}
props.remove("mediaSegments");
object.close();
// Cleanup segments
for (File f : files.values()) {
if (f.exists()) {
f.delete();
}
}
merged.delete();
transcoded.delete();
}
} catch (IOException ex) {
log.error("Error merging segments: ", ex);
}
} catch (StorageException ex) {
log.error("Error accessing object metadata: ", ex);
}
}
/**
* Transforming digital object method
*
* @params object: DigitalObject to be transformed
* @return transformed DigitalObject after transformation
* @throws TransformerException if the transformation fails
*/
@Override
public DigitalObject transform(DigitalObject object, String jsonConfig)
throws TransformerException {
if (testExecLevel() == null) {
log.error("FFmpeg is either not installed, or not executing!");
return object;
}
// Purge old data
reset();
oid = object.getId();
outputDir = new File(outputRoot, oid);
outputDir.mkdirs();
try {
itemConfig = new JsonConfigHelper(jsonConfig);
} catch (IOException ex) {
throw new TransformerException("Invalid configuration! '{}'", ex);
}
// Resolve multi-segment files first
mergeRate = get(itemConfig, "merging/mpegFrameRate", "25");
finalRate = get(itemConfig, "merging/finalFrameRate", "10");
finalFormat = get(itemConfig, "merging/finalFormat", "avi");
mergeSegments(object);
// Find the format 'group' this file is in
String sourceId = object.getSourceId();
String ext = FilenameUtils.getExtension(sourceId);
format = getFormat(ext);
// Return now if this isn't a format we care about
if (format == null) {
return object;
}
// log.debug("Supported format found: '{}' => '{}'", ext, format);
// Cache the file from storage
File file;
try {
file = cacheFile(object, sourceId);
} catch (IOException ex) {
addError(sourceId, "Error writing temp file", ex);
errorAndClose(object);
return object;
} catch (StorageException ex) {
addError(sourceId, "Error accessing storage data", ex);
errorAndClose(object);
return object;
}
if (!file.exists()) {
addError(sourceId, "Unknown error writing cache: does not exist");
errorAndClose(object);
return object;
}
// **************************************************************
// From here on we know (assume) that we SHOULD be able to support
// this object, so errors can't just throw exceptions. We should
// only return under certain circumstances (ie. not just because
// one rendition fails), and the object must get closed.
// **************************************************************
// Read any pre-existing rendition metadata from previous tranformations
// ++++++++++++++++++++++++++++
// TODO: This is useless until the last modified date can be retrieved
// against the source file. Storage API does not currently support this,
// it just returns a data stream.
//
// Once this feature exists the basic algorithm should be:
// 1) Retrieve old metadata
// 2) Loop through each rendition preparation as normal
// 3) When the transcoding is ready to start, use the parameters to
// query the database for the last time the exact transcoding was
// and comparing against last modifed.
// 4) If the transcoding is newer than the source, skip running FFmpeg
// and just use the same metadata as last time.
// ++++++++++++++++++++++++++++
// readMetadata(object);
// Check for a custom display type
String display = get(itemConfig, "displayTypes/" + format);
if (display != null) {
try {
Properties prop = object.getMetadata();
prop.setProperty("displayType", display);
prop.setProperty("previewType", display);
} catch (StorageException ex) {
addError("display", "Could not access object metadata", ex);
}
}
// Gather metadata
try {
info = ffmpeg.getInfo(file);
} catch (IOException ex) {
addError("metadata", "Error accessing metadata", ex);
errorAndClose(object);
return object;
}
// Can we even process this file?
if (!info.isSupported()) {
closeObject(object);
return object;
}
// What conversions are required for this format?
List<JsonConfigHelper> conversions = getJsonList(itemConfig, "transcodings/" + format);
for (JsonConfigHelper conversion : conversions) {
String name = conversion.get("alias");
// And what/how many renditions does it have?
List<JsonConfigHelper> renditions = conversion.getJsonList("renditions");
if (renditions == null || renditions.isEmpty()) {
addError(
"transcodings",
"Invalid or missing transcoding data:" + " '/transcodings/" + format + "'");
} else {
// Config look valid, lets give it a try
// log.debug("Starting renditions for '{}'", name);
for (JsonConfigHelper render : renditions) {
File converted = null;
// Render the output
try {
converted = convert(file, render, info);
} catch (Exception ex) {
String outputFile = render.get("name");
if (outputFile != null) {
addError(jsonKey(outputFile), "Error converting file", ex);
} else {
// Couldn't read the config for a name
addError("unknown", "Error converting file", ex);
}
}
// Now store the output if valid
if (converted != null) {
try {
Payload payload = createFfmpegPayload(object, converted);
// TODO: Type checking needs more work
// Indexing fails silently if you add two thumbnails
// or two previews
payload.setType(resolveType(render.get("type")));
payload.close();
} catch (Exception ex) {
addError(jsonKey(converted.getName()), "Error storing output", ex);
} finally {
converted.delete();
}
}
}
}
}
// Write metadata to storage
if (compileMetadata(object)) {
// Close normally
closeObject(object);
} else {
// Close with some errors
errorAndClose(object);
}
// Cleanup
if (file.exists()) {
file.delete();
}
return object;
}
