/**
* Compile all available metadata and add to object.
*
* <p>Note that a False response indicates data has been written into the 'errors' map and should
* be followed by an 'errorAndClose()' call.
*
* @return: boolean: False if any errors occurred, True otherwise
*/
private boolean compileMetadata(DigitalObject object) {
// Nothing to do
if (info == null) {
return true;
}
if (!info.isSupported()) {
return true;
}
// Get base metadata of source
JsonConfigHelper fullMetadata = null;
try {
fullMetadata = new JsonConfigHelper(info.toString());
} catch (IOException ex) {
addError("metadata", "Error parsing metadata output", ex);
return false;
}
// Add individual conversion(s) metadata
if (!metadata.isEmpty()) {
fullMetadata.setJsonMap("outputs", metadata);
}
// Write the file to disk
File metaFile;
try {
// log.debug("\nMetadata:\n{}", fullMetadata.toString());
metaFile = writeMetadata(fullMetadata.toString());
if (metaFile == null) {
addError("metadata", "Unknown error extracting metadata");
return false;
}
} catch (TransformerException ex) {
addError("metadata", "Error writing metadata to disk", ex);
return false;
}
// Store metadata
try {
Payload payload = createFfmpegPayload(object, metaFile);
payload.setType(PayloadType.Enrichment);
payload.close();
} catch (Exception ex) {
addError("metadata", "Error storing metadata payload", ex);
return false;
} finally {
metaFile.delete();
}
// Everything should be fine if we got here
return true;
}
/**
* Create ffmpeg error payload
*
* @param object : DigitalObject to store the payload
* @return Payload the error payload
* @throws FileNotFoundException if the file provided does not exist
* @throws UnsupportedEncodingException for encoding errors in the message
*/
public Payload createFfmpegErrorPayload(DigitalObject object)
throws StorageException, FileNotFoundException, UnsupportedEncodingException {
// Compile our error data
JsonConfigHelper content = new JsonConfigHelper();
content.setJsonMap("/", errors);
log.debug("\nErrors:\n{}", content.toString());
InputStream data = new ByteArrayInputStream(content.toString().getBytes("UTF-8"));
// Write to the object
Payload payload = StorageUtils.createOrUpdatePayload(object, ERROR_PAYLOAD, data);
payload.setType(PayloadType.Error);
payload.setContentType("application/json");
payload.setLabel("FFMPEG conversion errors");
return payload;
}
/**
* Check the object for a multi-segment source and merge them. Such sources must come from a
* harvester specifically designed to match this transformer. As such we can make certain
* assumptions, and if they are not met we just fail silently with a log entry.
*
* @param object: The digital object to modify
*/
private void mergeSegments(DigitalObject object) {
try {
// Retrieve (optional) segment information from metadata
Properties props = object.getMetadata();
String segs = props.getProperty("mediaSegments");
if (segs == null) {
return;
}
int segments = Integer.parseInt(segs);
if (segments <= 1) {
return;
}
// We need to do some merging, lets validate IDs first
log.info("Found {} source segments! Merging...", segments);
List<String> segmentIds = new ArrayList();
Set<String> payloadIds = object.getPayloadIdList();
// The first segment
String sourceId = object.getSourceId();
if (sourceId == null || !payloadIds.contains(sourceId)) {
log.error("Cannot find source payload.");
return;
}
segmentIds.add(sourceId);
// Find the other segments
for (int i = 1; i < segments; i++) {
// We won't know the extension though
String segmentId = "segment" + i + ".";
for (String pid : payloadIds) {
if (pid.startsWith(segmentId)) {
segmentIds.add(pid);
}
}
}
// Did we find every segment?
if (segmentIds.size() != segments) {
log.error("Unable to find all segments in payload list.");
return;
}
// Transcode all the files to neutral MPEGs first
Map<String, File> files = new HashMap();
for (String segment : segmentIds) {
try {
File file = basicMpeg(object, segment);
if (file != null) {
files.put(segment, file);
}
} catch (Exception ex) {
log.error("Error transcoding segment to MPEG: ", ex);
// Cleanup
for (File f : files.values()) {
if (f.exists()) {
f.delete();
}
}
return;
}
}
// Did every transcoding succeed?
if (files.size() != segments) {
log.error("At least one segment transcoding failed.");
// Cleanup
for (File f : files.values()) {
if (f.exists()) {
f.delete();
}
}
return;
}
// Now to try merging all the segments. In MPEG format
// they can just be concatenated.
try {
// Create our output file
String filename = "temp_" + MERGED_PAYLOAD + "mpg";
File merged = new File(outputDir, filename);
if (merged.exists()) {
merged.delete();
}
FileOutputStream out = new FileOutputStream(merged);
// Merge each segment in order
for (String sId : segmentIds) {
try {
mergeSegment(out, files.get(sId));
} catch (IOException ex) {
log.error("Failed to stream to merged file: ", ex);
out.close();
// Cleanup
for (File f : files.values()) {
if (f.exists()) {
f.delete();
}
}
merged.delete();
return;
}
}
out.close();
// Final step, run the output file through a transcoding to
// write the correct metadata (eg. duration)
filename = MERGED_PAYLOAD + finalFormat;
File transcoded = new File(outputDir, filename);
if (transcoded.exists()) {
transcoded.delete();
}
// Render
String stderr = mergeRender(merged, transcoded);
log.debug("=====\n{}", stderr);
if (transcoded.exists()) {
// Now we need to 'fix' the object, add the new source
FileInputStream fis = new FileInputStream(transcoded);
String pid = transcoded.getName();
Payload p = StorageUtils.createOrUpdatePayload(object, pid, fis);
fis.close();
p.setType(PayloadType.Source);
object.setSourceId(pid);
// Remove all the old segments
for (String sId : segmentIds) {
object.removePayload(sId);
}
props.remove("mediaSegments");
object.close();
// Cleanup segments
for (File f : files.values()) {
if (f.exists()) {
f.delete();
}
}
merged.delete();
transcoded.delete();
}
} catch (IOException ex) {
log.error("Error merging segments: ", ex);
}
} catch (StorageException ex) {
log.error("Error accessing object metadata: ", ex);
}
}
/**
* Transforming digital object method
*
* @params object: DigitalObject to be transformed
* @return transformed DigitalObject after transformation
* @throws TransformerException if the transformation fails
*/
@Override
public DigitalObject transform(DigitalObject object, String jsonConfig)
throws TransformerException {
if (testExecLevel() == null) {
log.error("FFmpeg is either not installed, or not executing!");
return object;
}
// Purge old data
reset();
oid = object.getId();
outputDir = new File(outputRoot, oid);
outputDir.mkdirs();
try {
itemConfig = new JsonConfigHelper(jsonConfig);
} catch (IOException ex) {
throw new TransformerException("Invalid configuration! '{}'", ex);
}
// Resolve multi-segment files first
mergeRate = get(itemConfig, "merging/mpegFrameRate", "25");
finalRate = get(itemConfig, "merging/finalFrameRate", "10");
finalFormat = get(itemConfig, "merging/finalFormat", "avi");
mergeSegments(object);
// Find the format 'group' this file is in
String sourceId = object.getSourceId();
String ext = FilenameUtils.getExtension(sourceId);
format = getFormat(ext);
// Return now if this isn't a format we care about
if (format == null) {
return object;
}
// log.debug("Supported format found: '{}' => '{}'", ext, format);
// Cache the file from storage
File file;
try {
file = cacheFile(object, sourceId);
} catch (IOException ex) {
addError(sourceId, "Error writing temp file", ex);
errorAndClose(object);
return object;
} catch (StorageException ex) {
addError(sourceId, "Error accessing storage data", ex);
errorAndClose(object);
return object;
}
if (!file.exists()) {
addError(sourceId, "Unknown error writing cache: does not exist");
errorAndClose(object);
return object;
}
// **************************************************************
// From here on we know (assume) that we SHOULD be able to support
// this object, so errors can't just throw exceptions. We should
// only return under certain circumstances (ie. not just because
// one rendition fails), and the object must get closed.
// **************************************************************
// Read any pre-existing rendition metadata from previous tranformations
// ++++++++++++++++++++++++++++
// TODO: This is useless until the last modified date can be retrieved
// against the source file. Storage API does not currently support this,
// it just returns a data stream.
//
// Once this feature exists the basic algorithm should be:
// 1) Retrieve old metadata
// 2) Loop through each rendition preparation as normal
// 3) When the transcoding is ready to start, use the parameters to
// query the database for the last time the exact transcoding was
// and comparing against last modifed.
// 4) If the transcoding is newer than the source, skip running FFmpeg
// and just use the same metadata as last time.
// ++++++++++++++++++++++++++++
// readMetadata(object);
// Check for a custom display type
String display = get(itemConfig, "displayTypes/" + format);
if (display != null) {
try {
Properties prop = object.getMetadata();
prop.setProperty("displayType", display);
prop.setProperty("previewType", display);
} catch (StorageException ex) {
addError("display", "Could not access object metadata", ex);
}
}
// Gather metadata
try {
info = ffmpeg.getInfo(file);
} catch (IOException ex) {
addError("metadata", "Error accessing metadata", ex);
errorAndClose(object);
return object;
}
// Can we even process this file?
if (!info.isSupported()) {
closeObject(object);
return object;
}
// What conversions are required for this format?
List<JsonConfigHelper> conversions = getJsonList(itemConfig, "transcodings/" + format);
for (JsonConfigHelper conversion : conversions) {
String name = conversion.get("alias");
// And what/how many renditions does it have?
List<JsonConfigHelper> renditions = conversion.getJsonList("renditions");
if (renditions == null || renditions.isEmpty()) {
addError(
"transcodings",
"Invalid or missing transcoding data:" + " '/transcodings/" + format + "'");
} else {
// Config look valid, lets give it a try
// log.debug("Starting renditions for '{}'", name);
for (JsonConfigHelper render : renditions) {
File converted = null;
// Render the output
try {
converted = convert(file, render, info);
} catch (Exception ex) {
String outputFile = render.get("name");
if (outputFile != null) {
addError(jsonKey(outputFile), "Error converting file", ex);
} else {
// Couldn't read the config for a name
addError("unknown", "Error converting file", ex);
}
}
// Now store the output if valid
if (converted != null) {
try {
Payload payload = createFfmpegPayload(object, converted);
// TODO: Type checking needs more work
// Indexing fails silently if you add two thumbnails
// or two previews
payload.setType(resolveType(render.get("type")));
payload.close();
} catch (Exception ex) {
addError(jsonKey(converted.getName()), "Error storing output", ex);
} finally {
converted.delete();
}
}
}
}
}
// Write metadata to storage
if (compileMetadata(object)) {
// Close normally
closeObject(object);
} else {
// Close with some errors
errorAndClose(object);
}
// Cleanup
if (file.exists()) {
file.delete();
}
return object;
}