Exemplo n.º 1
0
  /**
   * Compile all available metadata and add to object.
   *
   * <p>Note that a False response indicates data has been written into the 'errors' map and should
   * be followed by an 'errorAndClose()' call.
   *
   * @return: boolean: False if any errors occurred, True otherwise
   */
  private boolean compileMetadata(DigitalObject object) {
    // Nothing to do
    if (info == null) {
      return true;
    }
    if (!info.isSupported()) {
      return true;
    }

    // Get base metadata of source
    JsonConfigHelper fullMetadata = null;
    try {
      fullMetadata = new JsonConfigHelper(info.toString());
    } catch (IOException ex) {
      addError("metadata", "Error parsing metadata output", ex);
      return false;
    }

    // Add individual conversion(s) metadata
    if (!metadata.isEmpty()) {
      fullMetadata.setJsonMap("outputs", metadata);
    }

    // Write the file to disk
    File metaFile;
    try {
      // log.debug("\nMetadata:\n{}", fullMetadata.toString());
      metaFile = writeMetadata(fullMetadata.toString());
      if (metaFile == null) {
        addError("metadata", "Unknown error extracting metadata");
        return false;
      }
    } catch (TransformerException ex) {
      addError("metadata", "Error writing metadata to disk", ex);
      return false;
    }

    // Store metadata
    try {
      Payload payload = createFfmpegPayload(object, metaFile);
      payload.setType(PayloadType.Enrichment);
      payload.close();
    } catch (Exception ex) {
      addError("metadata", "Error storing metadata payload", ex);
      return false;
    } finally {
      metaFile.delete();
    }

    // Everything should be fine if we got here
    return true;
  }
Exemplo n.º 2
0
 /**
  * Create ffmpeg error payload
  *
  * @param object : DigitalObject to store the payload
  * @return Payload the error payload
  * @throws FileNotFoundException if the file provided does not exist
  * @throws UnsupportedEncodingException for encoding errors in the message
  */
 public Payload createFfmpegErrorPayload(DigitalObject object)
     throws StorageException, FileNotFoundException, UnsupportedEncodingException {
   // Compile our error data
   JsonConfigHelper content = new JsonConfigHelper();
   content.setJsonMap("/", errors);
   log.debug("\nErrors:\n{}", content.toString());
   InputStream data = new ByteArrayInputStream(content.toString().getBytes("UTF-8"));
   // Write to the object
   Payload payload = StorageUtils.createOrUpdatePayload(object, ERROR_PAYLOAD, data);
   payload.setType(PayloadType.Error);
   payload.setContentType("application/json");
   payload.setLabel("FFMPEG conversion errors");
   return payload;
 }
Exemplo n.º 3
0
  /**
   * Check the object for a multi-segment source and merge them. Such sources must come from a
   * harvester specifically designed to match this transformer. As such we can make certain
   * assumptions, and if they are not met we just fail silently with a log entry.
   *
   * @param object: The digital object to modify
   */
  private void mergeSegments(DigitalObject object) {
    try {
      // Retrieve (optional) segment information from metadata
      Properties props = object.getMetadata();
      String segs = props.getProperty("mediaSegments");
      if (segs == null) {
        return;
      }
      int segments = Integer.parseInt(segs);
      if (segments <= 1) {
        return;
      }

      // We need to do some merging, lets validate IDs first
      log.info("Found {} source segments! Merging...", segments);
      List<String> segmentIds = new ArrayList();
      Set<String> payloadIds = object.getPayloadIdList();
      // The first segment
      String sourceId = object.getSourceId();
      if (sourceId == null || !payloadIds.contains(sourceId)) {
        log.error("Cannot find source payload.");
        return;
      }
      segmentIds.add(sourceId);
      // Find the other segments
      for (int i = 1; i < segments; i++) {
        // We won't know the extension though
        String segmentId = "segment" + i + ".";
        for (String pid : payloadIds) {
          if (pid.startsWith(segmentId)) {
            segmentIds.add(pid);
          }
        }
      }

      // Did we find every segment?
      if (segmentIds.size() != segments) {
        log.error("Unable to find all segments in payload list.");
        return;
      }

      // Transcode all the files to neutral MPEGs first
      Map<String, File> files = new HashMap();
      for (String segment : segmentIds) {
        try {
          File file = basicMpeg(object, segment);
          if (file != null) {
            files.put(segment, file);
          }
        } catch (Exception ex) {
          log.error("Error transcoding segment to MPEG: ", ex);
          // Cleanup
          for (File f : files.values()) {
            if (f.exists()) {
              f.delete();
            }
          }
          return;
        }
      }

      // Did every transcoding succeed?
      if (files.size() != segments) {
        log.error("At least one segment transcoding failed.");
        // Cleanup
        for (File f : files.values()) {
          if (f.exists()) {
            f.delete();
          }
        }
        return;
      }

      // Now to try merging all the segments. In MPEG format
      // they can just be concatenated.
      try {
        // Create our output file
        String filename = "temp_" + MERGED_PAYLOAD + "mpg";
        File merged = new File(outputDir, filename);
        if (merged.exists()) {
          merged.delete();
        }
        FileOutputStream out = new FileOutputStream(merged);

        // Merge each segment in order
        for (String sId : segmentIds) {
          try {
            mergeSegment(out, files.get(sId));
          } catch (IOException ex) {
            log.error("Failed to stream to merged file: ", ex);
            out.close();
            // Cleanup
            for (File f : files.values()) {
              if (f.exists()) {
                f.delete();
              }
            }
            merged.delete();
            return;
          }
        }
        out.close();

        // Final step, run the output file through a transcoding to
        // write the correct metadata (eg. duration)
        filename = MERGED_PAYLOAD + finalFormat;
        File transcoded = new File(outputDir, filename);
        if (transcoded.exists()) {
          transcoded.delete();
        }

        // Render
        String stderr = mergeRender(merged, transcoded);
        log.debug("=====\n{}", stderr);
        if (transcoded.exists()) {
          // Now we need to 'fix' the object, add the new source
          FileInputStream fis = new FileInputStream(transcoded);
          String pid = transcoded.getName();
          Payload p = StorageUtils.createOrUpdatePayload(object, pid, fis);
          fis.close();
          p.setType(PayloadType.Source);
          object.setSourceId(pid);

          // Remove all the old segments
          for (String sId : segmentIds) {
            object.removePayload(sId);
          }
          props.remove("mediaSegments");
          object.close();

          // Cleanup segments
          for (File f : files.values()) {
            if (f.exists()) {
              f.delete();
            }
          }
          merged.delete();
          transcoded.delete();
        }
      } catch (IOException ex) {
        log.error("Error merging segments: ", ex);
      }

    } catch (StorageException ex) {
      log.error("Error accessing object metadata: ", ex);
    }
  }
Exemplo n.º 4
0
  /**
   * Transforming digital object method
   *
   * @params object: DigitalObject to be transformed
   * @return transformed DigitalObject after transformation
   * @throws TransformerException if the transformation fails
   */
  @Override
  public DigitalObject transform(DigitalObject object, String jsonConfig)
      throws TransformerException {
    if (testExecLevel() == null) {
      log.error("FFmpeg is either not installed, or not executing!");
      return object;
    }
    // Purge old data
    reset();
    oid = object.getId();
    outputDir = new File(outputRoot, oid);
    outputDir.mkdirs();

    try {
      itemConfig = new JsonConfigHelper(jsonConfig);
    } catch (IOException ex) {
      throw new TransformerException("Invalid configuration! '{}'", ex);
    }

    // Resolve multi-segment files first
    mergeRate = get(itemConfig, "merging/mpegFrameRate", "25");
    finalRate = get(itemConfig, "merging/finalFrameRate", "10");
    finalFormat = get(itemConfig, "merging/finalFormat", "avi");
    mergeSegments(object);

    // Find the format 'group' this file is in
    String sourceId = object.getSourceId();
    String ext = FilenameUtils.getExtension(sourceId);
    format = getFormat(ext);

    // Return now if this isn't a format we care about
    if (format == null) {
      return object;
    }
    // log.debug("Supported format found: '{}' => '{}'", ext, format);

    // Cache the file from storage
    File file;
    try {
      file = cacheFile(object, sourceId);
    } catch (IOException ex) {
      addError(sourceId, "Error writing temp file", ex);
      errorAndClose(object);
      return object;
    } catch (StorageException ex) {
      addError(sourceId, "Error accessing storage data", ex);
      errorAndClose(object);
      return object;
    }
    if (!file.exists()) {
      addError(sourceId, "Unknown error writing cache: does not exist");
      errorAndClose(object);
      return object;
    }

    // **************************************************************
    // From here on we know (assume) that we SHOULD be able to support
    // this object, so errors can't just throw exceptions. We should
    // only return under certain circumstances (ie. not just because
    // one rendition fails), and the object must get closed.
    // **************************************************************

    // Read any pre-existing rendition metadata from previous tranformations
    // ++++++++++++++++++++++++++++
    // TODO: This is useless until the last modified date can be retrieved
    // against the source file. Storage API does not currently support this,
    // it just returns a data stream.
    //
    // Once this feature exists the basic algorithm should be:
    // 1) Retrieve old metadata
    // 2) Loop through each rendition preparation as normal
    // 3) When the transcoding is ready to start, use the parameters to
    // query the database for the last time the exact transcoding was
    // and comparing against last modifed.
    // 4) If the transcoding is newer than the source, skip running FFmpeg
    // and just use the same metadata as last time.
    // ++++++++++++++++++++++++++++
    // readMetadata(object);

    // Check for a custom display type
    String display = get(itemConfig, "displayTypes/" + format);
    if (display != null) {
      try {
        Properties prop = object.getMetadata();
        prop.setProperty("displayType", display);
        prop.setProperty("previewType", display);
      } catch (StorageException ex) {
        addError("display", "Could not access object metadata", ex);
      }
    }

    // Gather metadata
    try {
      info = ffmpeg.getInfo(file);
    } catch (IOException ex) {
      addError("metadata", "Error accessing metadata", ex);
      errorAndClose(object);
      return object;
    }

    // Can we even process this file?
    if (!info.isSupported()) {
      closeObject(object);
      return object;
    }

    // What conversions are required for this format?
    List<JsonConfigHelper> conversions = getJsonList(itemConfig, "transcodings/" + format);
    for (JsonConfigHelper conversion : conversions) {
      String name = conversion.get("alias");
      // And what/how many renditions does it have?
      List<JsonConfigHelper> renditions = conversion.getJsonList("renditions");
      if (renditions == null || renditions.isEmpty()) {
        addError(
            "transcodings",
            "Invalid or missing transcoding data:" + " '/transcodings/" + format + "'");
      } else {
        // Config look valid, lets give it a try
        // log.debug("Starting renditions for '{}'", name);
        for (JsonConfigHelper render : renditions) {
          File converted = null;
          // Render the output
          try {
            converted = convert(file, render, info);
          } catch (Exception ex) {
            String outputFile = render.get("name");
            if (outputFile != null) {
              addError(jsonKey(outputFile), "Error converting file", ex);
            } else {
              // Couldn't read the config for a name
              addError("unknown", "Error converting file", ex);
            }
          }

          // Now store the output if valid
          if (converted != null) {
            try {
              Payload payload = createFfmpegPayload(object, converted);
              // TODO: Type checking needs more work
              // Indexing fails silently if you add two thumbnails
              // or two previews
              payload.setType(resolveType(render.get("type")));
              payload.close();
            } catch (Exception ex) {
              addError(jsonKey(converted.getName()), "Error storing output", ex);
            } finally {
              converted.delete();
            }
          }
        }
      }
    }

    // Write metadata to storage
    if (compileMetadata(object)) {
      // Close normally
      closeObject(object);
    } else {
      // Close with some errors
      errorAndClose(object);
    }
    // Cleanup
    if (file.exists()) {
      file.delete();
    }
    return object;
  }