/**
  * Create converted ffmpeg payload
  *
  * @param object DigitalObject to store the payload
  * @param file File to be stored as payload
  * @return Payload new payload
  * @throws StorageException if there is a problem trying to store
  * @throws FileNotFoundException if the file provided does not exist
  */
 public Payload createFfmpegPayload(DigitalObject object, File file)
     throws StorageException, FileNotFoundException {
   String name = file.getName();
   Payload payload = StorageUtils.createOrUpdatePayload(object, name, new FileInputStream(file));
   payload.setContentType(MimeTypeUtil.getMimeType(name));
   payload.setLabel(name);
   return payload;
 }
  /**
   * Compile all available metadata and add to object.
   *
   * <p>Note that a False response indicates data has been written into the 'errors' map and should
   * be followed by an 'errorAndClose()' call.
   *
   * @return: boolean: False if any errors occurred, True otherwise
   */
  private boolean compileMetadata(DigitalObject object) {
    // Nothing to do
    if (info == null) {
      return true;
    }
    if (!info.isSupported()) {
      return true;
    }

    // Get base metadata of source
    JsonConfigHelper fullMetadata = null;
    try {
      fullMetadata = new JsonConfigHelper(info.toString());
    } catch (IOException ex) {
      addError("metadata", "Error parsing metadata output", ex);
      return false;
    }

    // Add individual conversion(s) metadata
    if (!metadata.isEmpty()) {
      fullMetadata.setJsonMap("outputs", metadata);
    }

    // Write the file to disk
    File metaFile;
    try {
      // log.debug("\nMetadata:\n{}", fullMetadata.toString());
      metaFile = writeMetadata(fullMetadata.toString());
      if (metaFile == null) {
        addError("metadata", "Unknown error extracting metadata");
        return false;
      }
    } catch (TransformerException ex) {
      addError("metadata", "Error writing metadata to disk", ex);
      return false;
    }

    // Store metadata
    try {
      Payload payload = createFfmpegPayload(object, metaFile);
      payload.setType(PayloadType.Enrichment);
      payload.close();
    } catch (Exception ex) {
      addError("metadata", "Error storing metadata payload", ex);
      return false;
    } finally {
      metaFile.delete();
    }

    // Everything should be fine if we got here
    return true;
  }
 /**
  * Create ffmpeg error payload
  *
  * @param object : DigitalObject to store the payload
  * @return Payload the error payload
  * @throws FileNotFoundException if the file provided does not exist
  * @throws UnsupportedEncodingException for encoding errors in the message
  */
 public Payload createFfmpegErrorPayload(DigitalObject object)
     throws StorageException, FileNotFoundException, UnsupportedEncodingException {
   // Compile our error data
   JsonConfigHelper content = new JsonConfigHelper();
   content.setJsonMap("/", errors);
   log.debug("\nErrors:\n{}", content.toString());
   InputStream data = new ByteArrayInputStream(content.toString().getBytes("UTF-8"));
   // Write to the object
   Payload payload = StorageUtils.createOrUpdatePayload(object, ERROR_PAYLOAD, data);
   payload.setType(PayloadType.Error);
   payload.setContentType("application/json");
   payload.setLabel("FFMPEG conversion errors");
   return payload;
 }
 /**
  * Read FFMPEG metadata from the object if it exists
  *
  * @param object: The object to extract data from
  */
 private void readMetadata(DigitalObject object) {
   Set<String> pids = object.getPayloadIdList();
   if (pids.contains(METADATA_PAYLOAD)) {
     try {
       Payload payload = object.getPayload(METADATA_PAYLOAD);
       JsonConfigHelper data = new JsonConfigHelper(payload.open());
       payload.close();
       oldMetadata = data.getJsonMap("outputs");
       // for (String k : oldMetadata.keySet()) {
       // log.debug("\n====\n{}\n===\n{}", k, oldMetadata.get(k));
       // }
     } catch (IOException ex) {
       log.error("Error parsing metadata JSON: ", ex);
     } catch (StorageException ex) {
       log.error("Error accessing metadata payload: ", ex);
     }
   }
 }
  @Test
  public void addAndGetObject1() throws Exception {
    fs.addObject(newObject);
    FileSystemDigitalObject addedObject =
        (FileSystemDigitalObject) fs.getObject("oai:eprints.usq.edu.au:318");
    Assert.assertEquals(
        FilenameUtils.normalize(
            tmpDir
                + "/_fs_test/d0b1c5bd0660ad67a16b7111aafc9389/"
                + "e2/92/e292378c5b38b0d5a4aba11fd40e7151"),
        addedObject.getPath().getAbsolutePath());

    List<Payload> payloads = addedObject.getPayloadList();
    Assert.assertEquals(1, payloads.size());

    Payload payload = payloads.get(0);
    Assert.assertEquals("Dublin Core Metadata", payload.getLabel());
  }
 /**
  * Stream data from specified payload into a file in our temp cache.
  *
  * @param object: The digital object to use
  * @param pid: The payload ID to extract
  * @return File: The cached File
  * @throws FileNotFoundException: If accessing the cache fails
  * @throws StorageException: If accessing the object in storage fails
  * @throws IOException: If the data copy fails
  */
 private File cacheFile(DigitalObject object, String pid)
     throws FileNotFoundException, StorageException, IOException {
   // Get our cache location
   File file = new File(outputDir, pid);
   FileOutputStream tempFileOut = new FileOutputStream(file);
   // Get payload from storage
   Payload payload = object.getPayload(pid);
   try {
     // Copy to cache
     IOUtils.copy(payload.open(), tempFileOut);
   } catch (IOException ex) {
     payload.close();
     throw ex;
   }
   // Close and return
   payload.close();
   tempFileOut.close();
   return file;
 }
  @Test
  public void addAndGetObject2() throws Exception {
    fs.addObject(fileObject);
    FileSystemDigitalObject addedObject =
        (FileSystemDigitalObject) fs.getObject("/Users/fascinator/Documents/sample.odt");
    Assert.assertEquals(
        FilenameUtils.normalize(
            tmpDir
                + "/_fs_test/d0b1c5bd0660ad67a16b7111aafc9389/"
                + "11/b4/11b498d057256a0b602fa0e7c4073fc3"),
        addedObject.getPath().getAbsolutePath());

    List<Payload> payloads = addedObject.getPayloadList();
    Assert.assertEquals(2, payloads.size());

    Payload payload1 = addedObject.getPayload("sample.odt");
    Assert.assertEquals("ICE Sample Document", payload1.getLabel());

    Payload payload2 = addedObject.getPayload("images/ice-services.png");
    Assert.assertEquals("ICE Services Diagram", payload2.getLabel());
  }
  @Test
  public void transformAudio() throws Exception {
    Transformer t = new FfmpegTransformer(ffmpeg);
    t.init(new File(getClass().getResource("/ffmpeg-config.json").toURI()));
    DigitalObject outputObject = t.transform(sourceObject);

    // should have 2 payloads
    Assert.assertEquals("There should be 2 Payloads", 2, outputObject.getPayloadIdList().size());

    String preview = null;
    for (String i : outputObject.getPayloadIdList()) {
      Payload p = outputObject.getPayload(i);
      if (p.getType() == PayloadType.Preview) {
        preview = i;
      }
    }

    // should have a preview payload
    Assert.assertNotNull("Should have a Preview", preview);
    outputObject.close();
  }
 public void addPayload(String oid, Payload payload) {
   log.debug("Adding payload {} to {}", payload.getId(), oid);
   FileSystemDigitalObject fileObject = (FileSystemDigitalObject) getObject(oid);
   FileSystemPayload filePayload = new FileSystemPayload(fileObject.getPath(), payload);
   File payloadFile = filePayload.getFile();
   File parentDir = payloadFile.getParentFile();
   parentDir.mkdirs();
   try {
     FileOutputStream out = new FileOutputStream(payloadFile);
     IOUtils.copy(filePayload.getInputStream(), out);
     out.close();
   } catch (IOException ioe) {
     log.error("Failed to add payload", ioe);
   }
 }
  /**
   * Check the object for a multi-segment source and merge them. Such sources must come from a
   * harvester specifically designed to match this transformer. As such we can make certain
   * assumptions, and if they are not met we just fail silently with a log entry.
   *
   * @param object: The digital object to modify
   */
  private void mergeSegments(DigitalObject object) {
    try {
      // Retrieve (optional) segment information from metadata
      Properties props = object.getMetadata();
      String segs = props.getProperty("mediaSegments");
      if (segs == null) {
        return;
      }
      int segments = Integer.parseInt(segs);
      if (segments <= 1) {
        return;
      }

      // We need to do some merging, lets validate IDs first
      log.info("Found {} source segments! Merging...", segments);
      List<String> segmentIds = new ArrayList();
      Set<String> payloadIds = object.getPayloadIdList();
      // The first segment
      String sourceId = object.getSourceId();
      if (sourceId == null || !payloadIds.contains(sourceId)) {
        log.error("Cannot find source payload.");
        return;
      }
      segmentIds.add(sourceId);
      // Find the other segments
      for (int i = 1; i < segments; i++) {
        // We won't know the extension though
        String segmentId = "segment" + i + ".";
        for (String pid : payloadIds) {
          if (pid.startsWith(segmentId)) {
            segmentIds.add(pid);
          }
        }
      }

      // Did we find every segment?
      if (segmentIds.size() != segments) {
        log.error("Unable to find all segments in payload list.");
        return;
      }

      // Transcode all the files to neutral MPEGs first
      Map<String, File> files = new HashMap();
      for (String segment : segmentIds) {
        try {
          File file = basicMpeg(object, segment);
          if (file != null) {
            files.put(segment, file);
          }
        } catch (Exception ex) {
          log.error("Error transcoding segment to MPEG: ", ex);
          // Cleanup
          for (File f : files.values()) {
            if (f.exists()) {
              f.delete();
            }
          }
          return;
        }
      }

      // Did every transcoding succeed?
      if (files.size() != segments) {
        log.error("At least one segment transcoding failed.");
        // Cleanup
        for (File f : files.values()) {
          if (f.exists()) {
            f.delete();
          }
        }
        return;
      }

      // Now to try merging all the segments. In MPEG format
      // they can just be concatenated.
      try {
        // Create our output file
        String filename = "temp_" + MERGED_PAYLOAD + "mpg";
        File merged = new File(outputDir, filename);
        if (merged.exists()) {
          merged.delete();
        }
        FileOutputStream out = new FileOutputStream(merged);

        // Merge each segment in order
        for (String sId : segmentIds) {
          try {
            mergeSegment(out, files.get(sId));
          } catch (IOException ex) {
            log.error("Failed to stream to merged file: ", ex);
            out.close();
            // Cleanup
            for (File f : files.values()) {
              if (f.exists()) {
                f.delete();
              }
            }
            merged.delete();
            return;
          }
        }
        out.close();

        // Final step, run the output file through a transcoding to
        // write the correct metadata (eg. duration)
        filename = MERGED_PAYLOAD + finalFormat;
        File transcoded = new File(outputDir, filename);
        if (transcoded.exists()) {
          transcoded.delete();
        }

        // Render
        String stderr = mergeRender(merged, transcoded);
        log.debug("=====\n{}", stderr);
        if (transcoded.exists()) {
          // Now we need to 'fix' the object, add the new source
          FileInputStream fis = new FileInputStream(transcoded);
          String pid = transcoded.getName();
          Payload p = StorageUtils.createOrUpdatePayload(object, pid, fis);
          fis.close();
          p.setType(PayloadType.Source);
          object.setSourceId(pid);

          // Remove all the old segments
          for (String sId : segmentIds) {
            object.removePayload(sId);
          }
          props.remove("mediaSegments");
          object.close();

          // Cleanup segments
          for (File f : files.values()) {
            if (f.exists()) {
              f.delete();
            }
          }
          merged.delete();
          transcoded.delete();
        }
      } catch (IOException ex) {
        log.error("Error merging segments: ", ex);
      }

    } catch (StorageException ex) {
      log.error("Error accessing object metadata: ", ex);
    }
  }
  /**
   * Transforming digital object method
   *
   * @params object: DigitalObject to be transformed
   * @return transformed DigitalObject after transformation
   * @throws TransformerException if the transformation fails
   */
  @Override
  public DigitalObject transform(DigitalObject object, String jsonConfig)
      throws TransformerException {
    if (testExecLevel() == null) {
      log.error("FFmpeg is either not installed, or not executing!");
      return object;
    }
    // Purge old data
    reset();
    oid = object.getId();
    outputDir = new File(outputRoot, oid);
    outputDir.mkdirs();

    try {
      itemConfig = new JsonConfigHelper(jsonConfig);
    } catch (IOException ex) {
      throw new TransformerException("Invalid configuration! '{}'", ex);
    }

    // Resolve multi-segment files first
    mergeRate = get(itemConfig, "merging/mpegFrameRate", "25");
    finalRate = get(itemConfig, "merging/finalFrameRate", "10");
    finalFormat = get(itemConfig, "merging/finalFormat", "avi");
    mergeSegments(object);

    // Find the format 'group' this file is in
    String sourceId = object.getSourceId();
    String ext = FilenameUtils.getExtension(sourceId);
    format = getFormat(ext);

    // Return now if this isn't a format we care about
    if (format == null) {
      return object;
    }
    // log.debug("Supported format found: '{}' => '{}'", ext, format);

    // Cache the file from storage
    File file;
    try {
      file = cacheFile(object, sourceId);
    } catch (IOException ex) {
      addError(sourceId, "Error writing temp file", ex);
      errorAndClose(object);
      return object;
    } catch (StorageException ex) {
      addError(sourceId, "Error accessing storage data", ex);
      errorAndClose(object);
      return object;
    }
    if (!file.exists()) {
      addError(sourceId, "Unknown error writing cache: does not exist");
      errorAndClose(object);
      return object;
    }

    // **************************************************************
    // From here on we know (assume) that we SHOULD be able to support
    // this object, so errors can't just throw exceptions. We should
    // only return under certain circumstances (ie. not just because
    // one rendition fails), and the object must get closed.
    // **************************************************************

    // Read any pre-existing rendition metadata from previous tranformations
    // ++++++++++++++++++++++++++++
    // TODO: This is useless until the last modified date can be retrieved
    // against the source file. Storage API does not currently support this,
    // it just returns a data stream.
    //
    // Once this feature exists the basic algorithm should be:
    // 1) Retrieve old metadata
    // 2) Loop through each rendition preparation as normal
    // 3) When the transcoding is ready to start, use the parameters to
    // query the database for the last time the exact transcoding was
    // and comparing against last modifed.
    // 4) If the transcoding is newer than the source, skip running FFmpeg
    // and just use the same metadata as last time.
    // ++++++++++++++++++++++++++++
    // readMetadata(object);

    // Check for a custom display type
    String display = get(itemConfig, "displayTypes/" + format);
    if (display != null) {
      try {
        Properties prop = object.getMetadata();
        prop.setProperty("displayType", display);
        prop.setProperty("previewType", display);
      } catch (StorageException ex) {
        addError("display", "Could not access object metadata", ex);
      }
    }

    // Gather metadata
    try {
      info = ffmpeg.getInfo(file);
    } catch (IOException ex) {
      addError("metadata", "Error accessing metadata", ex);
      errorAndClose(object);
      return object;
    }

    // Can we even process this file?
    if (!info.isSupported()) {
      closeObject(object);
      return object;
    }

    // What conversions are required for this format?
    List<JsonConfigHelper> conversions = getJsonList(itemConfig, "transcodings/" + format);
    for (JsonConfigHelper conversion : conversions) {
      String name = conversion.get("alias");
      // And what/how many renditions does it have?
      List<JsonConfigHelper> renditions = conversion.getJsonList("renditions");
      if (renditions == null || renditions.isEmpty()) {
        addError(
            "transcodings",
            "Invalid or missing transcoding data:" + " '/transcodings/" + format + "'");
      } else {
        // Config look valid, lets give it a try
        // log.debug("Starting renditions for '{}'", name);
        for (JsonConfigHelper render : renditions) {
          File converted = null;
          // Render the output
          try {
            converted = convert(file, render, info);
          } catch (Exception ex) {
            String outputFile = render.get("name");
            if (outputFile != null) {
              addError(jsonKey(outputFile), "Error converting file", ex);
            } else {
              // Couldn't read the config for a name
              addError("unknown", "Error converting file", ex);
            }
          }

          // Now store the output if valid
          if (converted != null) {
            try {
              Payload payload = createFfmpegPayload(object, converted);
              // TODO: Type checking needs more work
              // Indexing fails silently if you add two thumbnails
              // or two previews
              payload.setType(resolveType(render.get("type")));
              payload.close();
            } catch (Exception ex) {
              addError(jsonKey(converted.getName()), "Error storing output", ex);
            } finally {
              converted.delete();
            }
          }
        }
      }
    }

    // Write metadata to storage
    if (compileMetadata(object)) {
      // Close normally
      closeObject(object);
    } else {
      // Close with some errors
      errorAndClose(object);
    }
    // Cleanup
    if (file.exists()) {
      file.delete();
    }
    return object;
  }