/**
   * Store the processed data and metadata in the system
   *
   * @param dataJson an instantiated JSON object containing data to store
   * @param metaJson an instantiated JSON object containing metadata to store
   * @throws HarvesterException if an error occurs
   */
  private void storeJsonInObject(JsonObject dataJson, JsonObject metaJson, String oid)
      throws HarvesterException {
    // Does the object already exist?
    DigitalObject object = null;
    try {
      object = getStorage().getObject(oid);
      boolean isModified = isModifiedRecord(oid, dataJson);
      storeJsonInPayload(dataJson, metaJson, object);
      object.getMetadata().setProperty("isModified", Boolean.toString(isModified));

    } catch (StorageException ex) {
      // This is going to be brand new
      try {
        object = StorageUtils.getDigitalObject(getStorage(), oid);
        storeJsonInPayload(dataJson, metaJson, object);
        // newRecordCount++;
        object.getMetadata().setProperty("isNew", "true");
      } catch (StorageException ex2) {
        throw new HarvesterException("Error creating new digital object: ", ex2);
      }
    }

    // Set the pending flag
    if (object != null) {
      try {
        object.getMetadata().setProperty("render-pending", "true");
        object.close();
      } catch (Exception ex) {
        log.error("Error setting 'render-pending' flag: ", ex);
      }
    }
  }
 /**
  * Create Payload method for ICE Error
  *
  * @param object : DigitalObject that store the payload
  * @param file : File to be stored as payload
  * @param message : Error message
  * @return transformed DigitalObject
  * @throws StorageException
  * @throws UnsupportedEncodingException
  */
 public DigitalObject createErrorPayload(DigitalObject object, String file, Exception ex)
     throws StorageException, UnsupportedEncodingException {
   String name = file + "_ice_error.htm";
   String message = ex.getMessage();
   if (message == null) {
     message = ex.toString();
   }
   Payload errorPayload =
       StorageUtils.createOrUpdatePayload(
           object, name, new ByteArrayInputStream(message.getBytes("UTF-8")));
   errorPayload.setType(PayloadType.Error);
   errorPayload.setLabel("ICE conversion errors");
   errorPayload.setContentType("text/html");
   return object;
 }
  /**
   * Create Payload method for ICE rendition files
   *
   * @param object : DigitalObject that store the payload
   * @param file : File to be stored as payload
   * @return transformed DigitalObject
   * @throws StorageException
   * @throws IOException
   */
  public DigitalObject createIcePayload(DigitalObject object, File file)
      throws StorageException, IOException, Exception {
    if (ZIP_MIME_TYPE.equals(MimeTypeUtil.getMimeType(file))) {
      ZipFile zipFile = new ZipFile(file);
      Enumeration<? extends ZipEntry> entries = zipFile.entries();
      while (entries.hasMoreElements()) {
        ZipEntry entry = entries.nextElement();
        if (!entry.isDirectory()) {
          String name = entry.getName();
          String mimeType = MimeTypeUtil.getMimeType(name);
          // log.info("(ZIP) Name : '" + name + "', MimeType : '" +
          // mimeType + "'");
          InputStream in = zipFile.getInputStream(entry);

          // If this is a HTML document we need to strip it down
          // to the 'body' tag and replace with a 'div'
          if (mimeType.equals(HTML_MIME_TYPE)) {
            try {
              log.debug("Stripping unnecessary HTML");
              // Parse the document
              Document doc = reader.loadDocumentFromStream(in);
              // Alter the body node
              Node node = doc.selectSingleNode("//*[local-name()='body']");
              node.setName("div");
              // Write out the new 'document'
              ByteArrayOutputStream out = new ByteArrayOutputStream();
              reader.docToStream(node, out);
              // Prep our inputstream again
              in = new ByteArrayInputStream(out.toByteArray());
            } catch (DocumentException ex) {
              createErrorPayload(object, name, ex);
              continue;
            } catch (Exception ex) {
              log.error("Error : ", ex);
              continue;
            }
          }

          // Determing the payload type to use in storage
          PayloadType pt = null;
          try {
            pt = assignType(object, name, mimeType);
          } catch (TransformerException ex) {
            throw new Exception("Error examining object to assign type: ", ex);
          }
          if (pt == null) {
            // We're done, this file is not being stored
            return object;
          }

          Payload icePayload = StorageUtils.createOrUpdatePayload(object, name, in);
          icePayload.setLabel(name);
          icePayload.setContentType(mimeType);
          icePayload.setType(pt);
          icePayload.close();
        }
      }
      zipFile.close();
    } else {
      String name = file.getName();
      String mimeType = MimeTypeUtil.getMimeType(name);
      // Determing the payload type to use in storage
      PayloadType pt = null;
      try {
        pt = assignType(object, name, mimeType);
      } catch (TransformerException ex) {
        throw new Exception("Error examining object to assign type: ", ex);
      }
      if (pt == null) {
        // We're done, this file is not being stored
        return object;
      }

      Payload icePayload =
          StorageUtils.createOrUpdatePayload(object, name, new FileInputStream(file));
      icePayload.setLabel(name);
      icePayload.setContentType(mimeType);
      icePayload.setType(pt);
      icePayload.close();
    }

    return object;
  }