/**
  * Creates a filter from a JSON object
  *
  * @param json The json to extract "field","operation" (optional) and "regex" from
  * @throws HarvesterException if
  *     <ul>
  *       <li>field is missing
  *       <li>operation is not a valid operation
  *       <li>regex is missing or not a valid regex string
  *     </ul>
  */
 public Filter(JsonSimple json) throws HarvesterException {
   field = json.getString(null, "field");
   if (field == null) {
     throw new HarvesterException(
         "In a filter definition, missing the mandatory attribute 'field'");
   }
   String matchTypeStr = json.getString("ANY", "multi");
   try {
     type = MultiMatchType.valueOf(matchTypeStr);
   } catch (Exception e) {
     throw new HarvesterException(
         "In a filter definition, invalid filter match type '"
             + matchTypeStr
             + "', valid values are "
             + MultiMatchType.values());
   }
   String regexStr = json.getString(null, "regex");
   if (regexStr == null) {
     throw new HarvesterException(
         "In a filter definition, missing the mandatory attribute 'regex'");
   }
   try {
     regex = Pattern.compile(regexStr, Pattern.MULTILINE);
   } catch (PatternSyntaxException e) {
     throw new HarvesterException(
         "In a filter definition, provided regex was invalid " + e.getMessage(), e);
   }
 }
  /**
   * Merge the newly processed data with an (possible) existing data already present, also convert
   * the completed JSON merge into a Stream for storage.
   *
   * @param dataJson an instantiated JSON object containing data to store
   * @param metaJson an instantiated JSON object containing metadata to store
   * @param existing an instantiated JsonSimple object with any existing data
   * @throws IOException if any character encoding issues effect the Stream
   */
  private InputStream streamMergedJson(
      JsonObject dataJson, JsonObject metaJson, JsonSimple existing) throws IOException {
    // Overwrite and/or create only nodes we consider new data
    existing.getJsonObject().put("recordIDPrefix", idPrefix);
    JsonObject existingData = existing.writeObject("data");
    existingData.putAll(dataJson);
    JsonObject existingMeta = existing.writeObject("metadata");
    existingMeta.putAll(metaJson);

    // Turn into a stream to return
    String jsonString = existing.toString(true);
    return IOUtils.toInputStream(jsonString, "UTF-8");
  }
 /**
  * Gets a string list from a JsonSimple object. Convenience method to return an empty list instead
  * of null if the node was not found.
  *
  * @param json a JsonSimple object
  * @param path path to the node
  * @return string list found at node, or empty if not found
  */
 private List<String> getStringList(JsonSimple json, Object... path) {
   List<String> list = json.getStringList(path);
   if (list == null) {
     list = Collections.emptyList();
   }
   return list;
 }
  /** Basic constructor, should be run automatically by Tapestry. */
  public PortalSecurityManagerImpl() throws IOException {
    // Get system configuration
    config = new JsonSimpleConfig();

    // For all SSO providers configured
    sso = new LinkedHashMap<String, SSOInterface>();
    for (String ssoId : config.getStringList("sso", "plugins")) {
      // Instantiate from the ServiceLoader
      SSOInterface valid = getSSOProvider(ssoId);
      if (valid == null) {
        log.error("Invalid SSO Implementation requested: '{}'", ssoId);
      } else {
        // Store valid implementations
        sso.put(ssoId, valid);
        log.info("SSO Provider instantiated: '{}'", ssoId);
      }
    }

    defaultPortal = config.getString(PortalManager.DEFAULT_PORTAL_NAME, "portal", "defaultView");
    serverUrlBase = config.getString(null, "urlBase");
    ssoLoginUrl = serverUrlBase + defaultPortal + SSO_LOGIN_PAGE;

    // Get exclusions Strings from config
    excStarts = config.getStringList("sso", "urlExclusions", "startsWith");
    excEnds = config.getStringList("sso", "urlExclusions", "endsWith");
    excEquals = config.getStringList("sso", "urlExclusions", "equals");

    // Trust tokens
    Map<String, JsonSimple> tokenMap = config.getJsonSimpleMap("sso", "trustTokens");
    tokens = new HashMap<String, String>();
    tokenExpiry = new HashMap<String, Long>();
    for (String key : tokenMap.keySet()) {
      JsonSimple tok = tokenMap.get(key);
      String publicKey = tok.getString(null, "publicKey");
      String privateKey = tok.getString(null, "privateKey");
      String expiry = tok.getString(TRUST_TOKEN_EXPIRY, "expiry");
      if (publicKey != null && privateKey != null) {
        // Valid key
        tokens.put(publicKey, privateKey);
        tokenExpiry.put(publicKey, Long.valueOf(expiry));
      } else {
        log.error("Invalid token data: '{}'", key);
      }
    }
  }
Ejemplo n.º 5
0
  /** Reset the transformer in preparation for a new object */
  private void reset() throws TransformerException {
    if (firstRun) {
      firstRun = false;
      // Output directory
      String outputPath = config.getString(null, "outputPath");
      if (outputPath == null) {
        throw new TransformerException("Output path not specified!");
      }
      outputDir = new File(outputPath);
      outputDir.mkdirs();

      // Rendition exclusions
      excludeList =
          Arrays.asList(StringUtils.split(config.getString(null, "excludeRenditionExt"), ','));

      // Conversion Service URL
      convertUrl = config.getString(null, "url");
      if (convertUrl == null) {
        throw new TransformerException("No ICE URL provided!");
      }
    }

    // Priority
    Boolean testResponse = itemConfig.getBoolean(null, "priority");
    if (testResponse != null) {
      // We found it in item config
      priority = testResponse;
    } else {
      // Try system config
      priority = config.getBoolean(true, "priority");
    }

    // Clear the old SAX reader
    reader = new SafeSAXReader();

    // Remove the last object
    thumbnails = null;
    previews = null;
  }
Ejemplo n.º 6
0
  /**
   * Main render method to send the file to ICE service
   *
   * @param sourceFile : File to be rendered
   * @return file returned by ICE service
   * @throws TransformerException
   */
  private File render(File sourceFile) throws TransformerException {
    log.info("Converting {}...", sourceFile);
    String filename = sourceFile.getName();
    String basename = FilenameUtils.getBaseName(filename);
    String ext = FilenameUtils.getExtension(filename);
    int status = HttpStatus.SC_OK;

    // Grab our config
    JsonObject object = itemConfig.getObject("resize");
    Map<String, JsonSimple> resizeConfig = JsonSimple.toJavaMap(object);

    if (resizeConfig == null || resizeConfig.isEmpty()) {
      // Try system config instead
      object = config.getObject("resize");
      resizeConfig = JsonSimple.toJavaMap(object);
      if (resizeConfig == null || resizeConfig.isEmpty()) {
        throw new TransformerException("No resizing configuration found.");
      }
    }

    String resizeJson = "";
    for (String key : resizeConfig.keySet()) {
      JsonSimple j = resizeConfig.get(key);
      resizeJson += "\"" + key + "\":" + j.toString() + ",";
    }

    PostMethod post = new PostMethod(convertUrl);
    try {
      Part[] parts = {
        new StringPart("zip", "on"),
        new StringPart("dc", "on"),
        new StringPart("toc", "on"),
        new StringPart("pdflink", "on"),
        new StringPart("addThumbnail", "on"),
        new StringPart("pathext", ext),
        new StringPart("template", getTemplate()),
        new StringPart(
            "multipleImageOptions", "{" + StringUtils.substringBeforeLast(resizeJson, ",") + "}"),
        new StringPart("mode", "download"),
        new FilePart("file", sourceFile)
      };
      post.setRequestEntity(new MultipartRequestEntity(parts, post.getParams()));
      BasicHttpClient client = new BasicHttpClient(convertUrl);
      log.debug("Using conversion URL: {}", convertUrl);
      status = client.executeMethod(post);
      log.debug("HTTP status: {} {}", status, HttpStatus.getStatusText(status));
    } catch (IOException ioe) {
      throw new TransformerException("Failed to send ICE conversion request", ioe);
    }
    try {
      if (status != HttpStatus.SC_OK) {
        String xmlError = post.getResponseBodyAsString();
        log.debug("Error: {}", xmlError);
        throw new TransformerException(xmlError);
      }
      String type = post.getResponseHeader("Content-Type").getValue();
      if ("application/zip".equals(type)) {
        filename = basename + ".zip";
      } else if (type.startsWith("image/")) {
        filename = basename + "_thumbnail.jpg";
      } else if ("video/x-flv".equals(type)) {
        filename = basename + ".flv";
      } else if ("audio/mpeg".equals(type)) {
        filename = basename + ".mp3";
      }
      File outputFile = new File(outputDir, filename);
      if (outputFile.exists()) {
        outputFile.delete();
      }
      InputStream in = post.getResponseBodyAsStream();
      FileOutputStream out = new FileOutputStream(outputFile);
      IOUtils.copy(in, out);
      in.close();
      out.close();
      log.debug("ICE output file: {}", outputFile);
      return outputFile;
    } catch (IOException ioe) {
      throw new TransformerException("Failed to process ICE output", ioe);
    }
  }
  /**
   * Initialise the CSV harvester plugin.
   *
   * @throws HarvesterException if an error occurred
   */
  @Override
  public void init() throws HarvesterException {
    JsonSimple options = new JsonSimple(getJsonConfig().getObject("harvester", "csv"));

    String filePath = options.getString(null, "fileLocation");
    if (filePath == null) {
      throw new HarvesterException("No data file provided!");
    }
    File csvDataFile = new File(filePath);
    if (csvDataFile == null || !csvDataFile.exists()) {
      throw new HarvesterException("Could not find CSV file '" + filePath + "'");
    }
    filename = csvDataFile.getName();

    idPrefix = options.getString("", "recordIDPrefix");
    maxRows = options.getInteger(-1, "maxRows");
    delimiter = options.getString(String.valueOf(DEFAULT_DELIMITER), "delimiter").charAt(0);
    ignoredFields = getStringList(options, "ignoreFields");
    includedFields = getStringList(options, "includedFields");
    multiValueFields = getStringList(options, "multiValueFields");
    multiValueFieldDelimiter =
        options
            .getString(
                String.valueOf(DEFAULT_MULTI_VALUE_FIELD_DELIMITER), "multiValueFieldDelimiter")
            .charAt(0);
    payloadId = options.getString(DEFAULT_PAYLOAD_ID, "payloadId");
    batchSize = options.getInteger(DEFAULT_BATCH_SIZE, "batchSize");
    hasMore = true;

    if (delimiter == multiValueFieldDelimiter) {
      throw new HarvesterException(
          "Cannot parse CSV: The requested delimiters for the CSV and multivalue fields are the same: "
              + delimiter);
    }

    try {
      // open the CSV file for reading
      Reader fileReader = new InputStreamReader(new FileInputStream(csvDataFile), "UTF-8");
      // char delimiter = options.getString(String.valueOf(DEFAULT_DELIMITER),
      // "delimiter").charAt(0);
      csvReader = new CSVReader(fileReader, delimiter);

      // configure the data fields
      if (options.getBoolean(true, "headerRow")) {
        dataFields = Arrays.asList(csvReader.readNext());
      } else {
        dataFields = getStringList(options, "headerList");
      }

      // check that the specified id column is valid
      idColumn = options.getString(null, "idColumn");
      if (idColumn != null && !dataFields.contains(idColumn)) {
        throw new HarvesterException(
            "ID column '" + idColumn + "' was invalid or not found in the data!");
      }

      // load filters, all filters must pass for the row to be considered
      filters = new HashMap<String, List<Filter>>();
      List<JsonSimple> filterConfig = options.getJsonSimpleList("filters");
      if (filterConfig != null) {
        for (JsonSimple singleFilterConfig : filterConfig) {
          Filter filter = new Filter(singleFilterConfig);
          String field = filter.getField();
          if (!dataFields.contains(field)) {
            throw new HarvesterException("Filter column '" + field + "' was not found in the data");
          }
          List<Filter> existingFilters = filters.get(field);
          if (existingFilters == null) {
            existingFilters = new ArrayList<Filter>();
            filters.put(field, existingFilters);
          }
          existingFilters.add(filter);
        }
      }

    } catch (IOException ioe) {
      throw new HarvesterException(ioe);
    }
  }