예제 #1
0
  /**
   * Configure CB4J record reader.
   *
   * @throws BatchConfigurationException thrown if record reader is not correctly configured
   */
  private void configureRecordReader() throws BatchConfigurationException {

    String inputDataProperty = configurationProperties.getProperty(BatchConstants.INPUT_DATA_PATH);
    String encodingProperty =
        configurationProperties.getProperty(BatchConstants.INPUT_DATA_ENCODING);
    String skipHeaderProperty =
        configurationProperties.getProperty(BatchConstants.INPUT_DATA_SKIP_HEADER);

    // check if input data file is specified
    if (inputDataProperty == null) {
      String error = "Configuration failed : input data file is mandatory but was not specified";
      logger.severe(error);
      throw new BatchConfigurationException(error);
    }

    try {

      boolean skipHeader;
      if (skipHeaderProperty != null) {
        skipHeader = Boolean.valueOf(skipHeaderProperty);
      } else {
        skipHeader = BatchConstants.DEFAULT_SKIP_HEADER;
        logger.info("Skip header property not specified, default to false");
      }

      String encoding;
      if (encodingProperty == null || (encodingProperty.length() == 0)) {
        encoding = BatchConstants.DEFAULT_FILE_ENCODING;
        logger.info(
            "No encoding specified for input data, using system default encoding : " + encoding);
      } else {
        if (Charset.availableCharsets().get(Charset.forName(encodingProperty).name()) == null
            || !Charset.isSupported(encodingProperty)) {
          encoding = BatchConstants.DEFAULT_FILE_ENCODING;
          logger.warning(
              "Encoding '"
                  + encodingProperty
                  + "' not supported, using system default encoding : "
                  + encoding);
        } else {
          encoding = encodingProperty;
          logger.info("Using '" + encoding + "' encoding for input file reading");
        }
      }
      recordReader = new RecordReaderImpl(inputDataProperty, encoding, skipHeader);
      logger.info("Data input file : " + inputDataProperty);
    } catch (FileNotFoundException e) {
      String error =
          "Configuration failed : input data file '" + inputDataProperty + "' could not be opened";
      logger.severe(error);
      throw new BatchConfigurationException(error);
    }
  }
예제 #2
0
  /**
   * Configure the default record mapper.
   *
   * @return the default implementation of record mapper
   * @throws BatchConfigurationException if the target type class is not found
   */
  private RecordMapper configureDefaultRecordMapper() throws BatchConfigurationException {

    RecordMapper recordMapper;

    String recordClassName = configurationProperties.getProperty(BatchConstants.INPUT_RECORD_CLASS);
    if (recordClassName == null || recordClassName.length() == 0) {
      try {
        Class recordProcessorClass = Class.forName(recordProcessor.getClass().getName());
        Method[] declaredMethods = recordProcessorClass.getDeclaredMethods();
        for (Method declaredMethod : declaredMethods) {
          if (declaredMethod.getName().equals("processRecord")) {
            recordClassName = declaredMethod.getParameterTypes()[0].getName();
            break;
          }
        }
      } catch (ClassNotFoundException e) {
        String error =
            "Configuration failed : unable to get record class name from registered record processor implementation.";
        logger.severe(error);
        throw new BatchConfigurationException(error, e);
      }
    }

    String[] headers;
    String headersProperty =
        configurationProperties.getProperty(BatchConstants.INPUT_RECORD_HEADERS);
    if (headersProperty
        == null) { // if no headers specified, use field names declared in the header record
      String headerRecord = recordReader.getHeaderRecord();
      Record record =
          recordParser.parseRecord(
              headerRecord,
              0); // use the record parser to parse the header record using the right delimiter
      List<Field> fields = record.getFields();
      headers = new String[fields.size()];
      for (int i = 0; i < fields.size(); i++) {
        headers[i] = fields.get(i).getContent();
      }
    } else { // headers specified, split the comma separated list
      headers = headersProperty.split(",");
    }

    try {
      recordMapper = new DefaultRecordMapperImpl(recordClassName, headers, typeConverters);
    } catch (ClassNotFoundException e) {
      String error = "Configuration failed : Class " + recordClassName + " not found.";
      logger.severe(error);
      throw new BatchConfigurationException(error, e);
    }

    return recordMapper;
  }
예제 #3
0
 public boolean getJmxEnabled() {
   return Boolean.valueOf(
       configurationProperties.getProperty(BatchConstants.OUTPUT_DATA_JMX_ENABLED));
 }
예제 #4
0
 public boolean getSkipHeader() {
   return Boolean.valueOf(
       configurationProperties.getProperty(BatchConstants.INPUT_DATA_SKIP_HEADER));
 }
예제 #5
0
 public boolean getAbortOnFirstMappingException() {
   return Boolean.valueOf(
       configurationProperties.getProperty(
           BatchConstants.OUTPUT_DATA_ABORT_ON_FIRST_MAPPING_EXCEPTION));
 }
예제 #6
0
 public boolean getAbortOnFirstError() {
   return Boolean.valueOf(
       configurationProperties.getProperty(BatchConstants.OUTPUT_DATA_ABORT_ON_FIRST_ERROR));
 }
예제 #7
0
 public boolean getAbortOnFirstReject() {
   return Boolean.valueOf(
       configurationProperties.getProperty(BatchConstants.OUTPUT_DATA_ABORT_ON_FIRST_REJECT));
 }
예제 #8
0
  /**
   * Configure CB4J record parser.
   *
   * @throws BatchConfigurationException thrown if record parser is not correctly configured
   */
  private void configureRecordParser() throws BatchConfigurationException {

    // read record type property and set default value if invalid input
    String recordTypeProperty =
        configurationProperties.getProperty(BatchConstants.INPUT_RECORD_TYPE);
    String recordType;
    if (recordTypeProperty == null || recordTypeProperty.length() == 0) {
      recordType = BatchConstants.DEFAULT_RECORD_TYPE;
      logger.info(
          "Record type property not specified, records will be considered as delimiter-separated values");
    } else if (!RecordType.DSV.toString().equalsIgnoreCase(recordTypeProperty)
        && !RecordType.FLR.toString().equalsIgnoreCase(recordTypeProperty)) {
      recordType = BatchConstants.DEFAULT_RECORD_TYPE;
      logger.warning(
          "Record type property '"
              + recordTypeProperty
              + "' is invalid, records will be considered as delimiter-separated values");
    } else {
      recordType = recordTypeProperty;
    }

    // fixed length record configuration
    if (RecordType.FLR.toString().equalsIgnoreCase(recordType)) {
      String fieldsLengthProperties =
          configurationProperties.getProperty(BatchConstants.INPUT_FIELD_LENGTHS);
      if (fieldsLengthProperties == null || fieldsLengthProperties.length() == 0) {
        String error =
            "Configuration failed : when using fixed length records, fields length values property '"
                + BatchConstants.INPUT_FIELD_LENGTHS
                + "' is mandatory but was not specified.";
        logger.severe(error);
        throw new BatchConfigurationException(error);
      } else {
        // parse fields length property and extract numeric values
        StringTokenizer stringTokenizer = new StringTokenizer(fieldsLengthProperties, ",");
        int[] fieldsLength = new int[stringTokenizer.countTokens()];
        int index = 0;
        while (stringTokenizer.hasMoreTokens()) {
          String length = stringTokenizer.nextToken();
          try {
            fieldsLength[index] = Integer.parseInt(length);
            index++;
          } catch (NumberFormatException e) {
            String error =
                "Configuration failed : field length '"
                    + length
                    + "' in property "
                    + BatchConstants.INPUT_FIELD_LENGTHS
                    + "="
                    + fieldsLengthProperties
                    + " is not numeric.";
            logger.severe(error);
            throw new BatchConfigurationException(error);
          }
        }
        recordParser = new FlrRecordParserImpl(fieldsLength);
      }
    } else { // delimited values configuration

      String recordSizeProperty =
          configurationProperties.getProperty(BatchConstants.INPUT_RECORD_SIZE);

      try {

        String fieldsDelimiter =
            configurationProperties.getProperty(BatchConstants.INPUT_FIELD_DELIMITER);
        if (fieldsDelimiter == null || fieldsDelimiter.length() == 0) {
          fieldsDelimiter = BatchConstants.DEFAULT_FIELD_DELIMITER;
          logger.info("No field delimiter specified, using default : '" + fieldsDelimiter + "'");
        }

        String trimWhitespacesProperty =
            configurationProperties.getProperty(BatchConstants.INPUT_FIELD_TRIM);
        boolean trimWhitespaces;
        if (trimWhitespacesProperty != null) {
          trimWhitespaces = Boolean.valueOf(trimWhitespacesProperty);
        } else {
          trimWhitespaces = BatchConstants.DEFAULT_FIELD_TRIM;
          logger.info("Trim whitespaces property not specified, default to " + trimWhitespaces);
        }

        String dataQualifierCharacterProperty =
            configurationProperties.getProperty(BatchConstants.INPUT_FIELD_QUALIFIER_CHAR);
        String dataQualifierCharacter = BatchConstants.DEFAULT_FIELD_QUALIFIER_CHAR;
        if (dataQualifierCharacterProperty != null && dataQualifierCharacterProperty.length() > 0) {
          dataQualifierCharacter = dataQualifierCharacterProperty;
        } else {
          logger.info(
              "Data qualifier character not specified, default to " + dataQualifierCharacter);
        }

        recordParser =
            new DsvRecordParserImpl(fieldsDelimiter, trimWhitespaces, dataQualifierCharacter);

        if (recordSizeProperty == null || recordSizeProperty.length() == 0) {
          logger.info(
              "Record size property not specified, it will be calculated from the header record");
          String headerRecord = recordReader.getHeaderRecord();
          Record record =
              recordParser.parseRecord(
                  headerRecord,
                  0); // use the record parser to parse the header record using the right delimiter
          recordSizeProperty = String.valueOf(record.getFields().size());
        }

        int recordSize = Integer.parseInt(recordSizeProperty);

        recordParser =
            new DsvRecordParserImpl(
                recordSize, fieldsDelimiter, trimWhitespaces, dataQualifierCharacter);

        logger.info("Record size : " + recordSize);
        logger.info("Fields delimiter : '" + fieldsDelimiter + "'");
        logger.info("Data qualifier character : '" + dataQualifierCharacter + "'");

      } catch (NumberFormatException e) {
        String error = "Record size property is not recognized as a number : " + recordSizeProperty;
        logger.severe(error);
        throw new BatchConfigurationException(error);
      }
    }
  }
예제 #9
0
  /**
   * Configure loggers for ignored/rejected/errors records.
   *
   * @throws BatchConfigurationException thrown if loggers for ignored/rejected/errors records are
   *     not correctly configured
   */
  private void configureRecordsLoggers() throws BatchConfigurationException {

    String inputDataProperty = configurationProperties.getProperty(BatchConstants.INPUT_DATA_PATH);

    ReportFormatter reportFormatter = new ReportFormatter();

    // ignored records logger
    String outputIgnored = configurationProperties.getProperty(BatchConstants.OUTPUT_DATA_IGNORED);
    if (outputIgnored == null || (outputIgnored.length() == 0)) {
      outputIgnored =
          BatchConfigurationUtil.removeExtension(inputDataProperty)
              + BatchConstants.DEFAULT_IGNORED_SUFFIX;
      logger.info("No log file specified for ignored records, using default : " + outputIgnored);
    }
    try {
      FileHandler ignoredRecordsHandler = new FileHandler(outputIgnored);
      ignoredRecordsHandler.setFormatter(reportFormatter);
      Logger ignoredRecordsReporter = Logger.getLogger(BatchConstants.LOGGER_CB4J_IGNORED);
      ignoredRecordsReporter.addHandler(ignoredRecordsHandler);
    } catch (IOException e) {
      String error = "Unable to use file for ignored records : " + outputIgnored;
      logger.severe(error);
      throw new BatchConfigurationException(error);
    }

    // rejected errors logger
    String outputRejected =
        configurationProperties.getProperty(BatchConstants.OUTPUT_DATA_REJECTED);
    if (outputRejected == null || (outputRejected.length() == 0)) {
      outputRejected =
          BatchConfigurationUtil.removeExtension(inputDataProperty)
              + BatchConstants.DEFAULT_REJECTED_SUFFIX;
      logger.info("No log file specified for rejected records, using default : " + outputRejected);
    }
    try {
      FileHandler rejectedRecordsHandler = new FileHandler(outputRejected);
      rejectedRecordsHandler.setFormatter(reportFormatter);
      Logger rejectedRecordsReporter = Logger.getLogger(BatchConstants.LOGGER_CB4J_REJECTED);
      rejectedRecordsReporter.addHandler(rejectedRecordsHandler);
    } catch (IOException e) {
      String error = "Unable to use file for rejected records : " + outputRejected;
      logger.severe(error);
      throw new BatchConfigurationException(error);
    }

    // errors record logger
    String outputErrors = configurationProperties.getProperty(BatchConstants.OUTPUT_DATA_ERRORS);
    if (outputErrors == null || (outputErrors.length() == 0)) {
      outputErrors =
          BatchConfigurationUtil.removeExtension(inputDataProperty)
              + BatchConstants.DEFAULT_ERRORS_SUFFIX;
      logger.info("No log file specified for error records, using default : " + outputErrors);
    }
    try {
      FileHandler errorRecordsHandler = new FileHandler(outputErrors);
      errorRecordsHandler.setFormatter(reportFormatter);
      Logger errorRecordsReporter = Logger.getLogger(BatchConstants.LOGGER_CB4J_ERRORS);
      errorRecordsReporter.addHandler(errorRecordsHandler);
    } catch (IOException e) {
      String error = "Unable to use file for error records : " + outputErrors;
      logger.severe(error);
      throw new BatchConfigurationException(error);
    }
  }
예제 #10
0
  /**
   * Configure the batch engine.
   *
   * @throws BatchConfigurationException thrown if :
   *     <ul>
   *       <li>One of the mandatory parameters is not specified, please refer to the reference
   *           documentation for all parameters details
   *       <li>Log files for ignored and rejected records cannot be used
   *       <li>One of the mandatory services is not specified, please refer to the reference
   *           documentation for all mandatory services implementations
   *     </ul>
   */
  public void configure() throws BatchConfigurationException {

    /*
     * Configure CB4J logger
     */
    configureCB4JLogger();

    logger.info("Configuration started at : " + new Date());

    /*
     * Check record processor
     */
    if (recordProcessor == null) {
      String error = "Configuration failed : no record processor registered";
      logger.severe(error);
      throw new BatchConfigurationException(error);
    }

    /*
     * Configure record reader
     */
    configureRecordReader();

    /*
     * Configure record parser
     */
    configureRecordParser();

    /*
     * Configure loggers for ignored/rejected/error records
     */
    configureRecordsLoggers();

    /*
     * Configure batch reporter : if no custom reporter registered, use default implementation
     */
    if (batchReporter == null) {
      batchReporter = new DefaultBatchReporterImpl();
    }
    batchReporter.init();

    /*
     * Configure record validator with provided validators : if no custom validator registered, use default implementation
     */
    if (recordValidator == null) {
      recordValidator = new DefaultRecordValidatorImpl(fieldValidators);
    }

    /*
     * Check record mapper : if no custom mapper registered, use default implementation
     */
    if (recordMapper == null) {
      recordMapper = configureDefaultRecordMapper();
    }

    /*
     * register JMX MBean
     */
    if (Boolean.valueOf(
        configurationProperties.getProperty(BatchConstants.OUTPUT_DATA_JMX_ENABLED))) {
      configureJmxMBean();
    }

    logger.info("Configuration successful");
    logger.info("Configuration parameters details : " + configurationProperties);
  }