/** * Returns the file name to use for the read-only version of the provided log file. * * <p>The file name is based on the lowest and highest key in the log file. * * @return the name to use for the read-only version of the log file * @throws ChangelogException If an error occurs. */ private String generateReadOnlyFileName(final LogFile<K, V> logFile) throws ChangelogException { final K lowestKey = logFile.getOldestRecord().getKey(); final K highestKey = logFile.getNewestRecord().getKey(); return recordParser.encodeKeyToString(lowestKey) + LOG_FILE_NAME_SEPARATOR + recordParser.encodeKeyToString(highestKey) + LOG_FILE_SUFFIX; }
/** * Returns the key bounds for the provided log file. * * @return the pair of (lowest key, highest key) that correspond to records stored in the * corresponding log file. * @throws ChangelogException if an error occurs while retrieving the keys */ private Pair<K, K> getKeyBounds(final LogFile<K, V> logFile) throws ChangelogException { try { final String name = logFile.getFile().getName(); final String[] keys = name.substring(0, name.length() - Log.LOG_FILE_SUFFIX.length()) .split(LOG_FILE_NAME_SEPARATOR); return Pair.of( recordParser.decodeKeyFromString(keys[0]), recordParser.decodeKeyFromString(keys[1])); } catch (Exception e) { throw new ChangelogException( ERR_CHANGELOG_UNABLE_TO_RETRIEVE_KEY_BOUNDS_FROM_FILE.get(logFile.getFile().getPath()), e); } }
private void openHeadLogFile() throws ChangelogException { final LogFile<K, V> head = LogFile.newAppendableLogFile(new File(logPath, HEAD_LOG_FILE_NAME), recordParser); final Record<K, V> newestRecord = head.getNewestRecord(); lastAppendedKey = newestRecord != null ? newestRecord.getKey() : null; logFiles.put(recordParser.getMaxKey(), head); }
/** * Configure the default record mapper. * * @return the default implementation of record mapper * @throws BatchConfigurationException if the target type class is not found */ private RecordMapper configureDefaultRecordMapper() throws BatchConfigurationException { RecordMapper recordMapper; String recordClassName = configurationProperties.getProperty(BatchConstants.INPUT_RECORD_CLASS); if (recordClassName == null || recordClassName.length() == 0) { try { Class recordProcessorClass = Class.forName(recordProcessor.getClass().getName()); Method[] declaredMethods = recordProcessorClass.getDeclaredMethods(); for (Method declaredMethod : declaredMethods) { if (declaredMethod.getName().equals("processRecord")) { recordClassName = declaredMethod.getParameterTypes()[0].getName(); break; } } } catch (ClassNotFoundException e) { String error = "Configuration failed : unable to get record class name from registered record processor implementation."; logger.severe(error); throw new BatchConfigurationException(error, e); } } String[] headers; String headersProperty = configurationProperties.getProperty(BatchConstants.INPUT_RECORD_HEADERS); if (headersProperty == null) { // if no headers specified, use field names declared in the header record String headerRecord = recordReader.getHeaderRecord(); Record record = recordParser.parseRecord( headerRecord, 0); // use the record parser to parse the header record using the right delimiter List<Field> fields = record.getFields(); headers = new String[fields.size()]; for (int i = 0; i < fields.size(); i++) { headers[i] = fields.get(i).getContent(); } } else { // headers specified, split the comma separated list headers = headersProperty.split(","); } try { recordMapper = new DefaultRecordMapperImpl(recordClassName, headers, typeConverters); } catch (ClassNotFoundException e) { String error = "Configuration failed : Class " + recordClassName + " not found."; logger.severe(error); throw new BatchConfigurationException(error, e); } return recordMapper; }
/** Update the cursors that were pointing to head after a rotation of the head log file. */ private void updateOpenedCursorsOnHeadAfterRotation( List<Pair<AbortableLogCursor<K, V>, CursorState<K, V>>> cursors) throws ChangelogException { for (Pair<AbortableLogCursor<K, V>, CursorState<K, V>> pair : cursors) { final CursorState<K, V> cursorState = pair.getSecond(); // Need to update the cursor only if it is pointing to the head log file if (cursorState.isValid() && isHeadLogFile(cursorState.logFile)) { final K previousKey = logFiles.lowerKey(recordParser.getMaxKey()); final LogFile<K, V> logFile = findLogFileFor(previousKey); final AbortableLogCursor<K, V> cursor = pair.getFirst(); cursor.reinitializeTo( new CursorState<K, V>(logFile, cursorState.filePosition, cursorState.record)); } } }
/** * Configure CB4J record parser. * * @throws BatchConfigurationException thrown if record parser is not correctly configured */ private void configureRecordParser() throws BatchConfigurationException { // read record type property and set default value if invalid input String recordTypeProperty = configurationProperties.getProperty(BatchConstants.INPUT_RECORD_TYPE); String recordType; if (recordTypeProperty == null || recordTypeProperty.length() == 0) { recordType = BatchConstants.DEFAULT_RECORD_TYPE; logger.info( "Record type property not specified, records will be considered as delimiter-separated values"); } else if (!RecordType.DSV.toString().equalsIgnoreCase(recordTypeProperty) && !RecordType.FLR.toString().equalsIgnoreCase(recordTypeProperty)) { recordType = BatchConstants.DEFAULT_RECORD_TYPE; logger.warning( "Record type property '" + recordTypeProperty + "' is invalid, records will be considered as delimiter-separated values"); } else { recordType = recordTypeProperty; } // fixed length record configuration if (RecordType.FLR.toString().equalsIgnoreCase(recordType)) { String fieldsLengthProperties = configurationProperties.getProperty(BatchConstants.INPUT_FIELD_LENGTHS); if (fieldsLengthProperties == null || fieldsLengthProperties.length() == 0) { String error = "Configuration failed : when using fixed length records, fields length values property '" + BatchConstants.INPUT_FIELD_LENGTHS + "' is mandatory but was not specified."; logger.severe(error); throw new BatchConfigurationException(error); } else { // parse fields length property and extract numeric values StringTokenizer stringTokenizer = new StringTokenizer(fieldsLengthProperties, ","); int[] fieldsLength = new int[stringTokenizer.countTokens()]; int index = 0; while (stringTokenizer.hasMoreTokens()) { String length = stringTokenizer.nextToken(); try { fieldsLength[index] = Integer.parseInt(length); index++; } catch (NumberFormatException e) { String error = "Configuration failed : field length '" + length + "' in property " + BatchConstants.INPUT_FIELD_LENGTHS + "=" + fieldsLengthProperties + " is not numeric."; logger.severe(error); throw new BatchConfigurationException(error); } } recordParser = new FlrRecordParserImpl(fieldsLength); } } else { // delimited values configuration String recordSizeProperty = configurationProperties.getProperty(BatchConstants.INPUT_RECORD_SIZE); try { String fieldsDelimiter = configurationProperties.getProperty(BatchConstants.INPUT_FIELD_DELIMITER); if (fieldsDelimiter == null || fieldsDelimiter.length() == 0) { fieldsDelimiter = BatchConstants.DEFAULT_FIELD_DELIMITER; logger.info("No field delimiter specified, using default : '" + fieldsDelimiter + "'"); } String trimWhitespacesProperty = configurationProperties.getProperty(BatchConstants.INPUT_FIELD_TRIM); boolean trimWhitespaces; if (trimWhitespacesProperty != null) { trimWhitespaces = Boolean.valueOf(trimWhitespacesProperty); } else { trimWhitespaces = BatchConstants.DEFAULT_FIELD_TRIM; logger.info("Trim whitespaces property not specified, default to " + trimWhitespaces); } String dataQualifierCharacterProperty = configurationProperties.getProperty(BatchConstants.INPUT_FIELD_QUALIFIER_CHAR); String dataQualifierCharacter = BatchConstants.DEFAULT_FIELD_QUALIFIER_CHAR; if (dataQualifierCharacterProperty != null && dataQualifierCharacterProperty.length() > 0) { dataQualifierCharacter = dataQualifierCharacterProperty; } else { logger.info( "Data qualifier character not specified, default to " + dataQualifierCharacter); } recordParser = new DsvRecordParserImpl(fieldsDelimiter, trimWhitespaces, dataQualifierCharacter); if (recordSizeProperty == null || recordSizeProperty.length() == 0) { logger.info( "Record size property not specified, it will be calculated from the header record"); String headerRecord = recordReader.getHeaderRecord(); Record record = recordParser.parseRecord( headerRecord, 0); // use the record parser to parse the header record using the right delimiter recordSizeProperty = String.valueOf(record.getFields().size()); } int recordSize = Integer.parseInt(recordSizeProperty); recordParser = new DsvRecordParserImpl( recordSize, fieldsDelimiter, trimWhitespaces, dataQualifierCharacter); logger.info("Record size : " + recordSize); logger.info("Fields delimiter : '" + fieldsDelimiter + "'"); logger.info("Data qualifier character : '" + dataQualifierCharacter + "'"); } catch (NumberFormatException e) { String error = "Record size property is not recognized as a number : " + recordSizeProperty; logger.severe(error); throw new BatchConfigurationException(error); } } }
/* (non-Javadoc) * @see org.xml.sax.XMLReader#parse(org.xml.sax.InputSource) */ public void parse(InputSource inputSource) throws IOException, SAXException { if (contentHandler == null) { throw new IllegalStateException("'contentHandler' not set. Cannot parse Record stream."); } if (execContext == null) { throw new IllegalStateException("'execContext' not set. Cannot parse Record stream."); } try { Reader recordReader; // Create the record parser.... RecordParser recordParser = parserFactory.newRecordParser(); recordParser.setRecordParserFactory(parserFactory); recordParser.setDataSource(inputSource); // Start the document and add the root "record-set" element... contentHandler.startDocument(); contentHandler.startElement( XMLConstants.NULL_NS_URI, rootElementName, StringUtils.EMPTY, EMPTY_ATTRIBS); // Output each of the CVS line entries... int lineNumber = 0; Record record = recordParser.nextRecord(); while (record != null) { lineNumber++; // First line is line "1" if (record != null) { List<Field> recordFields = record.getFields(); if (indent) { contentHandler.characters(INDENT_LF, 0, 1); contentHandler.characters(INDENTCHARS, 0, 1); } AttributesImpl attrs = new AttributesImpl(); attrs.addAttribute( XMLConstants.NULL_NS_URI, RECORD_NUMBER_ATTR, RECORD_NUMBER_ATTR, "xs:int", Integer.toString(lineNumber)); RecordMetaData recordMetaData = record.getRecordMetaData(); if (recordFields.size() < recordMetaData.getUnignoredFieldCount()) { attrs.addAttribute( XMLConstants.NULL_NS_URI, RECORD_TRUNCATED_ATTR, RECORD_TRUNCATED_ATTR, "xs:boolean", Boolean.TRUE.toString()); } contentHandler.startElement( XMLConstants.NULL_NS_URI, record.getName(), StringUtils.EMPTY, attrs); for (Field recordField : recordFields) { String fieldName = recordField.getName(); if (indent) { contentHandler.characters(INDENT_LF, 0, 1); contentHandler.characters(INDENTCHARS, 0, 2); } contentHandler.startElement( XMLConstants.NULL_NS_URI, fieldName, StringUtils.EMPTY, EMPTY_ATTRIBS); String value = recordField.getValue(); contentHandler.characters(value.toCharArray(), 0, value.length()); contentHandler.endElement(XMLConstants.NULL_NS_URI, fieldName, StringUtils.EMPTY); } if (indent) { contentHandler.characters(INDENT_LF, 0, 1); contentHandler.characters(INDENTCHARS, 0, 1); } contentHandler.endElement(XMLConstants.NULL_NS_URI, record.getName(), StringUtils.EMPTY); } record = recordParser.nextRecord(); } if (indent) { contentHandler.characters(INDENT_LF, 0, 1); } // Close out the "csv-set" root element and end the document.. contentHandler.endElement(XMLConstants.NULL_NS_URI, rootElementName, StringUtils.EMPTY); contentHandler.endDocument(); } finally { // These properties need to be reset for every execution (e.g. when reader is pooled). contentHandler = null; execContext = null; } }