/**
* Returns the file name to use for the read-only version of the provided log file.
*
* <p>The file name is based on the lowest and highest key in the log file.
*
* @return the name to use for the read-only version of the log file
* @throws ChangelogException If an error occurs.
*/
private String generateReadOnlyFileName(final LogFile<K, V> logFile) throws ChangelogException {
final K lowestKey = logFile.getOldestRecord().getKey();
final K highestKey = logFile.getNewestRecord().getKey();
return recordParser.encodeKeyToString(lowestKey)
+ LOG_FILE_NAME_SEPARATOR
+ recordParser.encodeKeyToString(highestKey)
+ LOG_FILE_SUFFIX;
}
/**
* Returns the key bounds for the provided log file.
*
* @return the pair of (lowest key, highest key) that correspond to records stored in the
* corresponding log file.
* @throws ChangelogException if an error occurs while retrieving the keys
*/
private Pair<K, K> getKeyBounds(final LogFile<K, V> logFile) throws ChangelogException {
try {
final String name = logFile.getFile().getName();
final String[] keys =
name.substring(0, name.length() - Log.LOG_FILE_SUFFIX.length())
.split(LOG_FILE_NAME_SEPARATOR);
return Pair.of(
recordParser.decodeKeyFromString(keys[0]), recordParser.decodeKeyFromString(keys[1]));
} catch (Exception e) {
throw new ChangelogException(
ERR_CHANGELOG_UNABLE_TO_RETRIEVE_KEY_BOUNDS_FROM_FILE.get(logFile.getFile().getPath()),
e);
}
}
private void openHeadLogFile() throws ChangelogException {
final LogFile<K, V> head =
LogFile.newAppendableLogFile(new File(logPath, HEAD_LOG_FILE_NAME), recordParser);
final Record<K, V> newestRecord = head.getNewestRecord();
lastAppendedKey = newestRecord != null ? newestRecord.getKey() : null;
logFiles.put(recordParser.getMaxKey(), head);
}
/**
* Configure the default record mapper.
*
* @return the default implementation of record mapper
* @throws BatchConfigurationException if the target type class is not found
*/
private RecordMapper configureDefaultRecordMapper() throws BatchConfigurationException {
RecordMapper recordMapper;
String recordClassName = configurationProperties.getProperty(BatchConstants.INPUT_RECORD_CLASS);
if (recordClassName == null || recordClassName.length() == 0) {
try {
Class recordProcessorClass = Class.forName(recordProcessor.getClass().getName());
Method[] declaredMethods = recordProcessorClass.getDeclaredMethods();
for (Method declaredMethod : declaredMethods) {
if (declaredMethod.getName().equals("processRecord")) {
recordClassName = declaredMethod.getParameterTypes()[0].getName();
break;
}
}
} catch (ClassNotFoundException e) {
String error =
"Configuration failed : unable to get record class name from registered record processor implementation.";
logger.severe(error);
throw new BatchConfigurationException(error, e);
}
}
String[] headers;
String headersProperty =
configurationProperties.getProperty(BatchConstants.INPUT_RECORD_HEADERS);
if (headersProperty
== null) { // if no headers specified, use field names declared in the header record
String headerRecord = recordReader.getHeaderRecord();
Record record =
recordParser.parseRecord(
headerRecord,
0); // use the record parser to parse the header record using the right delimiter
List<Field> fields = record.getFields();
headers = new String[fields.size()];
for (int i = 0; i < fields.size(); i++) {
headers[i] = fields.get(i).getContent();
}
} else { // headers specified, split the comma separated list
headers = headersProperty.split(",");
}
try {
recordMapper = new DefaultRecordMapperImpl(recordClassName, headers, typeConverters);
} catch (ClassNotFoundException e) {
String error = "Configuration failed : Class " + recordClassName + " not found.";
logger.severe(error);
throw new BatchConfigurationException(error, e);
}
return recordMapper;
}
/** Update the cursors that were pointing to head after a rotation of the head log file. */
private void updateOpenedCursorsOnHeadAfterRotation(
List<Pair<AbortableLogCursor<K, V>, CursorState<K, V>>> cursors) throws ChangelogException {
for (Pair<AbortableLogCursor<K, V>, CursorState<K, V>> pair : cursors) {
final CursorState<K, V> cursorState = pair.getSecond();
// Need to update the cursor only if it is pointing to the head log file
if (cursorState.isValid() && isHeadLogFile(cursorState.logFile)) {
final K previousKey = logFiles.lowerKey(recordParser.getMaxKey());
final LogFile<K, V> logFile = findLogFileFor(previousKey);
final AbortableLogCursor<K, V> cursor = pair.getFirst();
cursor.reinitializeTo(
new CursorState<K, V>(logFile, cursorState.filePosition, cursorState.record));
}
}
}
/**
* Configure CB4J record parser.
*
* @throws BatchConfigurationException thrown if record parser is not correctly configured
*/
private void configureRecordParser() throws BatchConfigurationException {
// read record type property and set default value if invalid input
String recordTypeProperty =
configurationProperties.getProperty(BatchConstants.INPUT_RECORD_TYPE);
String recordType;
if (recordTypeProperty == null || recordTypeProperty.length() == 0) {
recordType = BatchConstants.DEFAULT_RECORD_TYPE;
logger.info(
"Record type property not specified, records will be considered as delimiter-separated values");
} else if (!RecordType.DSV.toString().equalsIgnoreCase(recordTypeProperty)
&& !RecordType.FLR.toString().equalsIgnoreCase(recordTypeProperty)) {
recordType = BatchConstants.DEFAULT_RECORD_TYPE;
logger.warning(
"Record type property '"
+ recordTypeProperty
+ "' is invalid, records will be considered as delimiter-separated values");
} else {
recordType = recordTypeProperty;
}
// fixed length record configuration
if (RecordType.FLR.toString().equalsIgnoreCase(recordType)) {
String fieldsLengthProperties =
configurationProperties.getProperty(BatchConstants.INPUT_FIELD_LENGTHS);
if (fieldsLengthProperties == null || fieldsLengthProperties.length() == 0) {
String error =
"Configuration failed : when using fixed length records, fields length values property '"
+ BatchConstants.INPUT_FIELD_LENGTHS
+ "' is mandatory but was not specified.";
logger.severe(error);
throw new BatchConfigurationException(error);
} else {
// parse fields length property and extract numeric values
StringTokenizer stringTokenizer = new StringTokenizer(fieldsLengthProperties, ",");
int[] fieldsLength = new int[stringTokenizer.countTokens()];
int index = 0;
while (stringTokenizer.hasMoreTokens()) {
String length = stringTokenizer.nextToken();
try {
fieldsLength[index] = Integer.parseInt(length);
index++;
} catch (NumberFormatException e) {
String error =
"Configuration failed : field length '"
+ length
+ "' in property "
+ BatchConstants.INPUT_FIELD_LENGTHS
+ "="
+ fieldsLengthProperties
+ " is not numeric.";
logger.severe(error);
throw new BatchConfigurationException(error);
}
}
recordParser = new FlrRecordParserImpl(fieldsLength);
}
} else { // delimited values configuration
String recordSizeProperty =
configurationProperties.getProperty(BatchConstants.INPUT_RECORD_SIZE);
try {
String fieldsDelimiter =
configurationProperties.getProperty(BatchConstants.INPUT_FIELD_DELIMITER);
if (fieldsDelimiter == null || fieldsDelimiter.length() == 0) {
fieldsDelimiter = BatchConstants.DEFAULT_FIELD_DELIMITER;
logger.info("No field delimiter specified, using default : '" + fieldsDelimiter + "'");
}
String trimWhitespacesProperty =
configurationProperties.getProperty(BatchConstants.INPUT_FIELD_TRIM);
boolean trimWhitespaces;
if (trimWhitespacesProperty != null) {
trimWhitespaces = Boolean.valueOf(trimWhitespacesProperty);
} else {
trimWhitespaces = BatchConstants.DEFAULT_FIELD_TRIM;
logger.info("Trim whitespaces property not specified, default to " + trimWhitespaces);
}
String dataQualifierCharacterProperty =
configurationProperties.getProperty(BatchConstants.INPUT_FIELD_QUALIFIER_CHAR);
String dataQualifierCharacter = BatchConstants.DEFAULT_FIELD_QUALIFIER_CHAR;
if (dataQualifierCharacterProperty != null && dataQualifierCharacterProperty.length() > 0) {
dataQualifierCharacter = dataQualifierCharacterProperty;
} else {
logger.info(
"Data qualifier character not specified, default to " + dataQualifierCharacter);
}
recordParser =
new DsvRecordParserImpl(fieldsDelimiter, trimWhitespaces, dataQualifierCharacter);
if (recordSizeProperty == null || recordSizeProperty.length() == 0) {
logger.info(
"Record size property not specified, it will be calculated from the header record");
String headerRecord = recordReader.getHeaderRecord();
Record record =
recordParser.parseRecord(
headerRecord,
0); // use the record parser to parse the header record using the right delimiter
recordSizeProperty = String.valueOf(record.getFields().size());
}
int recordSize = Integer.parseInt(recordSizeProperty);
recordParser =
new DsvRecordParserImpl(
recordSize, fieldsDelimiter, trimWhitespaces, dataQualifierCharacter);
logger.info("Record size : " + recordSize);
logger.info("Fields delimiter : '" + fieldsDelimiter + "'");
logger.info("Data qualifier character : '" + dataQualifierCharacter + "'");
} catch (NumberFormatException e) {
String error = "Record size property is not recognized as a number : " + recordSizeProperty;
logger.severe(error);
throw new BatchConfigurationException(error);
}
}
}
/* (non-Javadoc)
* @see org.xml.sax.XMLReader#parse(org.xml.sax.InputSource)
*/
public void parse(InputSource inputSource) throws IOException, SAXException {
if (contentHandler == null) {
throw new IllegalStateException("'contentHandler' not set. Cannot parse Record stream.");
}
if (execContext == null) {
throw new IllegalStateException("'execContext' not set. Cannot parse Record stream.");
}
try {
Reader recordReader;
// Create the record parser....
RecordParser recordParser = parserFactory.newRecordParser();
recordParser.setRecordParserFactory(parserFactory);
recordParser.setDataSource(inputSource);
// Start the document and add the root "record-set" element...
contentHandler.startDocument();
contentHandler.startElement(
XMLConstants.NULL_NS_URI, rootElementName, StringUtils.EMPTY, EMPTY_ATTRIBS);
// Output each of the CVS line entries...
int lineNumber = 0;
Record record = recordParser.nextRecord();
while (record != null) {
lineNumber++; // First line is line "1"
if (record != null) {
List<Field> recordFields = record.getFields();
if (indent) {
contentHandler.characters(INDENT_LF, 0, 1);
contentHandler.characters(INDENTCHARS, 0, 1);
}
AttributesImpl attrs = new AttributesImpl();
attrs.addAttribute(
XMLConstants.NULL_NS_URI,
RECORD_NUMBER_ATTR,
RECORD_NUMBER_ATTR,
"xs:int",
Integer.toString(lineNumber));
RecordMetaData recordMetaData = record.getRecordMetaData();
if (recordFields.size() < recordMetaData.getUnignoredFieldCount()) {
attrs.addAttribute(
XMLConstants.NULL_NS_URI,
RECORD_TRUNCATED_ATTR,
RECORD_TRUNCATED_ATTR,
"xs:boolean",
Boolean.TRUE.toString());
}
contentHandler.startElement(
XMLConstants.NULL_NS_URI, record.getName(), StringUtils.EMPTY, attrs);
for (Field recordField : recordFields) {
String fieldName = recordField.getName();
if (indent) {
contentHandler.characters(INDENT_LF, 0, 1);
contentHandler.characters(INDENTCHARS, 0, 2);
}
contentHandler.startElement(
XMLConstants.NULL_NS_URI, fieldName, StringUtils.EMPTY, EMPTY_ATTRIBS);
String value = recordField.getValue();
contentHandler.characters(value.toCharArray(), 0, value.length());
contentHandler.endElement(XMLConstants.NULL_NS_URI, fieldName, StringUtils.EMPTY);
}
if (indent) {
contentHandler.characters(INDENT_LF, 0, 1);
contentHandler.characters(INDENTCHARS, 0, 1);
}
contentHandler.endElement(XMLConstants.NULL_NS_URI, record.getName(), StringUtils.EMPTY);
}
record = recordParser.nextRecord();
}
if (indent) {
contentHandler.characters(INDENT_LF, 0, 1);
}
// Close out the "csv-set" root element and end the document..
contentHandler.endElement(XMLConstants.NULL_NS_URI, rootElementName, StringUtils.EMPTY);
contentHandler.endDocument();
} finally {
// These properties need to be reset for every execution (e.g. when reader is pooled).
contentHandler = null;
execContext = null;
}
}
