// Dumps the contents of the file as ByteBuffer.
public void readFile_ByteBuffer() throws Exception {
OrcStruct lv_row = null;
Object lv_field_val = null;
ByteBuffer lv_row_buffer;
while (m_rr.hasNext()) {
byte[] lv_row_ba = new byte[4096];
lv_row_buffer = ByteBuffer.wrap(lv_row_ba);
lv_row = (OrcStruct) m_rr.next(lv_row);
for (int i = 0; i < m_fields.size(); i++) {
lv_field_val = lv_row.getFieldValue(i);
if (lv_field_val == null) {
lv_row_buffer.putInt(0);
continue;
}
String lv_field_val_str = lv_field_val.toString();
lv_row_buffer.putInt(lv_field_val_str.length());
if (lv_field_val != null) {
lv_row_buffer.put(lv_field_val_str.getBytes());
}
}
System.out.println(lv_row_buffer);
// System.out.println(new String(lv_row_buffer.array()));
}
}
public byte[] getNext() throws Exception {
if (!m_rr.hasNext()) {
return null;
}
OrcStruct lv_row = (OrcStruct) m_rr.next(null);
Object lv_field_val = null;
ByteBuffer lv_row_buffer;
byte[] lv_row_ba = new byte[4096];
lv_row_buffer = ByteBuffer.wrap(lv_row_ba);
for (int i = 0; i < m_fields.size(); i++) {
lv_field_val = lv_row.getFieldValue(i);
if (lv_field_val == null) {
lv_row_buffer.putInt(0);
continue;
}
String lv_field_val_str = lv_field_val.toString();
lv_row_buffer.putInt(lv_field_val_str.length());
if (lv_field_val != null) {
lv_row_buffer.put(lv_field_val_str.getBytes());
}
}
System.out.println(lv_row_buffer);
return lv_row_buffer.array();
}
// Dumps the content of the file. The columns are '|' separated.
public void readFile_String() throws Exception {
OrcStruct lv_row = null;
Object lv_field_val = null;
StringBuilder lv_row_string = new StringBuilder(1024);
while (m_rr.hasNext()) {
lv_row = (OrcStruct) m_rr.next(lv_row);
lv_row_string.setLength(0);
for (int i = 0; i < m_fields.size(); i++) {
lv_field_val = lv_row.getFieldValue(i);
if (lv_field_val != null) {
lv_row_string.append(lv_field_val);
}
lv_row_string.append('|');
}
System.out.println(lv_row_string);
}
}
public boolean seekToRow(long pv_rowNumber) throws IOException {
if ((pv_rowNumber < 0) || (pv_rowNumber >= m_reader.getNumberOfRows())) {
return false;
}
m_rr.seekToRow(pv_rowNumber);
return true;
}
void checkFormat(Job job) throws Exception {
TaskAttemptContext attemptContext =
new TaskAttemptContext(job.getConfiguration(), new TaskAttemptID("123", 0, false, 1, 2));
MyClassMessagePackBase64LineInputFormat format = new MyClassMessagePackBase64LineInputFormat();
FileInputFormat.setInputPaths(job, workDir);
List<InputSplit> splits = format.getSplits(job);
for (int j = 0; j < splits.size(); j++) {
RecordReader<LongWritable, MyClassWritable> reader =
format.createRecordReader(splits.get(j), attemptContext);
reader.initialize(splits.get(j), attemptContext);
int count = 0;
try {
while (reader.nextKeyValue()) {
LongWritable key = reader.getCurrentKey();
MyClassWritable val = reader.getCurrentValue();
MyClass mc = val.get();
assertEquals(mc.v, count);
assertEquals(mc.s, Integer.toString(count));
count++;
}
} finally {
reader.close();
}
}
}
/**
* Configure the default record mapper.
*
* @return the default implementation of record mapper
* @throws BatchConfigurationException if the target type class is not found
*/
private RecordMapper configureDefaultRecordMapper() throws BatchConfigurationException {
RecordMapper recordMapper;
String recordClassName = configurationProperties.getProperty(BatchConstants.INPUT_RECORD_CLASS);
if (recordClassName == null || recordClassName.length() == 0) {
try {
Class recordProcessorClass = Class.forName(recordProcessor.getClass().getName());
Method[] declaredMethods = recordProcessorClass.getDeclaredMethods();
for (Method declaredMethod : declaredMethods) {
if (declaredMethod.getName().equals("processRecord")) {
recordClassName = declaredMethod.getParameterTypes()[0].getName();
break;
}
}
} catch (ClassNotFoundException e) {
String error =
"Configuration failed : unable to get record class name from registered record processor implementation.";
logger.severe(error);
throw new BatchConfigurationException(error, e);
}
}
String[] headers;
String headersProperty =
configurationProperties.getProperty(BatchConstants.INPUT_RECORD_HEADERS);
if (headersProperty
== null) { // if no headers specified, use field names declared in the header record
String headerRecord = recordReader.getHeaderRecord();
Record record =
recordParser.parseRecord(
headerRecord,
0); // use the record parser to parse the header record using the right delimiter
List<Field> fields = record.getFields();
headers = new String[fields.size()];
for (int i = 0; i < fields.size(); i++) {
headers[i] = fields.get(i).getContent();
}
} else { // headers specified, split the comma separated list
headers = headersProperty.split(",");
}
try {
recordMapper = new DefaultRecordMapperImpl(recordClassName, headers, typeConverters);
} catch (ClassNotFoundException e) {
String error = "Configuration failed : Class " + recordClassName + " not found.";
logger.severe(error);
throw new BatchConfigurationException(error, e);
}
return recordMapper;
}
public void testFormat() throws Exception {
JobConf job = new JobConf(conf);
FileSystem fs = FileSystem.getLocal(conf);
Path dir = new Path(System.getProperty("test.build.data", ".") + "/mapred");
Path file = new Path(dir, "test.seq");
Reporter reporter = Reporter.NULL;
int seed = new Random().nextInt();
// LOG.info("seed = "+seed);
Random random = new Random(seed);
fs.delete(dir, true);
FileInputFormat.setInputPaths(job, dir);
// for a variety of lengths
for (int length = 0; length < MAX_LENGTH; length += random.nextInt(MAX_LENGTH / 10) + 1) {
// LOG.info("creating; entries = " + length);
// create a file with length entries
SequenceFile.Writer writer =
SequenceFile.createWriter(fs, conf, file, IntWritable.class, BytesWritable.class);
try {
for (int i = 0; i < length; i++) {
IntWritable key = new IntWritable(i);
byte[] data = new byte[random.nextInt(10)];
random.nextBytes(data);
BytesWritable value = new BytesWritable(data);
writer.append(key, value);
}
} finally {
writer.close();
}
// try splitting the file in a variety of sizes
InputFormat<IntWritable, BytesWritable> format =
new SequenceFileInputFormat<IntWritable, BytesWritable>();
IntWritable key = new IntWritable();
BytesWritable value = new BytesWritable();
for (int i = 0; i < 3; i++) {
int numSplits = random.nextInt(MAX_LENGTH / (SequenceFile.SYNC_INTERVAL / 20)) + 1;
// LOG.info("splitting: requesting = " + numSplits);
InputSplit[] splits = format.getSplits(job, numSplits);
// LOG.info("splitting: got = " + splits.length);
// check each split
BitSet bits = new BitSet(length);
for (int j = 0; j < splits.length; j++) {
RecordReader<IntWritable, BytesWritable> reader =
format.getRecordReader(splits[j], job, reporter);
try {
int count = 0;
while (reader.next(key, value)) {
// if (bits.get(key.get())) {
// LOG.info("splits["+j+"]="+splits[j]+" : " +
// key.get());
// LOG.info("@"+reader.getPos());
// }
assertFalse("Key in multiple partitions.", bits.get(key.get()));
bits.set(key.get());
count++;
}
// LOG.info("splits["+j+"]="+splits[j]+" count=" +
// count);
} finally {
reader.close();
}
}
assertEquals("Some keys in no partition.", length, bits.cardinality());
}
}
}
// This method is just for experimentation.
public void testRead() throws Exception {
m_reader = OrcFile.createReader(m_file_path, OrcFile.readerOptions(m_conf));
System.out.println("Reader: " + m_reader);
System.out.println("# Rows: " + m_reader.getNumberOfRows());
m_types = m_reader.getTypes();
System.out.println("# Types in the file: " + m_types.size());
for (int i = 0; i < m_types.size(); i++) {
System.out.println("Type " + i + ": " + m_types.get(i).getKind());
}
System.out.println("Compression: " + m_reader.getCompression());
if (m_reader.getCompression() != CompressionKind.NONE) {
System.out.println("Compression size: " + m_reader.getCompressionSize());
}
StructObjectInspector m_oi = (StructObjectInspector) m_reader.getObjectInspector();
System.out.println("object inspector type category: " + m_oi.getCategory());
System.out.println("object inspector type name : " + m_oi.getTypeName());
m_fields = m_oi.getAllStructFieldRefs();
System.out.println("Number of columns in the table: " + m_fields.size());
RecordReader m_rr = m_reader.rows();
// Print the type info:
for (int i = 0; i < m_fields.size(); i++) {
System.out.println("Column " + i + " name: " + m_fields.get(i).getFieldName());
ObjectInspector lv_foi = m_fields.get(i).getFieldObjectInspector();
System.out.println("Column " + i + " type category: " + lv_foi.getCategory());
System.out.println("Column " + i + " type name: " + lv_foi.getTypeName());
// Object lv_column_val = m_oi.getStructFieldData(lv_row, m_fields.get(i));
// System.out.print("Column " + i + " value: " + lv_row.getFieldValue(i));
}
OrcStruct lv_row = null;
Object lv_field_val = null;
StringBuilder lv_row_string = new StringBuilder(1024);
while (m_rr.hasNext()) {
lv_row = (OrcStruct) m_rr.next(lv_row);
lv_row_string.setLength(0);
for (int i = 0; i < m_fields.size(); i++) {
lv_field_val = lv_row.getFieldValue(i);
if (lv_field_val != null) {
lv_row_string.append(lv_field_val);
}
lv_row_string.append('|');
}
System.out.println(lv_row_string);
}
/**
* Typecasting to appropriate type based on the 'kind' if (OrcProto.Type.Kind.INT ==
* m_types.get(1).getKind()) { IntWritable lvf_1_val = (IntWritable) lv_row.getFieldValue(0);
* System.out.println("Column 1 value: " + lvf_1_val); }
*/
}
/**
* Configure CB4J record parser.
*
* @throws BatchConfigurationException thrown if record parser is not correctly configured
*/
private void configureRecordParser() throws BatchConfigurationException {
// read record type property and set default value if invalid input
String recordTypeProperty =
configurationProperties.getProperty(BatchConstants.INPUT_RECORD_TYPE);
String recordType;
if (recordTypeProperty == null || recordTypeProperty.length() == 0) {
recordType = BatchConstants.DEFAULT_RECORD_TYPE;
logger.info(
"Record type property not specified, records will be considered as delimiter-separated values");
} else if (!RecordType.DSV.toString().equalsIgnoreCase(recordTypeProperty)
&& !RecordType.FLR.toString().equalsIgnoreCase(recordTypeProperty)) {
recordType = BatchConstants.DEFAULT_RECORD_TYPE;
logger.warning(
"Record type property '"
+ recordTypeProperty
+ "' is invalid, records will be considered as delimiter-separated values");
} else {
recordType = recordTypeProperty;
}
// fixed length record configuration
if (RecordType.FLR.toString().equalsIgnoreCase(recordType)) {
String fieldsLengthProperties =
configurationProperties.getProperty(BatchConstants.INPUT_FIELD_LENGTHS);
if (fieldsLengthProperties == null || fieldsLengthProperties.length() == 0) {
String error =
"Configuration failed : when using fixed length records, fields length values property '"
+ BatchConstants.INPUT_FIELD_LENGTHS
+ "' is mandatory but was not specified.";
logger.severe(error);
throw new BatchConfigurationException(error);
} else {
// parse fields length property and extract numeric values
StringTokenizer stringTokenizer = new StringTokenizer(fieldsLengthProperties, ",");
int[] fieldsLength = new int[stringTokenizer.countTokens()];
int index = 0;
while (stringTokenizer.hasMoreTokens()) {
String length = stringTokenizer.nextToken();
try {
fieldsLength[index] = Integer.parseInt(length);
index++;
} catch (NumberFormatException e) {
String error =
"Configuration failed : field length '"
+ length
+ "' in property "
+ BatchConstants.INPUT_FIELD_LENGTHS
+ "="
+ fieldsLengthProperties
+ " is not numeric.";
logger.severe(error);
throw new BatchConfigurationException(error);
}
}
recordParser = new FlrRecordParserImpl(fieldsLength);
}
} else { // delimited values configuration
String recordSizeProperty =
configurationProperties.getProperty(BatchConstants.INPUT_RECORD_SIZE);
try {
String fieldsDelimiter =
configurationProperties.getProperty(BatchConstants.INPUT_FIELD_DELIMITER);
if (fieldsDelimiter == null || fieldsDelimiter.length() == 0) {
fieldsDelimiter = BatchConstants.DEFAULT_FIELD_DELIMITER;
logger.info("No field delimiter specified, using default : '" + fieldsDelimiter + "'");
}
String trimWhitespacesProperty =
configurationProperties.getProperty(BatchConstants.INPUT_FIELD_TRIM);
boolean trimWhitespaces;
if (trimWhitespacesProperty != null) {
trimWhitespaces = Boolean.valueOf(trimWhitespacesProperty);
} else {
trimWhitespaces = BatchConstants.DEFAULT_FIELD_TRIM;
logger.info("Trim whitespaces property not specified, default to " + trimWhitespaces);
}
String dataQualifierCharacterProperty =
configurationProperties.getProperty(BatchConstants.INPUT_FIELD_QUALIFIER_CHAR);
String dataQualifierCharacter = BatchConstants.DEFAULT_FIELD_QUALIFIER_CHAR;
if (dataQualifierCharacterProperty != null && dataQualifierCharacterProperty.length() > 0) {
dataQualifierCharacter = dataQualifierCharacterProperty;
} else {
logger.info(
"Data qualifier character not specified, default to " + dataQualifierCharacter);
}
recordParser =
new DsvRecordParserImpl(fieldsDelimiter, trimWhitespaces, dataQualifierCharacter);
if (recordSizeProperty == null || recordSizeProperty.length() == 0) {
logger.info(
"Record size property not specified, it will be calculated from the header record");
String headerRecord = recordReader.getHeaderRecord();
Record record =
recordParser.parseRecord(
headerRecord,
0); // use the record parser to parse the header record using the right delimiter
recordSizeProperty = String.valueOf(record.getFields().size());
}
int recordSize = Integer.parseInt(recordSizeProperty);
recordParser =
new DsvRecordParserImpl(
recordSize, fieldsDelimiter, trimWhitespaces, dataQualifierCharacter);
logger.info("Record size : " + recordSize);
logger.info("Fields delimiter : '" + fieldsDelimiter + "'");
logger.info("Data qualifier character : '" + dataQualifierCharacter + "'");
} catch (NumberFormatException e) {
String error = "Record size property is not recognized as a number : " + recordSizeProperty;
logger.severe(error);
throw new BatchConfigurationException(error);
}
}
}