public void testInputFormat() {
try {
JobConf conf = new JobConf();
String TMP_DIR = System.getProperty("test.build.data", "/tmp");
Path filename = new Path("file:///" + TMP_DIR + "/tmpSeqFile");
SequenceFile.Writer sfw =
SequenceFile.createWriter(
FileSystem.getLocal(conf),
conf,
filename,
ChukwaArchiveKey.class,
ChunkImpl.class,
SequenceFile.CompressionType.NONE,
Reporter.NULL);
StringBuilder buf = new StringBuilder();
int offsets[] = new int[lines.length];
for (int i = 0; i < lines.length; ++i) {
buf.append(lines[i]);
buf.append("\n");
offsets[i] = buf.length() - 1;
}
ChukwaArchiveKey key = new ChukwaArchiveKey(0, "datatype", "sname", 0);
ChunkImpl val = new ChunkImpl("datatype", "sname", 0, buf.toString().getBytes(), null);
val.setRecordOffsets(offsets);
sfw.append(key, val);
sfw.append(key, val); // write it twice
sfw.close();
long len = FileSystem.getLocal(conf).getFileStatus(filename).getLen();
InputSplit split = new FileSplit(filename, 0, len, (String[]) null);
ChukwaInputFormat in = new ChukwaInputFormat();
RecordReader<LongWritable, Text> r = in.getRecordReader(split, conf, Reporter.NULL);
LongWritable l = r.createKey();
Text line = r.createValue();
for (int i = 0; i < lines.length * 2; ++i) {
boolean succeeded = r.next(l, line);
assertTrue(succeeded);
assertEquals(i, l.get());
assertEquals(lines[i % lines.length], line.toString());
System.out.println("read line: " + l.get() + " " + line);
}
boolean succeeded = r.next(l, line);
assertFalse(succeeded);
} catch (IOException e) {
e.printStackTrace();
fail("IO exception " + e);
}
}
public byte[] getNext() throws Exception {
if (!m_rr.hasNext()) {
return null;
}
OrcStruct lv_row = (OrcStruct) m_rr.next(null);
Object lv_field_val = null;
ByteBuffer lv_row_buffer;
byte[] lv_row_ba = new byte[4096];
lv_row_buffer = ByteBuffer.wrap(lv_row_ba);
for (int i = 0; i < m_fields.size(); i++) {
lv_field_val = lv_row.getFieldValue(i);
if (lv_field_val == null) {
lv_row_buffer.putInt(0);
continue;
}
String lv_field_val_str = lv_field_val.toString();
lv_row_buffer.putInt(lv_field_val_str.length());
if (lv_field_val != null) {
lv_row_buffer.put(lv_field_val_str.getBytes());
}
}
System.out.println(lv_row_buffer);
return lv_row_buffer.array();
}
// Dumps the contents of the file as ByteBuffer.
public void readFile_ByteBuffer() throws Exception {
OrcStruct lv_row = null;
Object lv_field_val = null;
ByteBuffer lv_row_buffer;
while (m_rr.hasNext()) {
byte[] lv_row_ba = new byte[4096];
lv_row_buffer = ByteBuffer.wrap(lv_row_ba);
lv_row = (OrcStruct) m_rr.next(lv_row);
for (int i = 0; i < m_fields.size(); i++) {
lv_field_val = lv_row.getFieldValue(i);
if (lv_field_val == null) {
lv_row_buffer.putInt(0);
continue;
}
String lv_field_val_str = lv_field_val.toString();
lv_row_buffer.putInt(lv_field_val_str.length());
if (lv_field_val != null) {
lv_row_buffer.put(lv_field_val_str.getBytes());
}
}
System.out.println(lv_row_buffer);
// System.out.println(new String(lv_row_buffer.array()));
}
}
@Override
public String[] next() {
if (!hasNext()) {
throw new NoSuchElementException();
}
return reader.next();
}
public synchronized boolean next(K key, V value) throws IOException {
setProgress(getProgress());
long beforePos = getPos();
boolean ret = rawIn.next(key, value);
if (ret) {
inputRecordCounter.increment(1);
inputByteCounter.increment(getPos() - beforePos);
}
return ret;
}
private static List<Text> readSplit(TextInputFormat format, InputSplit split, JobConf job)
throws IOException {
List<Text> result = new ArrayList<Text>();
RecordReader<LongWritable, Text> reader = format.getRecordReader(split, job, voidReporter);
LongWritable key = reader.createKey();
Text value = reader.createValue();
while (reader.next(key, value)) {
result.add(value);
value = reader.createValue();
}
reader.close();
return result;
}
// Dumps the content of the file. The columns are '|' separated.
public void readFile_String() throws Exception {
OrcStruct lv_row = null;
Object lv_field_val = null;
StringBuilder lv_row_string = new StringBuilder(1024);
while (m_rr.hasNext()) {
lv_row = (OrcStruct) m_rr.next(lv_row);
lv_row_string.setLength(0);
for (int i = 0; i < m_fields.size(); i++) {
lv_field_val = lv_row.getFieldValue(i);
if (lv_field_val != null) {
lv_row_string.append(lv_field_val);
}
lv_row_string.append('|');
}
System.out.println(lv_row_string);
}
}
void next(OrcStruct next) throws IOException {
if (recordReader.hasNext()) {
nextRecord = (OrcStruct) recordReader.next(next);
// set the key
key.setValues(
OrcRecordUpdater.getOriginalTransaction(nextRecord),
OrcRecordUpdater.getBucket(nextRecord),
OrcRecordUpdater.getRowId(nextRecord),
OrcRecordUpdater.getCurrentTransaction(nextRecord));
// if this record is larger than maxKey, we need to stop
if (maxKey != null && key.compareRow(maxKey) > 0) {
LOG.debug("key " + key + " > maxkey " + maxKey);
nextRecord = null;
recordReader.close();
}
} else {
nextRecord = null;
recordReader.close();
}
}
@Test
public void testFormat() throws Exception {
JobConf job = new JobConf(defaultConf);
Path file = new Path(workDir, "test.txt");
// A reporter that does nothing
Reporter reporter = Reporter.NULL;
int seed = new Random().nextInt();
LOG.info("seed = " + seed);
Random random = new Random(seed);
localFs.delete(workDir, true);
FileInputFormat.setInputPaths(job, workDir);
// for a variety of lengths
for (int length = 0; length < MAX_LENGTH; length += random.nextInt(MAX_LENGTH / 10) + 1) {
LOG.debug("creating; entries = " + length);
// create a file with length entries
Writer writer = new OutputStreamWriter(localFs.create(file));
try {
for (int i = 0; i < length; i++) {
writer.write(Integer.toString(i));
writer.write("\n");
}
} finally {
writer.close();
}
// try splitting the file in a variety of sizes
TextInputFormat format = new TextInputFormat();
format.configure(job);
LongWritable key = new LongWritable();
Text value = new Text();
for (int i = 0; i < 3; i++) {
int numSplits = random.nextInt(MAX_LENGTH / 20) + 1;
LOG.debug("splitting: requesting = " + numSplits);
InputSplit[] splits = format.getSplits(job, numSplits);
LOG.debug("splitting: got = " + splits.length);
if (length == 0) {
assertEquals(
"Files of length 0 are not returned from FileInputFormat.getSplits().",
1,
splits.length);
assertEquals("Empty file length == 0", 0, splits[0].getLength());
}
// check each split
BitSet bits = new BitSet(length);
for (int j = 0; j < splits.length; j++) {
LOG.debug("split[" + j + "]= " + splits[j]);
RecordReader<LongWritable, Text> reader =
format.getRecordReader(splits[j], job, reporter);
try {
int count = 0;
while (reader.next(key, value)) {
int v = Integer.parseInt(value.toString());
LOG.debug("read " + v);
if (bits.get(v)) {
LOG.warn(
"conflict with " + v + " in split " + j + " at position " + reader.getPos());
}
assertFalse("Key in multiple partitions.", bits.get(v));
bits.set(v);
count++;
}
LOG.debug("splits[" + j + "]=" + splits[j] + " count=" + count);
} finally {
reader.close();
}
}
assertEquals("Some keys in no partition.", length, bits.cardinality());
}
}
}
@Test
public void testSplitableCodecs() throws IOException {
JobConf conf = new JobConf(defaultConf);
int seed = new Random().nextInt();
// Create the codec
CompressionCodec codec = null;
try {
codec =
(CompressionCodec)
ReflectionUtils.newInstance(
conf.getClassByName("org.apache.hadoop.io.compress.BZip2Codec"), conf);
} catch (ClassNotFoundException cnfe) {
throw new IOException("Illegal codec!");
}
Path file = new Path(workDir, "test" + codec.getDefaultExtension());
// A reporter that does nothing
Reporter reporter = Reporter.NULL;
LOG.info("seed = " + seed);
Random random = new Random(seed);
FileSystem localFs = FileSystem.getLocal(conf);
localFs.delete(workDir, true);
FileInputFormat.setInputPaths(conf, workDir);
final int MAX_LENGTH = 500000;
// for a variety of lengths
for (int length = MAX_LENGTH / 2;
length < MAX_LENGTH;
length += random.nextInt(MAX_LENGTH / 4) + 1) {
LOG.info("creating; entries = " + length);
// create a file with length entries
Writer writer = new OutputStreamWriter(codec.createOutputStream(localFs.create(file)));
try {
for (int i = 0; i < length; i++) {
writer.write(Integer.toString(i));
writer.write("\n");
}
} finally {
writer.close();
}
// try splitting the file in a variety of sizes
TextInputFormat format = new TextInputFormat();
format.configure(conf);
LongWritable key = new LongWritable();
Text value = new Text();
for (int i = 0; i < 3; i++) {
int numSplits = random.nextInt(MAX_LENGTH / 2000) + 1;
LOG.info("splitting: requesting = " + numSplits);
InputSplit[] splits = format.getSplits(conf, numSplits);
LOG.info("splitting: got = " + splits.length);
// check each split
BitSet bits = new BitSet(length);
for (int j = 0; j < splits.length; j++) {
LOG.debug("split[" + j + "]= " + splits[j]);
RecordReader<LongWritable, Text> reader =
format.getRecordReader(splits[j], conf, reporter);
try {
int counter = 0;
while (reader.next(key, value)) {
int v = Integer.parseInt(value.toString());
LOG.debug("read " + v);
if (bits.get(v)) {
LOG.warn(
"conflict with " + v + " in split " + j + " at position " + reader.getPos());
}
assertFalse("Key in multiple partitions.", bits.get(v));
bits.set(v);
counter++;
}
if (counter > 0) {
LOG.info("splits[" + j + "]=" + splits[j] + " count=" + counter);
} else {
LOG.debug("splits[" + j + "]=" + splits[j] + " count=" + counter);
}
} finally {
reader.close();
}
}
assertEquals("Some keys in no partition.", length, bits.cardinality());
}
}
}
public void testFormat() throws Exception {
JobConf job = new JobConf(conf);
FileSystem fs = FileSystem.getLocal(conf);
Path dir = new Path(System.getProperty("test.build.data", ".") + "/mapred");
Path file = new Path(dir, "test.seq");
Reporter reporter = Reporter.NULL;
int seed = new Random().nextInt();
// LOG.info("seed = "+seed);
Random random = new Random(seed);
fs.delete(dir, true);
FileInputFormat.setInputPaths(job, dir);
// for a variety of lengths
for (int length = 0; length < MAX_LENGTH; length += random.nextInt(MAX_LENGTH / 10) + 1) {
// LOG.info("creating; entries = " + length);
// create a file with length entries
SequenceFile.Writer writer =
SequenceFile.createWriter(fs, conf, file, IntWritable.class, BytesWritable.class);
try {
for (int i = 0; i < length; i++) {
IntWritable key = new IntWritable(i);
byte[] data = new byte[random.nextInt(10)];
random.nextBytes(data);
BytesWritable value = new BytesWritable(data);
writer.append(key, value);
}
} finally {
writer.close();
}
// try splitting the file in a variety of sizes
InputFormat<IntWritable, BytesWritable> format =
new SequenceFileInputFormat<IntWritable, BytesWritable>();
IntWritable key = new IntWritable();
BytesWritable value = new BytesWritable();
for (int i = 0; i < 3; i++) {
int numSplits = random.nextInt(MAX_LENGTH / (SequenceFile.SYNC_INTERVAL / 20)) + 1;
// LOG.info("splitting: requesting = " + numSplits);
InputSplit[] splits = format.getSplits(job, numSplits);
// LOG.info("splitting: got = " + splits.length);
// check each split
BitSet bits = new BitSet(length);
for (int j = 0; j < splits.length; j++) {
RecordReader<IntWritable, BytesWritable> reader =
format.getRecordReader(splits[j], job, reporter);
try {
int count = 0;
while (reader.next(key, value)) {
// if (bits.get(key.get())) {
// LOG.info("splits["+j+"]="+splits[j]+" : " +
// key.get());
// LOG.info("@"+reader.getPos());
// }
assertFalse("Key in multiple partitions.", bits.get(key.get()));
bits.set(key.get());
count++;
}
// LOG.info("splits["+j+"]="+splits[j]+" count=" +
// count);
} finally {
reader.close();
}
}
assertEquals("Some keys in no partition.", length, bits.cardinality());
}
}
}
// This method is just for experimentation.
public void testRead() throws Exception {
m_reader = OrcFile.createReader(m_file_path, OrcFile.readerOptions(m_conf));
System.out.println("Reader: " + m_reader);
System.out.println("# Rows: " + m_reader.getNumberOfRows());
m_types = m_reader.getTypes();
System.out.println("# Types in the file: " + m_types.size());
for (int i = 0; i < m_types.size(); i++) {
System.out.println("Type " + i + ": " + m_types.get(i).getKind());
}
System.out.println("Compression: " + m_reader.getCompression());
if (m_reader.getCompression() != CompressionKind.NONE) {
System.out.println("Compression size: " + m_reader.getCompressionSize());
}
StructObjectInspector m_oi = (StructObjectInspector) m_reader.getObjectInspector();
System.out.println("object inspector type category: " + m_oi.getCategory());
System.out.println("object inspector type name : " + m_oi.getTypeName());
m_fields = m_oi.getAllStructFieldRefs();
System.out.println("Number of columns in the table: " + m_fields.size());
RecordReader m_rr = m_reader.rows();
// Print the type info:
for (int i = 0; i < m_fields.size(); i++) {
System.out.println("Column " + i + " name: " + m_fields.get(i).getFieldName());
ObjectInspector lv_foi = m_fields.get(i).getFieldObjectInspector();
System.out.println("Column " + i + " type category: " + lv_foi.getCategory());
System.out.println("Column " + i + " type name: " + lv_foi.getTypeName());
// Object lv_column_val = m_oi.getStructFieldData(lv_row, m_fields.get(i));
// System.out.print("Column " + i + " value: " + lv_row.getFieldValue(i));
}
OrcStruct lv_row = null;
Object lv_field_val = null;
StringBuilder lv_row_string = new StringBuilder(1024);
while (m_rr.hasNext()) {
lv_row = (OrcStruct) m_rr.next(lv_row);
lv_row_string.setLength(0);
for (int i = 0; i < m_fields.size(); i++) {
lv_field_val = lv_row.getFieldValue(i);
if (lv_field_val != null) {
lv_row_string.append(lv_field_val);
}
lv_row_string.append('|');
}
System.out.println(lv_row_string);
}
/**
* Typecasting to appropriate type based on the 'kind' if (OrcProto.Type.Kind.INT ==
* m_types.get(1).getKind()) { IntWritable lvf_1_val = (IntWritable) lv_row.getFieldValue(0);
* System.out.println("Column 1 value: " + lvf_1_val); }
*/
}
