/** Test using the gzip codec for reading */
@Test
public void testGzip() throws IOException {
JobConf job = new JobConf(defaultConf);
CompressionCodec gzip = new GzipCodec();
ReflectionUtils.setConf(gzip, job);
localFs.delete(workDir, true);
writeFile(
localFs,
new Path(workDir, "part1.txt.gz"),
gzip,
"the quick\nbrown\nfox jumped\nover\n the lazy\n dog\n");
writeFile(localFs, new Path(workDir, "part2.txt.gz"), gzip, "this is a test\nof gzip\n");
FileInputFormat.setInputPaths(job, workDir);
TextInputFormat format = new TextInputFormat();
format.configure(job);
InputSplit[] splits = format.getSplits(job, 100);
assertEquals("compressed splits == 2", 2, splits.length);
FileSplit tmp = (FileSplit) splits[0];
if (tmp.getPath().getName().equals("part2.txt.gz")) {
splits[0] = splits[1];
splits[1] = tmp;
}
List<Text> results = readSplit(format, splits[0], job);
assertEquals("splits[0] length", 6, results.size());
assertEquals("splits[0][5]", " dog", results.get(5).toString());
results = readSplit(format, splits[1], job);
assertEquals("splits[1] length", 2, results.size());
assertEquals("splits[1][0]", "this is a test", results.get(0).toString());
assertEquals("splits[1][1]", "of gzip", results.get(1).toString());
}
/** Test using the gzip codec and an empty input file */
@Test
public void testGzipEmpty() throws IOException {
JobConf job = new JobConf(defaultConf);
CompressionCodec gzip = new GzipCodec();
ReflectionUtils.setConf(gzip, job);
localFs.delete(workDir, true);
writeFile(localFs, new Path(workDir, "empty.gz"), gzip, "");
FileInputFormat.setInputPaths(job, workDir);
TextInputFormat format = new TextInputFormat();
format.configure(job);
InputSplit[] splits = format.getSplits(job, 100);
assertEquals(
"Compressed files of length 0 are not returned from FileInputFormat.getSplits().",
1,
splits.length);
List<Text> results = readSplit(format, splits[0], job);
assertEquals("Compressed empty file length == 0", 0, results.size());
}
private static List<Text> readSplit(TextInputFormat format, InputSplit split, JobConf job)
throws IOException {
List<Text> result = new ArrayList<Text>();
RecordReader<LongWritable, Text> reader = format.getRecordReader(split, job, voidReporter);
LongWritable key = reader.createKey();
Text value = reader.createValue();
while (reader.next(key, value)) {
result.add(value);
value = reader.createValue();
}
reader.close();
return result;
}
public int run(String[] args) throws Exception {
if (args.length < 4) {
System.out.println("ERROR: Please Enter args : input output type(text|seq) splitChar(9=\t)");
return JobClient.SUCCESS;
}
String input = args[0];
String output = args[1];
String type = args[2];
String splitChar = args[3];
JobConf config = new JobConf(getConf(), getClass());
config.set("user.split", splitChar);
config.setJobName("File Filter -" + System.currentTimeMillis());
config.setNumReduceTasks(10);
config.setReducerClass(IdentityReducer.class);
config.setMapperClass(FileTestMapper.class);
if ("text".equals(type)) {
config.setInputFormat(TextInputFormat.class);
TextInputFormat.addInputPath(config, new Path(input));
} else {
config.setInputFormat(SequenceFileInputFormat.class);
SequenceFileInputFormat.addInputPath(config, new Path(input));
}
config.setMapOutputKeyClass(Text.class);
config.setMapOutputValueClass(Text.class);
config.setOutputKeyClass(Text.class);
config.setOutputValueClass(Text.class);
// if output path exists then return
FileSystem fs = FileSystem.get(config);
Path outputPath = new Path(output);
FileOutputFormat.setOutputPath(config, outputPath);
if (!fs.exists(outputPath)) {
JobClient.runJob(config);
} else {
System.out.println("You has finished this job today ! " + outputPath);
}
return JobClient.SUCCESS;
}
public int run(String[] args) throws Exception {
if (args.length < 1) {
args = new String[] {DateStringUtils.now()};
System.out.println(
"ERROR: Please Enter Date , eg. 20101010 ! now use default => " + DateStringUtils.now());
}
JobConf config = new JobConf(getConf(), getClass());
config.set("user.args", Utils.asString(args));
config.setJobName(getClass() + "-" + System.currentTimeMillis());
config.setNumReduceTasks(100);
config.setMapperClass(getClass());
config.setReducerClass(getClass());
config.setInputFormat(getInputFormat());
config.setMapOutputKeyClass(Text.class);
config.setMapOutputValueClass(Text.class);
// add input paths
for (String path : getInputPath(args)) {
if (TextInputFormat.class.equals(getInputFormat())) {
TextInputFormat.addInputPath(config, new Path(path));
} else if (SequenceFileInputFormat.class.equals(getInputFormat())) {
SequenceFileInputFormat.addInputPath(config, new Path(path));
}
}
config.setOutputKeyClass(Text.class);
config.setOutputValueClass(Text.class);
// if output path exists then return
FileSystem fs = FileSystem.get(config);
Path outputPath = new Path(getOutputPath(args));
FileOutputFormat.setOutputPath(config, outputPath);
if (!fs.exists(outputPath)) {
JobClient.runJob(config);
} else {
System.out.println("You has finished this job today ! " + outputPath);
}
return JobClient.SUCCESS;
}
@Test
public void testFormat() throws Exception {
JobConf job = new JobConf(defaultConf);
Path file = new Path(workDir, "test.txt");
// A reporter that does nothing
Reporter reporter = Reporter.NULL;
int seed = new Random().nextInt();
LOG.info("seed = " + seed);
Random random = new Random(seed);
localFs.delete(workDir, true);
FileInputFormat.setInputPaths(job, workDir);
// for a variety of lengths
for (int length = 0; length < MAX_LENGTH; length += random.nextInt(MAX_LENGTH / 10) + 1) {
LOG.debug("creating; entries = " + length);
// create a file with length entries
Writer writer = new OutputStreamWriter(localFs.create(file));
try {
for (int i = 0; i < length; i++) {
writer.write(Integer.toString(i));
writer.write("\n");
}
} finally {
writer.close();
}
// try splitting the file in a variety of sizes
TextInputFormat format = new TextInputFormat();
format.configure(job);
LongWritable key = new LongWritable();
Text value = new Text();
for (int i = 0; i < 3; i++) {
int numSplits = random.nextInt(MAX_LENGTH / 20) + 1;
LOG.debug("splitting: requesting = " + numSplits);
InputSplit[] splits = format.getSplits(job, numSplits);
LOG.debug("splitting: got = " + splits.length);
if (length == 0) {
assertEquals(
"Files of length 0 are not returned from FileInputFormat.getSplits().",
1,
splits.length);
assertEquals("Empty file length == 0", 0, splits[0].getLength());
}
// check each split
BitSet bits = new BitSet(length);
for (int j = 0; j < splits.length; j++) {
LOG.debug("split[" + j + "]= " + splits[j]);
RecordReader<LongWritable, Text> reader =
format.getRecordReader(splits[j], job, reporter);
try {
int count = 0;
while (reader.next(key, value)) {
int v = Integer.parseInt(value.toString());
LOG.debug("read " + v);
if (bits.get(v)) {
LOG.warn(
"conflict with " + v + " in split " + j + " at position " + reader.getPos());
}
assertFalse("Key in multiple partitions.", bits.get(v));
bits.set(v);
count++;
}
LOG.debug("splits[" + j + "]=" + splits[j] + " count=" + count);
} finally {
reader.close();
}
}
assertEquals("Some keys in no partition.", length, bits.cardinality());
}
}
}
@Test
public void testSplitableCodecs() throws IOException {
JobConf conf = new JobConf(defaultConf);
int seed = new Random().nextInt();
// Create the codec
CompressionCodec codec = null;
try {
codec =
(CompressionCodec)
ReflectionUtils.newInstance(
conf.getClassByName("org.apache.hadoop.io.compress.BZip2Codec"), conf);
} catch (ClassNotFoundException cnfe) {
throw new IOException("Illegal codec!");
}
Path file = new Path(workDir, "test" + codec.getDefaultExtension());
// A reporter that does nothing
Reporter reporter = Reporter.NULL;
LOG.info("seed = " + seed);
Random random = new Random(seed);
FileSystem localFs = FileSystem.getLocal(conf);
localFs.delete(workDir, true);
FileInputFormat.setInputPaths(conf, workDir);
final int MAX_LENGTH = 500000;
// for a variety of lengths
for (int length = MAX_LENGTH / 2;
length < MAX_LENGTH;
length += random.nextInt(MAX_LENGTH / 4) + 1) {
LOG.info("creating; entries = " + length);
// create a file with length entries
Writer writer = new OutputStreamWriter(codec.createOutputStream(localFs.create(file)));
try {
for (int i = 0; i < length; i++) {
writer.write(Integer.toString(i));
writer.write("\n");
}
} finally {
writer.close();
}
// try splitting the file in a variety of sizes
TextInputFormat format = new TextInputFormat();
format.configure(conf);
LongWritable key = new LongWritable();
Text value = new Text();
for (int i = 0; i < 3; i++) {
int numSplits = random.nextInt(MAX_LENGTH / 2000) + 1;
LOG.info("splitting: requesting = " + numSplits);
InputSplit[] splits = format.getSplits(conf, numSplits);
LOG.info("splitting: got = " + splits.length);
// check each split
BitSet bits = new BitSet(length);
for (int j = 0; j < splits.length; j++) {
LOG.debug("split[" + j + "]= " + splits[j]);
RecordReader<LongWritable, Text> reader =
format.getRecordReader(splits[j], conf, reporter);
try {
int counter = 0;
while (reader.next(key, value)) {
int v = Integer.parseInt(value.toString());
LOG.debug("read " + v);
if (bits.get(v)) {
LOG.warn(
"conflict with " + v + " in split " + j + " at position " + reader.getPos());
}
assertFalse("Key in multiple partitions.", bits.get(v));
bits.set(v);
counter++;
}
if (counter > 0) {
LOG.info("splits[" + j + "]=" + splits[j] + " count=" + counter);
} else {
LOG.debug("splits[" + j + "]=" + splits[j] + " count=" + counter);
}
} finally {
reader.close();
}
}
assertEquals("Some keys in no partition.", length, bits.cardinality());
}
}
}
/** @param jobConf */
@Override
public void configure(JobConf jobConf) {
super.configure(jobConf);
}