/**
* Tests that '\n', '\r' and '\r\n' are treated as record delims when using bzip just like they
* are when using uncompressed text
*/
@Test
public void testRecordDelims() throws Exception {
String[] inputData =
new String[] {
"1\t2\r3\t4", // '\r' case - this will be split into two tuples
"5\t6\r", // '\r\n' case
"7\t8", // '\n' case
"9\t10\r" // '\r\n' at the end of file
};
// bzip compressed input
File in = File.createTempFile("junit", ".bz2");
String compressedInputFileName = in.getAbsolutePath();
in.deleteOnExit();
String clusterCompressedFilePath = Util.removeColon(compressedInputFileName);
String unCompressedInputFileName = "testRecordDelims-uncomp.txt";
Util.createInputFile(cluster, unCompressedInputFileName, inputData);
try {
CBZip2OutputStream cos = new CBZip2OutputStream(new FileOutputStream(in));
for (int i = 0; i < inputData.length; i++) {
StringBuffer sb = new StringBuffer();
sb.append(inputData[i]).append("\n");
byte bytes[] = sb.toString().getBytes();
cos.write(bytes);
}
cos.close();
Util.copyFromLocalToCluster(cluster, compressedInputFileName, clusterCompressedFilePath);
// pig script to read uncompressed input
String script = "a = load '" + unCompressedInputFileName + "';";
PigServer pig = new PigServer(ExecType.MAPREDUCE, cluster.getProperties());
pig.registerQuery(script);
Iterator<Tuple> it1 = pig.openIterator("a");
// pig script to read compressed input
script = "a = load '" + Util.encodeEscape(clusterCompressedFilePath) + "';";
pig.registerQuery(script);
Iterator<Tuple> it2 = pig.openIterator("a");
while (it1.hasNext()) {
Tuple t1 = it1.next();
Tuple t2 = it2.next();
Assert.assertEquals(t1, t2);
}
Assert.assertFalse(it2.hasNext());
} finally {
in.delete();
Util.deleteFile(cluster, unCompressedInputFileName);
Util.deleteFile(cluster, clusterCompressedFilePath);
}
}
/** Tests the end-to-end writing and reading of a BZip file. */
@Test
public void testBzipInPig() throws Exception {
PigServer pig = new PigServer(ExecType.MAPREDUCE, cluster.getProperties());
File in = File.createTempFile("junit", ".bz2");
in.deleteOnExit();
File out = File.createTempFile("junit", ".bz2");
out.delete();
String clusterOutput = Util.removeColon(out.getAbsolutePath());
CBZip2OutputStream cos = new CBZip2OutputStream(new FileOutputStream(in));
for (int i = 1; i < 100; i++) {
StringBuffer sb = new StringBuffer();
sb.append(i).append("\n").append(-i).append("\n");
byte bytes[] = sb.toString().getBytes();
cos.write(bytes);
}
cos.close();
pig.registerQuery(
"AA = load '"
+ Util.generateURI(Util.encodeEscape(in.getAbsolutePath()), pig.getPigContext())
+ "';");
pig.registerQuery("A = foreach (group (filter AA by $0 > 0) all) generate flatten($1);");
pig.registerQuery("store A into '" + Util.encodeEscape(clusterOutput) + "';");
FileSystem fs =
FileSystem.get(ConfigurationUtil.toConfiguration(pig.getPigContext().getProperties()));
FSDataInputStream is = fs.open(new Path(clusterOutput + "/part-r-00000.bz2"));
CBZip2InputStream cis = new CBZip2InputStream(is, -1, out.length());
// Just a sanity check, to make sure it was a bzip file; we
// will do the value verification later
assertEquals(100, cis.read(new byte[100]));
cis.close();
pig.registerQuery("B = load '" + Util.encodeEscape(clusterOutput) + "';");
Iterator<Tuple> i = pig.openIterator("B");
HashMap<Integer, Integer> map = new HashMap<Integer, Integer>();
while (i.hasNext()) {
Integer val = DataType.toInteger(i.next().get(0));
map.put(val, val);
}
assertEquals(new Integer(99), new Integer(map.keySet().size()));
for (int j = 1; j < 100; j++) {
assertEquals(new Integer(j), map.get(j));
}
in.delete();
Util.deleteFile(cluster, clusterOutput);
}
/**
* Tests that Pig throws an Exception when the input files to be loaded are actually a result of
* concatenating 2 or more bz2 files. Pig should not silently ignore part of the input data.
*/
@Test(expected = IOException.class)
public void testBZ2Concatenation() throws Exception {
String[] inputData1 = new String[] {"1\ta", "2\taa"};
String[] inputData2 = new String[] {"1\tb", "2\tbb"};
String[] inputDataMerged = new String[] {"1\ta", "2\taa", "1\tb", "2\tbb"};
// bzip compressed input file1
File in1 = File.createTempFile("junit", ".bz2");
String compressedInputFileName1 = in1.getAbsolutePath();
in1.deleteOnExit();
// file2
File in2 = File.createTempFile("junit", ".bz2");
String compressedInputFileName2 = in2.getAbsolutePath();
in1.deleteOnExit();
String unCompressedInputFileName = "testRecordDelims-uncomp.txt";
Util.createInputFile(cluster, unCompressedInputFileName, inputDataMerged);
try {
CBZip2OutputStream cos = new CBZip2OutputStream(new FileOutputStream(in1));
for (int i = 0; i < inputData1.length; i++) {
StringBuffer sb = new StringBuffer();
sb.append(inputData1[i]).append("\n");
byte bytes[] = sb.toString().getBytes();
cos.write(bytes);
}
cos.close();
CBZip2OutputStream cos2 = new CBZip2OutputStream(new FileOutputStream(in2));
for (int i = 0; i < inputData2.length; i++) {
StringBuffer sb = new StringBuffer();
sb.append(inputData2[i]).append("\n");
byte bytes[] = sb.toString().getBytes();
cos2.write(bytes);
}
cos2.close();
// cat
catInto(compressedInputFileName2, compressedInputFileName1);
Util.copyFromLocalToCluster(cluster, compressedInputFileName1, compressedInputFileName1);
// pig script to read uncompressed input
String script = "a = load '" + Util.encodeEscape(unCompressedInputFileName) + "';";
PigServer pig = new PigServer(ExecType.MAPREDUCE, cluster.getProperties());
pig.registerQuery(script);
Iterator<Tuple> it1 = pig.openIterator("a");
// pig script to read compressed concatenated input
script = "a = load '" + Util.encodeEscape(compressedInputFileName1) + "';";
pig.registerQuery(script);
Iterator<Tuple> it2 = pig.openIterator("a");
while (it1.hasNext()) {
Tuple t1 = it1.next();
Tuple t2 = it2.next();
Assert.assertEquals(t1, t2);
}
Assert.assertFalse(it2.hasNext());
} finally {
in1.delete();
in2.delete();
Util.deleteFile(cluster, unCompressedInputFileName);
}
}