@Test
public void testBzipStoreInMultiQuery2() throws Exception {
String[] inputData = new String[] {"1\t2\r3\t4"};
String inputFileName = "input2.txt";
Util.createInputFile(cluster, inputFileName, inputData);
PigServer pig = new PigServer(ExecType.MAPREDUCE, cluster.getProperties());
PigContext pigContext = pig.getPigContext();
pigContext.getProperties().setProperty("output.compression.enabled", "true");
pigContext
.getProperties()
.setProperty("output.compression.codec", "org.apache.hadoop.io.compress.BZip2Codec");
pig.setBatchOn();
pig.registerQuery("a = load '" + inputFileName + "';");
pig.registerQuery("store a into 'output2.bz2';");
pig.registerQuery("store a into 'output2';");
pig.executeBatch();
FileSystem fs =
FileSystem.get(ConfigurationUtil.toConfiguration(pig.getPigContext().getProperties()));
FileStatus stat = fs.getFileStatus(new Path("output2/part-m-00000.bz2"));
assertTrue(stat.getLen() > 0);
stat = fs.getFileStatus(new Path("output2.bz2/part-m-00000.bz2"));
assertTrue(stat.getLen() > 0);
}
/** Tests the end-to-end writing and reading of a BZip file. */
@Test
public void testBzipInPig() throws Exception {
PigServer pig = new PigServer(ExecType.MAPREDUCE, cluster.getProperties());
File in = File.createTempFile("junit", ".bz2");
in.deleteOnExit();
File out = File.createTempFile("junit", ".bz2");
out.delete();
String clusterOutput = Util.removeColon(out.getAbsolutePath());
CBZip2OutputStream cos = new CBZip2OutputStream(new FileOutputStream(in));
for (int i = 1; i < 100; i++) {
StringBuffer sb = new StringBuffer();
sb.append(i).append("\n").append(-i).append("\n");
byte bytes[] = sb.toString().getBytes();
cos.write(bytes);
}
cos.close();
pig.registerQuery(
"AA = load '"
+ Util.generateURI(Util.encodeEscape(in.getAbsolutePath()), pig.getPigContext())
+ "';");
pig.registerQuery("A = foreach (group (filter AA by $0 > 0) all) generate flatten($1);");
pig.registerQuery("store A into '" + Util.encodeEscape(clusterOutput) + "';");
FileSystem fs =
FileSystem.get(ConfigurationUtil.toConfiguration(pig.getPigContext().getProperties()));
FSDataInputStream is = fs.open(new Path(clusterOutput + "/part-r-00000.bz2"));
CBZip2InputStream cis = new CBZip2InputStream(is, -1, out.length());
// Just a sanity check, to make sure it was a bzip file; we
// will do the value verification later
assertEquals(100, cis.read(new byte[100]));
cis.close();
pig.registerQuery("B = load '" + Util.encodeEscape(clusterOutput) + "';");
Iterator<Tuple> i = pig.openIterator("B");
HashMap<Integer, Integer> map = new HashMap<Integer, Integer>();
while (i.hasNext()) {
Integer val = DataType.toInteger(i.next().get(0));
map.put(val, val);
}
assertEquals(new Integer(99), new Integer(map.keySet().size()));
for (int j = 1; j < 100; j++) {
assertEquals(new Integer(j), map.get(j));
}
in.delete();
Util.deleteFile(cluster, clusterOutput);
}
public void testSkewedJoinManyReducers() throws IOException {
pigServer.getPigContext().getProperties().setProperty("pig.skewedjoin.reduce.maxtuple", "2");
pigServer.registerQuery("A = LOAD '" + INPUT_FILE6 + "' as (id,name);");
pigServer.registerQuery("B = LOAD '" + INPUT_FILE7 + "' as (id,name);");
DataBag dbfrj = BagFactory.getInstance().newDefaultBag(),
dbrj = BagFactory.getInstance().newDefaultBag();
{
pigServer.registerQuery("E = join A by id, B by id using \"skewed\" parallel 300;");
Iterator<Tuple> iter = pigServer.openIterator("E");
while (iter.hasNext()) {
dbfrj.add(iter.next());
}
}
{
pigServer.registerQuery("E = join A by id, B by id;");
Iterator<Tuple> iter = pigServer.openIterator("E");
while (iter.hasNext()) {
dbrj.add(iter.next());
}
}
Assert.assertEquals(dbfrj.size(), dbrj.size());
Assert.assertEquals(true, TestHelper.compareBags(dbfrj, dbrj));
}
// See PIG-1434
@Test
public void testScalarWithTwoBranches() throws Exception {
String[] inputA = {"1\t5", "2\t10", "3\t20"};
String[] inputX = {"pig", "hadoop", "rocks"};
String output = BUILD_TEST_TMP + "testScalarWithTwoBranchesDir";
TestScalarAliases.deleteDirectory(new File(output));
// Test the use of scalars in expressions
String inputPathA = BUILD_TEST_TMP + "testScalarWithTwoBranchesA";
TestScalarAliases.createLocalInputFile(inputPathA, inputA);
String inputPathX = BUILD_TEST_TMP + "testScalarWithTwoBranchesX";
TestScalarAliases.createLocalInputFile(inputPathX, inputX);
// Test in script mode
pigServer.setBatchOn();
pigServer.registerQuery("A = LOAD '" + inputPathA + "' as (a0: long, a1: double);");
pigServer.registerQuery("B = group A all;");
pigServer.registerQuery("C = foreach B generate COUNT(A) as count, MAX(A.$1) as max;");
pigServer.registerQuery("X = LOAD '" + inputPathX + "' as (names: chararray);");
pigServer.registerQuery("Y = foreach X generate names, C.max;");
pigServer.registerQuery("Store Y into '" + output + "';");
pigServer.executeBatch();
// Check output
pigServer.registerQuery("Z = LOAD '" + output + "' as (a0: chararray, a1: double);");
Iterator<Tuple> iter = pigServer.openIterator("Z");
Tuple t = iter.next();
assertTrue(t.toString().equals("(pig,20.0)"));
t = iter.next();
assertTrue(t.toString().equals("(hadoop,20.0)"));
t = iter.next();
assertTrue(t.toString().equals("(rocks,20.0)"));
assertFalse(iter.hasNext());
// Check in non-batch mode
iter = pigServer.openIterator("Y");
t = iter.next();
assertTrue(t.toString().equals("(pig,20.0)"));
t = iter.next();
assertTrue(t.toString().equals("(hadoop,20.0)"));
t = iter.next();
assertTrue(t.toString().equals("(rocks,20.0)"));
assertFalse(iter.hasNext());
pigServer.getPigContext().getProperties().remove("tez.am.inline.task.execution.max-tasks");
}
/** Tests the end-to-end writing and reading of an empty BZip file. */
@Test
public void testEmptyBzipInPig() throws Exception {
PigServer pig = new PigServer(ExecType.MAPREDUCE, cluster.getProperties());
File in = File.createTempFile("junit", ".tmp");
in.deleteOnExit();
File out = File.createTempFile("junit", ".bz2");
out.delete();
String clusterOutputFilePath = Util.removeColon(out.getAbsolutePath());
FileOutputStream fos = new FileOutputStream(in);
fos.write("55\n".getBytes());
fos.close();
System.out.println(in.getAbsolutePath());
pig.registerQuery(
"AA = load '"
+ Util.generateURI(Util.encodeEscape(in.getAbsolutePath()), pig.getPigContext())
+ "';");
pig.registerQuery("A=foreach (group (filter AA by $0 < '0') all) generate flatten($1);");
pig.registerQuery("store A into '" + Util.encodeEscape(clusterOutputFilePath) + "';");
FileSystem fs =
FileSystem.get(ConfigurationUtil.toConfiguration(pig.getPigContext().getProperties()));
FSDataInputStream is = fs.open(new Path(clusterOutputFilePath + "/part-r-00000.bz2"));
CBZip2InputStream cis = new CBZip2InputStream(is, -1, out.length());
// Just a sanity check, to make sure it was a bzip file; we
// will do the value verification later
assertEquals(-1, cis.read(new byte[100]));
cis.close();
pig.registerQuery("B = load '" + Util.encodeEscape(clusterOutputFilePath) + "';");
pig.openIterator("B");
in.delete();
Util.deleteFile(cluster, clusterOutputFilePath);
}
@Test
public void testDefineUDF() throws Exception {
PigServer ps = new PigServer(ExecType.LOCAL);
String inputData[] = {
"dshfdskfwww.xyz.com/sportsjoadfjdslpdshfdskfwww.xyz.com/sportsjoadfjdsl",
"kas;dka;sd",
"jsjsjwww.xyz.com/sports",
"jsdLSJDcom/sports",
"wwwJxyzMcom/sports"
};
File f = Util.createFile(inputData);
String[] queryLines =
new String[] {
// the reason we have 4 backslashes below is we really want to put two backslashes but
// since this is to be represented in a Java String, we escape each backslash with one
// more
// backslash - hence 4. In a pig script in a file, this would be
// www\\.xyz\\.com
"define minelogs org.apache.pig.test.RegexGroupCount('www\\\\.xyz\\\\.com/sports');",
"A = load '"
+ Util.generateURI(Util.encodeEscape(f.getAbsolutePath()), ps.getPigContext())
+ "' using PigStorage() as (source : chararray);",
"B = foreach A generate minelogs(source) as sportslogs;"
};
for (String line : queryLines) {
ps.registerQuery(line);
}
Iterator<Tuple> it = ps.openIterator("B");
int[] expectedResults = new int[] {2, 0, 1, 0, 0};
int i = 0;
while (it.hasNext()) {
Tuple t = it.next();
assertEquals(expectedResults[i++], t.get(0));
}
}
// See PIG-1714
@Test
public void testBzipStoreInMultiQuery3() throws Exception {
String[] inputData = new String[] {"1\t2\r3\t4"};
String inputFileName = "input3.txt";
Util.createInputFile(cluster, inputFileName, inputData);
String inputScript =
"set mapred.output.compress true\n"
+ "set mapreduce.output.fileoutputformat.compress true\n"
+ "set mapred.output.compression.codec org.apache.hadoop.io.compress.BZip2Codec\n"
+ "set mapreduce.output.fileoutputformat.compress.codec org.apache.hadoop.io.compress.BZip2Codec\n"
+ "a = load '"
+ inputFileName
+ "';\n"
+ "store a into 'output3.bz2';\n"
+ "store a into 'output3';";
String inputScriptName = "script3.txt";
PrintWriter pw = new PrintWriter(new FileWriter(inputScriptName));
pw.println(inputScript);
pw.close();
PigServer pig = new PigServer(ExecType.MAPREDUCE, cluster.getProperties());
FileInputStream fis = new FileInputStream(inputScriptName);
pig.registerScript(fis);
FileSystem fs =
FileSystem.get(ConfigurationUtil.toConfiguration(pig.getPigContext().getProperties()));
FileStatus stat = fs.getFileStatus(new Path("output3/part-m-00000.bz2"));
assertTrue(stat.getLen() > 0);
stat = fs.getFileStatus(new Path("output3.bz2/part-m-00000.bz2"));
assertTrue(stat.getLen() > 0);
}
public TestSkewedJoin() throws ExecException, IOException {
pigServer = new PigServer(ExecType.MAPREDUCE, cluster.getProperties());
// pigServer = new PigServer(ExecType.LOCAL);
pigServer.getPigContext().getProperties().setProperty("pig.skewedjoin.reduce.maxtuple", "5");
pigServer.getPigContext().getProperties().setProperty("pig.skewedjoin.reduce.memusage", "0.01");
}