@Test
public void testNewMergeMismatchType2() throws Throwable {
LogicalSchema a =
org.apache.pig.newplan.logical.Util.translateSchema(
Utils.getSchemaFromString("a1:chararray, b1:(b11:double, b12:(b121:int)), c1:long"));
LogicalSchema b =
org.apache.pig.newplan.logical.Util.translateSchema(
Utils.getSchemaFromString("a2:bytearray, b2:(b21:double, b22:long), c2:int"));
LogicalSchema mergedSchema = LogicalSchema.merge(a, b, LogicalSchema.MergeMode.Union);
LogicalSchema expected =
org.apache.pig.newplan.logical.Util.translateSchema(
Utils.getSchemaFromString("a1:chararray, b1:(), c1:long"));
expected.getField(1).schema = new LogicalSchema();
Assert.assertTrue(LogicalSchema.equals(mergedSchema, expected, false, false));
try {
LogicalSchema.merge(a, b, LogicalSchema.MergeMode.LoadForEach);
Assert.fail();
} catch (FrontendException e) {
Assert.assertTrue(e.getErrorCode() == 1031);
}
try {
LogicalSchema.merge(b, a, LogicalSchema.MergeMode.LoadForEach);
Assert.fail();
} catch (FrontendException e) {
Assert.assertTrue(e.getErrorCode() == 1031);
}
}
@Override
public Schema getOutputSchema(Schema input) {
ArrayList<String> bagNames = new ArrayList<String>(input.size() / 2);
Map<String, String> bagNameToJoinPrefix = new HashMap<String, String>(input.size() / 2);
Map<String, Integer> bagNameToSize = new HashMap<String, Integer>(input.size() / 2);
Schema outputSchema = null;
Schema bagSchema = new Schema();
try {
int i = 0;
// all even fields should be bags, odd fields are key names
String bagName = null;
String tupleName = null;
for (FieldSchema outerField : input.getFields()) {
if (i++ % 2 == 1) continue;
bagName = outerField.alias;
bagNames.add(bagName);
if (bagName == null) bagName = "null";
if (outerField.schema == null)
throw new RuntimeException(
"Expected input format of (bag, 'field') pairs. "
+ "Did not receive a bag at index: "
+ i
+ ", alias: "
+ bagName
+ ". "
+ "Instead received type: "
+ DataType.findTypeName(outerField.type)
+ " in schema:"
+ input.toString());
FieldSchema tupleField = outerField.schema.getField(0);
tupleName = tupleField.alias;
bagNameToJoinPrefix.put(bagName, getPrefixedAliasName(outerField.alias, tupleName));
if (tupleField.schema == null) {
log.error(
String.format(
"could not get schema for inner tuple %s in bag %s", tupleName, bagName));
} else {
bagNameToSize.put(bagName, tupleField.schema.size());
for (FieldSchema innerField : tupleField.schema.getFields()) {
String innerFieldName = innerField.alias;
if (innerFieldName == null) innerFieldName = "null";
String outputFieldName = bagName + "::" + innerFieldName;
bagSchema.add(new FieldSchema(outputFieldName, innerField.type));
}
}
}
outputSchema = new Schema(new Schema.FieldSchema("joined", bagSchema, DataType.BAG));
log.debug("output schema: " + outputSchema.toString());
} catch (FrontendException e) {
e.printStackTrace();
throw new RuntimeException(e);
}
Properties properties = getInstanceProperties();
properties.put(BAG_NAMES_PROPERTY, bagNames);
properties.put(BAG_NAME_TO_JOIN_PREFIX_PROPERTY, bagNameToJoinPrefix);
properties.put(BAG_NAME_TO_SIZE_PROPERTY, bagNameToSize);
return outputSchema;
}
@Test
public void testErrorMessageUndefinedAliasInGroupByStatement() throws Exception {
String queryA = "A = load 'nosuchfile' using PigStorage() as (f1:chararray,f2:chararray);";
String queryB = "B = GROUP B by f1;";
PigServer ps = new PigServer(ExecType.LOCAL);
ps.registerQuery(queryA);
try {
ps.registerQuery(queryB);
} catch (FrontendException e) {
Assert.assertTrue(e.getMessage().contains("Undefined alias:"));
return;
}
Assert.fail();
}
@Test
public void testNewMergeDifferentSize1() throws Throwable {
LogicalSchema a = Utils.parseSchema("a1:bytearray, b1:long, c1:long");
LogicalSchema b = Utils.parseSchema("a2:bytearray, b2:long");
LogicalSchema mergedSchema = LogicalSchema.merge(a, b, LogicalSchema.MergeMode.Union);
Assert.assertTrue(mergedSchema == null);
try {
LogicalSchema.merge(a, b, LogicalSchema.MergeMode.LoadForEach);
Assert.fail();
} catch (FrontendException e) {
Assert.assertTrue(e.getErrorCode() == 1031);
}
}
@Test
public void testSplitWithNotEvalCondition() throws Exception {
String defineQ =
"define minelogs org.apache.pig.test.RegexGroupCount('www\\\\.xyz\\\\.com/sports');";
String defineL = "a = load 'nosuchfile' " + " using PigStorage() as (source : chararray);";
String defineSplit =
"SPLIT a INTO a1 IF (minelogs(source) > 0 ), a2 IF (NOT (minelogs(source)>0));"; // (NOT
// (
// minelogs(source) ) > 0) ;";
PigServer ps = new PigServer(ExecType.LOCAL);
ps.registerQuery(defineQ);
ps.registerQuery(defineL);
try {
ps.registerQuery(defineSplit);
} catch (FrontendException e) {
Assert.fail(e.getMessage());
}
}
@Override
public Schema outputSchema(Schema input) {
try {
Schema.FieldSchema inputFieldSchema = input.getField(0);
if (inputFieldSchema.type != DataType.BAG) {
throw new RuntimeException("Expected a BAG as input");
}
return new Schema(
new Schema.FieldSchema(
getSchemaName(this.getClass().getName().toLowerCase(), input),
inputFieldSchema.schema,
DataType.BAG));
} catch (FrontendException e) {
e.printStackTrace();
throw new RuntimeException(e);
}
}
@Test
public void testRun_setsErrorThrowableOnPigStats() {
File outputFile = null;
try {
String filename =
this.getClass().getSimpleName() + "_" + "testRun_setsErrorThrowableOnPigStats";
outputFile = File.createTempFile(filename, ".out");
outputFile.delete();
File scriptFile = File.createTempFile(filename, ".pig");
BufferedWriter bw = new BufferedWriter(new FileWriter(scriptFile));
bw.write("a = load 'test/org/apache/pig/test/data/passwd';\n");
bw.write("b = group a by $0\n");
bw.write("c = foreach b generate group, COUNT(a) as cnt;\n");
bw.write("store c into 'out'\n");
bw.close();
Main.run(new String[] {"-x", "local", scriptFile.getAbsolutePath()}, null);
PigStats stats = PigStats.get();
Throwable t = stats.getErrorThrowable();
assertTrue(t instanceof FrontendException);
FrontendException fe = (FrontendException) t;
SourceLocation sl = fe.getSourceLocation();
assertEquals(2, sl.line());
assertEquals(15, sl.offset());
Throwable cause = fe.getCause();
assertTrue(cause instanceof ParserException);
} catch (Exception e) {
log.error("Encountered exception", e);
fail("Encountered Exception");
}
}