@Test
public void testNewNormalNestedMerge2() throws Exception {
LogicalSchema a =
org.apache.pig.newplan.logical.Util.translateSchema(
Utils.getSchemaFromString(
"a1:(a11:chararray, a12:float), b1:(b11:chararray, b12:float), c1:long"));
LogicalSchema b =
org.apache.pig.newplan.logical.Util.translateSchema(
Utils.getSchemaFromString("a2:bytearray, b2:(b21:double, b22:long), c2:chararray"));
LogicalSchema mergedSchema = LogicalSchema.merge(a, b, LogicalSchema.MergeMode.Union);
LogicalSchema expected =
org.apache.pig.newplan.logical.Util.translateSchema(
Utils.getSchemaFromString("a1:(a11:chararray, a12:float), b1:(), c1:bytearray"));
expected.getField(1).schema = new LogicalSchema();
Assert.assertTrue(LogicalSchema.equals(mergedSchema, expected, false, false));
mergedSchema = LogicalSchema.merge(a, b, LogicalSchema.MergeMode.LoadForEach);
expected =
org.apache.pig.newplan.logical.Util.translateSchema(
Utils.getSchemaFromString(
"a1:(a11:chararray, a12:float), b1:(b11:chararray, b12:float), c1:long"));
Assert.assertTrue(LogicalSchema.equals(mergedSchema, expected, false, false));
mergedSchema = LogicalSchema.merge(b, a, LogicalSchema.MergeMode.LoadForEach);
expected =
org.apache.pig.newplan.logical.Util.translateSchema(
Utils.getSchemaFromString(
"a2:(a11:chararray, a12:float), b2:(b21:double, b22:long), c2:chararray"));
Assert.assertTrue(LogicalSchema.equals(mergedSchema, expected, false, false));
}
@Test
public void testNewMergeMismatchType2() throws Throwable {
LogicalSchema a =
org.apache.pig.newplan.logical.Util.translateSchema(
Utils.getSchemaFromString("a1:chararray, b1:(b11:double, b12:(b121:int)), c1:long"));
LogicalSchema b =
org.apache.pig.newplan.logical.Util.translateSchema(
Utils.getSchemaFromString("a2:bytearray, b2:(b21:double, b22:long), c2:int"));
LogicalSchema mergedSchema = LogicalSchema.merge(a, b, LogicalSchema.MergeMode.Union);
LogicalSchema expected =
org.apache.pig.newplan.logical.Util.translateSchema(
Utils.getSchemaFromString("a1:chararray, b1:(), c1:long"));
expected.getField(1).schema = new LogicalSchema();
Assert.assertTrue(LogicalSchema.equals(mergedSchema, expected, false, false));
try {
LogicalSchema.merge(a, b, LogicalSchema.MergeMode.LoadForEach);
Assert.fail();
} catch (FrontendException e) {
Assert.assertTrue(e.getErrorCode() == 1031);
}
try {
LogicalSchema.merge(b, a, LogicalSchema.MergeMode.LoadForEach);
Assert.fail();
} catch (FrontendException e) {
Assert.assertTrue(e.getErrorCode() == 1031);
}
}
@Test
// See PIG-730
public void testMergeSchemaWithTwoLevelAccess() throws Exception {
// Generate two schemas
Schema s1 = Utils.getSchemaFromString("a:{t:(a0:int, a1:int)}");
Schema s2 = Utils.getSchemaFromString("b:{t:(b0:int, b1:int)}");
s1.getField(0).schema.setTwoLevelAccessRequired(true);
s1.getField(0).schema.setTwoLevelAccessRequired(false);
Schema s3 = Schema.mergeSchema(s1, s2, true);
Assert.assertEquals(s3, s2);
}
// PIG-2146
@Test
public void testSchemaInStoreForDistinctLimit() throws Exception {
// test if the POStore in the 2nd mr plan (that stores the actual output)
// has a schema
String query =
"a = load 'input1' as (a : int,b :float ,c : int);"
+ "b = distinct a;"
+ "c = limit b 10;"
+ "store c into 'output';";
PhysicalPlan pp = Util.buildPp(pigServer, query);
MROperPlan mrPlan = Util.buildMRPlan(pp, pc);
MapReduceOper secondMrOper = mrPlan.getLeaves().get(0);
POStore store = (POStore) secondMrOper.reducePlan.getLeaves().get(0);
assertEquals(
"compare load and store schema",
store.getSchema(),
Utils.getSchemaFromString("a : int,b :float ,c : int"));
}
public void prepareToWrite(RecordWriter writer) {
// Get the schema string from the UDFContext object.
UDFContext udfc = UDFContext.getUDFContext();
Properties p = udfc.getUDFProperties(this.getClass(), new String[] {udfContextSignature});
String strSchema = p.getProperty(SCHEMA_SIGNATURE);
if (strSchema != null) {
// Parse the schema from the string stored in the properties object.
try {
schema = new ResourceSchema(Utils.getSchemaFromString(strSchema));
} catch (ParserException pex) {
logger.warn("Could not parse schema for storing.");
}
}
if (headerTreatment == Headers.DEFAULT) {
headerTreatment = Headers.SKIP_OUTPUT_HEADER;
}
// PigStorage's prepareToWrite()
super.prepareToWrite(writer);
}