示例#1
0
  @Test
  public void testNewNormalNestedMerge2() throws Exception {
    LogicalSchema a =
        org.apache.pig.newplan.logical.Util.translateSchema(
            Utils.getSchemaFromString(
                "a1:(a11:chararray, a12:float), b1:(b11:chararray, b12:float), c1:long"));
    LogicalSchema b =
        org.apache.pig.newplan.logical.Util.translateSchema(
            Utils.getSchemaFromString("a2:bytearray, b2:(b21:double, b22:long), c2:chararray"));

    LogicalSchema mergedSchema = LogicalSchema.merge(a, b, LogicalSchema.MergeMode.Union);
    LogicalSchema expected =
        org.apache.pig.newplan.logical.Util.translateSchema(
            Utils.getSchemaFromString("a1:(a11:chararray, a12:float), b1:(), c1:bytearray"));
    expected.getField(1).schema = new LogicalSchema();
    Assert.assertTrue(LogicalSchema.equals(mergedSchema, expected, false, false));

    mergedSchema = LogicalSchema.merge(a, b, LogicalSchema.MergeMode.LoadForEach);
    expected =
        org.apache.pig.newplan.logical.Util.translateSchema(
            Utils.getSchemaFromString(
                "a1:(a11:chararray, a12:float), b1:(b11:chararray, b12:float), c1:long"));
    Assert.assertTrue(LogicalSchema.equals(mergedSchema, expected, false, false));

    mergedSchema = LogicalSchema.merge(b, a, LogicalSchema.MergeMode.LoadForEach);
    expected =
        org.apache.pig.newplan.logical.Util.translateSchema(
            Utils.getSchemaFromString(
                "a2:(a11:chararray, a12:float), b2:(b21:double, b22:long), c2:chararray"));
    Assert.assertTrue(LogicalSchema.equals(mergedSchema, expected, false, false));
  }
示例#2
0
  @Test
  public void testNewMergeMismatchType2() throws Throwable {
    LogicalSchema a =
        org.apache.pig.newplan.logical.Util.translateSchema(
            Utils.getSchemaFromString("a1:chararray, b1:(b11:double, b12:(b121:int)), c1:long"));
    LogicalSchema b =
        org.apache.pig.newplan.logical.Util.translateSchema(
            Utils.getSchemaFromString("a2:bytearray, b2:(b21:double, b22:long), c2:int"));

    LogicalSchema mergedSchema = LogicalSchema.merge(a, b, LogicalSchema.MergeMode.Union);
    LogicalSchema expected =
        org.apache.pig.newplan.logical.Util.translateSchema(
            Utils.getSchemaFromString("a1:chararray, b1:(), c1:long"));
    expected.getField(1).schema = new LogicalSchema();
    Assert.assertTrue(LogicalSchema.equals(mergedSchema, expected, false, false));

    try {
      LogicalSchema.merge(a, b, LogicalSchema.MergeMode.LoadForEach);
      Assert.fail();
    } catch (FrontendException e) {
      Assert.assertTrue(e.getErrorCode() == 1031);
    }

    try {
      LogicalSchema.merge(b, a, LogicalSchema.MergeMode.LoadForEach);
      Assert.fail();
    } catch (FrontendException e) {
      Assert.assertTrue(e.getErrorCode() == 1031);
    }
  }
示例#3
0
 @Test
 // See PIG-730
 public void testMergeSchemaWithTwoLevelAccess() throws Exception {
   // Generate two schemas
   Schema s1 = Utils.getSchemaFromString("a:{t:(a0:int, a1:int)}");
   Schema s2 = Utils.getSchemaFromString("b:{t:(b0:int, b1:int)}");
   s1.getField(0).schema.setTwoLevelAccessRequired(true);
   s1.getField(0).schema.setTwoLevelAccessRequired(false);
   Schema s3 = Schema.mergeSchema(s1, s2, true);
   Assert.assertEquals(s3, s2);
 }
示例#4
0
  // PIG-2146
  @Test
  public void testSchemaInStoreForDistinctLimit() throws Exception {
    // test if the POStore in the 2nd mr plan (that stores the actual output)
    // has a schema
    String query =
        "a = load 'input1' as (a : int,b :float ,c : int);"
            + "b  = distinct a;"
            + "c = limit b 10;"
            + "store c into 'output';";

    PhysicalPlan pp = Util.buildPp(pigServer, query);
    MROperPlan mrPlan = Util.buildMRPlan(pp, pc);
    MapReduceOper secondMrOper = mrPlan.getLeaves().get(0);
    POStore store = (POStore) secondMrOper.reducePlan.getLeaves().get(0);
    assertEquals(
        "compare load and store schema",
        store.getSchema(),
        Utils.getSchemaFromString("a : int,b :float ,c : int"));
  }
示例#5
0
  public void prepareToWrite(RecordWriter writer) {
    // Get the schema string from the UDFContext object.
    UDFContext udfc = UDFContext.getUDFContext();
    Properties p = udfc.getUDFProperties(this.getClass(), new String[] {udfContextSignature});

    String strSchema = p.getProperty(SCHEMA_SIGNATURE);
    if (strSchema != null) {
      // Parse the schema from the string stored in the properties object.
      try {
        schema = new ResourceSchema(Utils.getSchemaFromString(strSchema));
      } catch (ParserException pex) {
        logger.warn("Could not parse schema for storing.");
      }
    }

    if (headerTreatment == Headers.DEFAULT) {
      headerTreatment = Headers.SKIP_OUTPUT_HEADER;
    }

    // PigStorage's prepareToWrite()
    super.prepareToWrite(writer);
  }