예제 #1
0
  @Test
  public void testNewMergeMismatchType2() throws Throwable {
    LogicalSchema a =
        org.apache.pig.newplan.logical.Util.translateSchema(
            Utils.getSchemaFromString("a1:chararray, b1:(b11:double, b12:(b121:int)), c1:long"));
    LogicalSchema b =
        org.apache.pig.newplan.logical.Util.translateSchema(
            Utils.getSchemaFromString("a2:bytearray, b2:(b21:double, b22:long), c2:int"));

    LogicalSchema mergedSchema = LogicalSchema.merge(a, b, LogicalSchema.MergeMode.Union);
    LogicalSchema expected =
        org.apache.pig.newplan.logical.Util.translateSchema(
            Utils.getSchemaFromString("a1:chararray, b1:(), c1:long"));
    expected.getField(1).schema = new LogicalSchema();
    Assert.assertTrue(LogicalSchema.equals(mergedSchema, expected, false, false));

    try {
      LogicalSchema.merge(a, b, LogicalSchema.MergeMode.LoadForEach);
      Assert.fail();
    } catch (FrontendException e) {
      Assert.assertTrue(e.getErrorCode() == 1031);
    }

    try {
      LogicalSchema.merge(b, a, LogicalSchema.MergeMode.LoadForEach);
      Assert.fail();
    } catch (FrontendException e) {
      Assert.assertTrue(e.getErrorCode() == 1031);
    }
  }
예제 #2
0
 @Override
 public Schema getOutputSchema(Schema input) {
   ArrayList<String> bagNames = new ArrayList<String>(input.size() / 2);
   Map<String, String> bagNameToJoinPrefix = new HashMap<String, String>(input.size() / 2);
   Map<String, Integer> bagNameToSize = new HashMap<String, Integer>(input.size() / 2);
   Schema outputSchema = null;
   Schema bagSchema = new Schema();
   try {
     int i = 0;
     // all even fields should be bags, odd fields are key names
     String bagName = null;
     String tupleName = null;
     for (FieldSchema outerField : input.getFields()) {
       if (i++ % 2 == 1) continue;
       bagName = outerField.alias;
       bagNames.add(bagName);
       if (bagName == null) bagName = "null";
       if (outerField.schema == null)
         throw new RuntimeException(
             "Expected input format of (bag, 'field') pairs. "
                 + "Did not receive a bag at index: "
                 + i
                 + ", alias: "
                 + bagName
                 + ". "
                 + "Instead received type: "
                 + DataType.findTypeName(outerField.type)
                 + " in schema:"
                 + input.toString());
       FieldSchema tupleField = outerField.schema.getField(0);
       tupleName = tupleField.alias;
       bagNameToJoinPrefix.put(bagName, getPrefixedAliasName(outerField.alias, tupleName));
       if (tupleField.schema == null) {
         log.error(
             String.format(
                 "could not get schema for inner tuple %s in bag %s", tupleName, bagName));
       } else {
         bagNameToSize.put(bagName, tupleField.schema.size());
         for (FieldSchema innerField : tupleField.schema.getFields()) {
           String innerFieldName = innerField.alias;
           if (innerFieldName == null) innerFieldName = "null";
           String outputFieldName = bagName + "::" + innerFieldName;
           bagSchema.add(new FieldSchema(outputFieldName, innerField.type));
         }
       }
     }
     outputSchema = new Schema(new Schema.FieldSchema("joined", bagSchema, DataType.BAG));
     log.debug("output schema: " + outputSchema.toString());
   } catch (FrontendException e) {
     e.printStackTrace();
     throw new RuntimeException(e);
   }
   Properties properties = getInstanceProperties();
   properties.put(BAG_NAMES_PROPERTY, bagNames);
   properties.put(BAG_NAME_TO_JOIN_PREFIX_PROPERTY, bagNameToJoinPrefix);
   properties.put(BAG_NAME_TO_SIZE_PROPERTY, bagNameToSize);
   return outputSchema;
 }
예제 #3
0
 @Test
 public void testErrorMessageUndefinedAliasInGroupByStatement() throws Exception {
   String queryA = "A = load 'nosuchfile'  using PigStorage() as (f1:chararray,f2:chararray);";
   String queryB = "B = GROUP B by f1;";
   PigServer ps = new PigServer(ExecType.LOCAL);
   ps.registerQuery(queryA);
   try {
     ps.registerQuery(queryB);
   } catch (FrontendException e) {
     Assert.assertTrue(e.getMessage().contains("Undefined alias:"));
     return;
   }
   Assert.fail();
 }
예제 #4
0
  @Test
  public void testNewMergeDifferentSize1() throws Throwable {
    LogicalSchema a = Utils.parseSchema("a1:bytearray, b1:long, c1:long");
    LogicalSchema b = Utils.parseSchema("a2:bytearray, b2:long");

    LogicalSchema mergedSchema = LogicalSchema.merge(a, b, LogicalSchema.MergeMode.Union);
    Assert.assertTrue(mergedSchema == null);

    try {
      LogicalSchema.merge(a, b, LogicalSchema.MergeMode.LoadForEach);
      Assert.fail();
    } catch (FrontendException e) {
      Assert.assertTrue(e.getErrorCode() == 1031);
    }
  }
예제 #5
0
 @Test
 public void testSplitWithNotEvalCondition() throws Exception {
   String defineQ =
       "define minelogs org.apache.pig.test.RegexGroupCount('www\\\\.xyz\\\\.com/sports');";
   String defineL = "a = load 'nosuchfile' " + " using PigStorage() as (source : chararray);";
   String defineSplit =
       "SPLIT a INTO a1 IF (minelogs(source) > 0 ), a2 IF (NOT (minelogs(source)>0));"; //    (NOT
                                                                                        // (
                                                                                        // minelogs(source) ) > 0) ;";
   PigServer ps = new PigServer(ExecType.LOCAL);
   ps.registerQuery(defineQ);
   ps.registerQuery(defineL);
   try {
     ps.registerQuery(defineSplit);
   } catch (FrontendException e) {
     Assert.fail(e.getMessage());
   }
 }
  @Override
  public Schema outputSchema(Schema input) {
    try {
      Schema.FieldSchema inputFieldSchema = input.getField(0);

      if (inputFieldSchema.type != DataType.BAG) {
        throw new RuntimeException("Expected a BAG as input");
      }

      return new Schema(
          new Schema.FieldSchema(
              getSchemaName(this.getClass().getName().toLowerCase(), input),
              inputFieldSchema.schema,
              DataType.BAG));
    } catch (FrontendException e) {
      e.printStackTrace();
      throw new RuntimeException(e);
    }
  }
예제 #7
0
파일: TestMain.java 프로젝트: scr/pig
  @Test
  public void testRun_setsErrorThrowableOnPigStats() {
    File outputFile = null;
    try {
      String filename =
          this.getClass().getSimpleName() + "_" + "testRun_setsErrorThrowableOnPigStats";
      outputFile = File.createTempFile(filename, ".out");
      outputFile.delete();

      File scriptFile = File.createTempFile(filename, ".pig");
      BufferedWriter bw = new BufferedWriter(new FileWriter(scriptFile));
      bw.write("a = load 'test/org/apache/pig/test/data/passwd';\n");
      bw.write("b = group a by $0\n");
      bw.write("c = foreach b generate group, COUNT(a) as cnt;\n");
      bw.write("store c into 'out'\n");
      bw.close();

      Main.run(new String[] {"-x", "local", scriptFile.getAbsolutePath()}, null);
      PigStats stats = PigStats.get();

      Throwable t = stats.getErrorThrowable();
      assertTrue(t instanceof FrontendException);

      FrontendException fe = (FrontendException) t;
      SourceLocation sl = fe.getSourceLocation();
      assertEquals(2, sl.line());
      assertEquals(15, sl.offset());

      Throwable cause = fe.getCause();
      assertTrue(cause instanceof ParserException);

    } catch (Exception e) {
      log.error("Encountered exception", e);
      fail("Encountered Exception");
    }
  }