示例#1
0
  @Test
  public void testMoreThanEightProps() {
    List<Field> fields = new ArrayList<Field>(30);
    for (int i = 0; i < 30; i++) {
      fields.add(Field.create("field" + i, Type.STRING));
    }
    Schema schema = new Schema("bigmammaschema", fields);

    ITuple tuple = new Tuple(schema);
    for (int i = 0; i < 30; i++) {
      // Fill in with some default values
      tuple.set("field" + i, "defaultValue" + i);
    }

    NullableTuple nullableTuple = new NullableTuple(tuple);
    for (int i = 0; i < 30; i++) {
      // Assert the default values
      assertEquals("defaultValue" + i, nullableTuple.get("field" + i));
    }

    // Set fields to null one by one and assert that things go well
    for (int i = 0; i < 30; i++) {
      nullableTuple.set("field" + i, null);
      assertEquals(null, nullableTuple.getNullable("field" + i));
      nullableTuple.set("field" + i, "defaultValue" + i);
      assertEquals("defaultValue" + i, nullableTuple.getNullable("field" + i));
    }
  }
示例#2
0
  /**
   * Compares sequentially the fields from two tuples and returns which field they differ from. Use
   * custom comparators when provided. The provided RawComparators must implement "compare" so we
   * should use them.
   *
   * <p>Important. The contract of this method is that the tuples will differ always between
   * minField and maxField. If they are equal then an Exception is thrown.
   */
  private int indexMismatch(ITuple tuple1, ITuple tuple2, int minFieldIndex, int maxFieldIndex) {
    int schemaId1 = tupleMRConfig.getSchemaIdByName(tuple1.getSchema().getName());
    int schemaId2 = tupleMRConfig.getSchemaIdByName(tuple2.getSchema().getName());
    int[] translationTuple1 = serInfo.getGroupSchemaIndexTranslation(schemaId1);
    int[] translationTuple2 = serInfo.getGroupSchemaIndexTranslation(schemaId2);

    for (int i = minFieldIndex; i <= maxFieldIndex; i++) {
      Object obj1 = tuple1.get(translationTuple1[i]);
      Object obj2 = tuple2.get(translationTuple2[i]);
      @SuppressWarnings("unchecked")
      RawComparator<Object> customComparator = (RawComparator<Object>) customComparators[i];

      if (customComparator != null) {
        if (customComparator.compare(obj1, obj2) != 0) {
          return i;
        }
      } else {
        if (!obj1.equals(obj2)) {
          return i;
        }
      }
    }
    throw new RuntimeException(
        "Illegal state.The tuples "
            + tuple1
            + " and "
            + tuple2
            + " compare the same between indexes "
            + minFieldIndex
            + " and "
            + maxFieldIndex);
  }
 @Override
 public void reduce(
     ITuple group, Iterable<ITuple> tuples, TupleMRContext context, Collector collector)
     throws IOException, InterruptedException, TupleMRException {
   int count = 0;
   ITuple outputTuple = null;
   for (ITuple tuple : tuples) {
     outputTuple = tuple;
     count += (Integer) tuple.get(1);
   }
   outputTuple.set(1, count);
   collector.write(outputTuple, NullWritable.get());
 }
示例#4
0
  @Override
  public final void reduce(DatumWrapper<ITuple> key, Iterable<NullWritable> values, Context context)
      throws IOException, InterruptedException {

    try {
      Iterator<NullWritable> iterator = values.iterator();
      tupleIterator.setIterator(iterator);
      ITuple currentTuple = key.datum();
      ITuple previousKey = key.previousDatum();
      int indexMismatch;
      if (firstRun) {
        indexMismatch = minDepth;
        firstRun = false;
      } else {
        indexMismatch = indexMismatch(previousKey, currentTuple, 0, maxDepth);
        if (indexMismatch < minDepth) {
          indexMismatch = minDepth;
        }
        for (int i = maxDepth; i >= indexMismatch; i--) {
          handler.onCloseGroup(
              i, groupSchema.getField(i).getName(), previousKey, this.context, collector);
        }
      }

      for (int i = indexMismatch; i <= maxDepth; i++) {
        handler.onOpenGroup(
            i, groupSchema.getField(i).getName(), currentTuple, this.context, collector);
      }

      // We set a view over the group fields to the method.
      if (isMultipleSources) {
        int schemaId = tupleMRConfig.getSchemaIdByName(currentTuple.getSchema().getName());
        int[] indexTranslation = serInfo.getGroupSchemaIndexTranslation(schemaId);
        groupTuple.setContained(currentTuple, indexTranslation);
      } else {
        groupTuple.setContained(currentTuple);
      }

      handler.reduce(groupTuple, tupleIterator, this.context, collector);

      // This loop consumes the remaining elements that reduce didn't consume
      // The goal of this is to correctly set the last element in the next
      // onCloseGroup() call
      while (iterator.hasNext()) {
        iterator.next();
      }
    } catch (TupleMRException e) {
      throw new RuntimeException(e);
    }
  }
    @Override
    public void reduce(
        ITuple group, Iterable<ITuple> tuples, TupleMRContext context, Collector collector)
        throws IOException, InterruptedException, TupleMRException {

      if (outputCount == null) {
        outputCount = new IntWritable();
      }
      int count = 0;
      for (ITuple tuple : tuples) {
        count += (Integer) tuple.get(1);
      }
      outputCount.set(count);
      collector.write((Text) group.get(0), outputCount);
    }
示例#6
0
  @Test
  public void testModifying() {
    Schema schema =
        new Schema(
            "testSchema", Fields.parse("a:string, b:int, c:double, d:float, e:boolean, f:long"));
    ITuple tuple = new Tuple(schema);
    tuple.set(0, "foo");
    tuple.set(1, null);
    tuple.set(2, 20d);
    tuple.set(3, null);
    tuple.set(4, false);
    tuple.set(5, null);

    NullableTuple nullableTuple = new NullableTuple(tuple);
    assertEquals("foo", nullableTuple.getNullable(0).toString());
    assertEquals(null, nullableTuple.getNullable(1));
    assertEquals(20d, nullableTuple.getNullable(2));
    assertEquals(null, nullableTuple.getNullable(3));
    assertEquals(false, nullableTuple.getNullable(4));
    assertEquals(null, nullableTuple.getNullable(5));

    nullableTuple.set(2, null);
    nullableTuple.set(4, null);
    nullableTuple.set(0, null);
    nullableTuple.set(1, 10);
    nullableTuple.set(3, 20f);
    nullableTuple.set(5, 30l);

    assertEquals(null, nullableTuple.getNullable(0));
    assertEquals(10, nullableTuple.getNullable(1));
    assertEquals(null, nullableTuple.getNullable(2));
    assertEquals(20f, nullableTuple.getNullable(3));
    assertEquals(null, nullableTuple.getNullable(4));
    assertEquals(30l, nullableTuple.getNullable(5));
  }
示例#7
0
 protected static void fillObject(boolean isRandom, ITuple tuple, Field field, int index) {
   Object instance = ReflectionUtils.newInstance(field.getObjectClass(), null);
   if (instance instanceof A) {
     A a = (A) instance;
     a.setId(isRandom ? random.nextInt() + "" : "");
     a.setUrl(isRandom ? random.nextLong() + "" : "");
   }
   tuple.set(index, instance);
 }
示例#8
0
 protected static void fillString(boolean isRandom, ITuple tuple, int index) {
   if (isRandom) {
     switch (random.nextInt(4)) {
       case 0:
         tuple.set(index, "");
         break;
       case 1:
         tuple.set(index, random.nextLong() + "");
         break;
       case 2:
         tuple.set(index, new Utf8(random.nextLong() + ""));
         break;
       case 3:
         tuple.set(index, new Text(random.nextLong() + ""));
         break;
     }
   } else {
     tuple.set(index, "");
   }
 }
示例#9
0
    @Override
    public void reduce(
        ITuple group, Iterable<ITuple> tuples, TupleMRContext context, Collector collector)
        throws IOException, InterruptedException, TupleMRException {

      String groupString = group.get(0).toString();
      if (groupString.equals("FR")) {
        FR_PRESENT = true;
        if (!ES_PRESENT) {
          throw new AssertionError("ES should have come before FR");
        }
      } else if (groupString.equals("ES")) {
        ES_PRESENT = true;
      }
      for (Object tuple : tuples) {
        List<String> savedTuples = records.get(groupString);
        if (savedTuples == null) {
          savedTuples = new ArrayList<String>();
          records.put(groupString, savedTuples);
        }
        savedTuples.add(tuple.toString());
      }
    }
示例#10
0
 /** Fills the fields specified by the range (minIndex, maxIndex) with random data. */
 protected static void fillTuple(boolean isRandom, ITuple tuple, int minIndex, int maxIndex) {
   try {
     for (int i = minIndex; i <= maxIndex; i++) {
       Field field = tuple.getSchema().getField(i);
       switch (field.getType()) {
         case INT:
           tuple.set(i, isRandom ? random.nextInt() : 0);
           break;
         case LONG:
           tuple.set(i, isRandom ? random.nextLong() : 0);
           break;
         case BOOLEAN:
           tuple.set(i, isRandom ? random.nextBoolean() : false);
           break;
         case DOUBLE:
           tuple.set(i, isRandom ? random.nextDouble() : 0.0);
           break;
         case FLOAT:
           tuple.set(i, isRandom ? random.nextFloat() : 0f);
           break;
         case STRING:
           fillString(isRandom, tuple, i);
           break;
         case ENUM:
           fillEnum(isRandom, field, tuple, i);
           break;
         case OBJECT:
           fillObject(isRandom, tuple, field, i);
           break;
         default:
           throw new IllegalArgumentException("Not supported type " + field.getType());
       }
     }
   } catch (Exception e) {
     throw new RuntimeException(e);
   }
 }
示例#11
0
 protected static void fillTuple(boolean random, ITuple tuple) {
   fillTuple(random, tuple, 0, tuple.getSchema().getFields().size() - 1);
 }
示例#12
0
 protected static void fillEnum(boolean isRandom, Field field, ITuple tuple, int index)
     throws Exception {
   Method method = field.getObjectClass().getMethod("values", (Class[]) null);
   Enum[] values = (Enum[]) method.invoke(null);
   tuple.set(index, values[isRandom ? random.nextInt(values.length) : 0]);
 }
示例#13
0
  public void createInput(String input, Configuration conf)
      throws IOException, InterruptedException {
    Path inPath = new Path(input);
    FileSystem fs = FileSystem.get(inPath.toUri(), conf);
    TupleFile.Writer writer = new TupleFile.Writer(fs, conf, inPath, TopicalWordCount.getSchema());

    // Topic 1, words: { a, 10 } { b, 1 } , { c, 5 }
    // Top 2 words = a(10), c(5)
    ITuple tuple = new Tuple(TopicalWordCount.getSchema());
    tuple.set("word", "a");
    tuple.set("topic", 1);
    tuple.set("count", 10);
    writer.append(tuple);

    tuple.set("word", "b");
    tuple.set("topic", 1);
    tuple.set("count", 1);
    writer.append(tuple);

    tuple.set("word", "c");
    tuple.set("topic", 1);
    tuple.set("count", 5);
    writer.append(tuple);

    // Topic 2, words: { a, 10 } { b, 9 } , { c, 5 }
    // Top 2 words = a(10), b(9)
    tuple.set("word", "a");
    tuple.set("topic", 2);
    tuple.set("count", 10);
    writer.append(tuple);

    tuple.set("word", "b");
    tuple.set("topic", 2);
    tuple.set("count", 9);
    writer.append(tuple);

    tuple.set("word", "c");
    tuple.set("topic", 2);
    tuple.set("count", 5);
    writer.append(tuple);

    writer.close();
  }