예제 #1
0
파일: Unique.java 프로젝트: bs1/cascading
    @Override
    public boolean isRemove(
        FlowProcess flowProcess, FilterCall<LinkedHashMap<Tuple, Object>> filterCall) {
      // we assume its more painful to create lots of tuple copies vs comparisons
      Tuple args = TupleHasher.wrapTuple(tupleHasher, filterCall.getArguments().getTuple());

      switch (include) {
        case ALL:
          break;

        case NO_NULLS:
          if (Tuples.frequency(args, null) == args.size()) return true;

          break;
      }

      if (filterCall.getContext().containsKey(args)) {
        flowProcess.increment(Cache.Num_Keys_Hit, 1);
        return true;
      }

      // only do the copy here
      filterCall
          .getContext()
          .put(TupleHasher.wrapTuple(tupleHasher, filterCall.getArguments().getTupleCopy()), null);

      flowProcess.increment(Cache.Num_Keys_Missed, 1);

      return false;
    }
예제 #2
0
  private void set(int[] pos, Type[] types, Tuple tuple, CoercibleType[] coercions) {
    verifyModifiable();

    if (pos.length != tuple.size())
      throw new TupleException(
          "given tuple not same size as position array: "
              + pos.length
              + ", tuple: "
              + tuple.print());

    int count = 0;

    for (int i : pos) {
      Object element = tuple.elements.get(count);

      if (types != null) {
        Type type = types[i];
        element = coercions[count].coerce(element, type);
      }

      elements.set(i, element);

      count++;
    }
  }
예제 #3
0
  public void setTuple(Tuple tuple) {
    if (getFields().size() != tuple.size()) {
      throw new IllegalArgumentException("Size of tuple doesn't match current fields");
    }

    _tupleEntry.setTuple(tuple);
    reset();
  }
예제 #4
0
  /**
   * Create a new datum with field names defined by <fields>, and field values contained in <tuple>
   *
   * <p>WARNING - <tuple> will be kept as the data container, so don't call this with a tuple
   * provided by a Cascading operation/iterator, as those get reused.
   *
   * @param fields Names of fields
   * @param tuple Data for the datum
   */
  public BaseDatum(Fields fields, Tuple tuple) {
    if (fields.size() != tuple.size()) {
      throw new IllegalArgumentException(
          "Size of fields must be the same as the size of the tuple: " + fields + "/" + tuple);
    }

    _tupleEntry = new TupleEntry(fields, tuple);
  }
예제 #5
0
  /**
   * Sets the values in the given positions to the values from the given Tuple.
   *
   * @param pos of type int[]
   * @param tuple of type Tuple
   */
  void set(int[] pos, Tuple tuple) {
    verifyModifiable();

    if (pos.length != tuple.size())
      throw new TupleException(
          "given tuple not same size as position array, tuple: " + tuple.print());

    int count = 0;
    for (int i : pos) elements.set(i, tuple.elements.get(count++));
  }
예제 #6
0
  public HadoopCoGroupClosure(
      FlowProcess flowProcess, int numSelfJoins, Fields[] groupingFields, Fields[] valueFields) {
    super(flowProcess, groupingFields, valueFields);
    this.numSelfJoins = numSelfJoins;

    this.emptyTuple = Tuple.size(groupingFields[0].size());

    FactoryLoader loader = FactoryLoader.getInstance();

    this.tupleCollectionFactory =
        loader.loadFactoryFrom(
            flowProcess, TUPLE_COLLECTION_FACTORY, HadoopTupleCollectionFactory.class);

    initLists();
  }
예제 #7
0
  @Test
  public void testParserDeclared6() throws IOException {
    RegexParser splitter = new RegexParser(new Fields("lhs"), "(\\S+)\\s+\\S+", new int[] {1});
    Tuple arguments = new Tuple("foo\tbar");
    Fields resultFields = Fields.size(1);

    TupleListCollector collector = invokeFunction(splitter, arguments, resultFields);

    assertEquals("wrong size", 1, collector.size());

    Iterator<Tuple> iterator = collector.iterator();

    Tuple tuple = iterator.next();

    assertEquals("wrong tupel size", 1, tuple.size());
    assertEquals("not equal: tuple.get(0)", "foo", tuple.getObject(0));
  }
예제 #8
0
  /** Contributed by gicode */
  @Test
  public void testParserDeclared5() throws IOException {
    RegexParser splitter = new RegexParser(new Fields("bar"), "^GET /foo\\?bar=([^\\&]+)&");
    Tuple arguments = new Tuple("GET /foo?bar=z123&baz=2");
    Fields resultFields = Fields.size(1);

    TupleListCollector collector = invokeFunction(splitter, arguments, resultFields);

    assertEquals("wrong size", 1, collector.size());

    Iterator<Tuple> iterator = collector.iterator();

    Tuple tuple = iterator.next();

    assertEquals("wrong tuple size", 1, tuple.size());
    assertEquals("not equal: tuple.get(0)", "z123", tuple.getObject(0));
  }
예제 #9
0
 private TupleEntry getEntry(Tuple tuple) {
   return new TupleEntry(Fields.size(tuple.size()), tuple);
 }
예제 #10
0
 /**
  * Create an empty datum with field names defined by <fields>
  *
  * @param fields Names of fields
  */
 public BaseDatum(Fields fields) {
   this(new TupleEntry(fields, Tuple.size(fields.size())));
 }