@Override
public boolean isRemove(
FlowProcess flowProcess, FilterCall<LinkedHashMap<Tuple, Object>> filterCall) {
// we assume its more painful to create lots of tuple copies vs comparisons
Tuple args = TupleHasher.wrapTuple(tupleHasher, filterCall.getArguments().getTuple());
switch (include) {
case ALL:
break;
case NO_NULLS:
if (Tuples.frequency(args, null) == args.size()) return true;
break;
}
if (filterCall.getContext().containsKey(args)) {
flowProcess.increment(Cache.Num_Keys_Hit, 1);
return true;
}
// only do the copy here
filterCall
.getContext()
.put(TupleHasher.wrapTuple(tupleHasher, filterCall.getArguments().getTupleCopy()), null);
flowProcess.increment(Cache.Num_Keys_Missed, 1);
return false;
}
private void set(int[] pos, Type[] types, Tuple tuple, CoercibleType[] coercions) {
verifyModifiable();
if (pos.length != tuple.size())
throw new TupleException(
"given tuple not same size as position array: "
+ pos.length
+ ", tuple: "
+ tuple.print());
int count = 0;
for (int i : pos) {
Object element = tuple.elements.get(count);
if (types != null) {
Type type = types[i];
element = coercions[count].coerce(element, type);
}
elements.set(i, element);
count++;
}
}
public void setTuple(Tuple tuple) {
if (getFields().size() != tuple.size()) {
throw new IllegalArgumentException("Size of tuple doesn't match current fields");
}
_tupleEntry.setTuple(tuple);
reset();
}
/**
* Create a new datum with field names defined by <fields>, and field values contained in <tuple>
*
* <p>WARNING - <tuple> will be kept as the data container, so don't call this with a tuple
* provided by a Cascading operation/iterator, as those get reused.
*
* @param fields Names of fields
* @param tuple Data for the datum
*/
public BaseDatum(Fields fields, Tuple tuple) {
if (fields.size() != tuple.size()) {
throw new IllegalArgumentException(
"Size of fields must be the same as the size of the tuple: " + fields + "/" + tuple);
}
_tupleEntry = new TupleEntry(fields, tuple);
}
/**
* Sets the values in the given positions to the values from the given Tuple.
*
* @param pos of type int[]
* @param tuple of type Tuple
*/
void set(int[] pos, Tuple tuple) {
verifyModifiable();
if (pos.length != tuple.size())
throw new TupleException(
"given tuple not same size as position array, tuple: " + tuple.print());
int count = 0;
for (int i : pos) elements.set(i, tuple.elements.get(count++));
}
public HadoopCoGroupClosure(
FlowProcess flowProcess, int numSelfJoins, Fields[] groupingFields, Fields[] valueFields) {
super(flowProcess, groupingFields, valueFields);
this.numSelfJoins = numSelfJoins;
this.emptyTuple = Tuple.size(groupingFields[0].size());
FactoryLoader loader = FactoryLoader.getInstance();
this.tupleCollectionFactory =
loader.loadFactoryFrom(
flowProcess, TUPLE_COLLECTION_FACTORY, HadoopTupleCollectionFactory.class);
initLists();
}
@Test
public void testParserDeclared6() throws IOException {
RegexParser splitter = new RegexParser(new Fields("lhs"), "(\\S+)\\s+\\S+", new int[] {1});
Tuple arguments = new Tuple("foo\tbar");
Fields resultFields = Fields.size(1);
TupleListCollector collector = invokeFunction(splitter, arguments, resultFields);
assertEquals("wrong size", 1, collector.size());
Iterator<Tuple> iterator = collector.iterator();
Tuple tuple = iterator.next();
assertEquals("wrong tupel size", 1, tuple.size());
assertEquals("not equal: tuple.get(0)", "foo", tuple.getObject(0));
}
/** Contributed by gicode */
@Test
public void testParserDeclared5() throws IOException {
RegexParser splitter = new RegexParser(new Fields("bar"), "^GET /foo\\?bar=([^\\&]+)&");
Tuple arguments = new Tuple("GET /foo?bar=z123&baz=2");
Fields resultFields = Fields.size(1);
TupleListCollector collector = invokeFunction(splitter, arguments, resultFields);
assertEquals("wrong size", 1, collector.size());
Iterator<Tuple> iterator = collector.iterator();
Tuple tuple = iterator.next();
assertEquals("wrong tuple size", 1, tuple.size());
assertEquals("not equal: tuple.get(0)", "z123", tuple.getObject(0));
}
private TupleEntry getEntry(Tuple tuple) {
return new TupleEntry(Fields.size(tuple.size()), tuple);
}
/**
* Create an empty datum with field names defined by <fields>
*
* @param fields Names of fields
*/
public BaseDatum(Fields fields) {
this(new TupleEntry(fields, Tuple.size(fields.size())));
}