@Override
public void coGroup(Iterator<Record> records1, Iterator<Record> records2, Collector<Record> out) {
while (records1.hasNext()) {
out.collect(records1.next());
}
while (records2.hasNext()) {
out.collect(records2.next());
}
}
@Override
public void coGroup(
Iterator<Record> candidates, Iterator<Record> current, Collector<Record> out)
throws Exception {
if (!current.hasNext()) {
throw new Exception("Error: Id not encountered before.");
}
Record old = current.next();
long oldId = old.getField(1, LongValue.class).getValue();
long minimumComponentID = Long.MAX_VALUE;
while (candidates.hasNext()) {
long candidateComponentID = candidates.next().getField(1, LongValue.class).getValue();
if (candidateComponentID < minimumComponentID) {
minimumComponentID = candidateComponentID;
}
}
if (minimumComponentID < oldId) {
newComponentId.setValue(minimumComponentID);
old.setField(1, newComponentId);
out.collect(old);
}
}
@Override
public void reduce(Iterable<Integer> records, Collector<Integer> out) {
if (bcValue == null) {
return;
}
final int x = bcValue;
for (Integer y : records) {
if (y > x) {
out.collect(y);
return;
}
}
out.collect(bcValue);
}
@Override
public void apply(
Integer key, TimeWindow window, Iterable<Integer> values, Collector<Integer> out) {
for (Integer val : values) {
assertEquals(key, val);
out.collect(val);
}
}
@Override
public void flatMap(String input, Collector<Edit> out) throws Exception {
ObjectMapper mapper = new ObjectMapper();
Edit edit = mapper.readValue(input, Edit.class);
if (edit.parse()) {
out.collect(edit);
}
}
/** Close method to be used if the user defined function extends the RichFunction class */
public void close() {
isRunning = false;
collector.close();
try {
FunctionUtils.closeFunction(userFunction);
} catch (Exception e) {
throw new RuntimeException("Error when closing the function: " + e.getMessage());
}
}
@Override
public void run() throws Exception {
// cache references on the stack
final MutableObjectIterator<T> input = this.taskContext.getInput(0);
final Collector<T> output = this.taskContext.getOutputCollector();
if (objectReuseEnabled) {
T record = this.taskContext.<T>getInputSerializer(0).getSerializer().createInstance();
while (this.running && ((record = input.next(record)) != null)) {
output.collect(record);
}
} else {
T record;
TypeSerializer<T> serializer = this.taskContext.<T>getInputSerializer(0).getSerializer();
while (this.running && ((record = input.next(serializer.createInstance())) != null)) {
output.collect(record);
}
}
}
@Override
public void map(Record record, Collector<Record> out) throws Exception {
final long start = System.currentTimeMillis();
long remaining = WAIT_TIME_PER_RECORD;
do {
try {
Thread.sleep(remaining);
} catch (InterruptedException iex) {
}
} while ((remaining = WAIT_TIME_PER_RECORD - System.currentTimeMillis() + start) > 0);
out.collect(record);
}
/**
* Project "supplier".
*
* <p>Output Schema: Key: nationkey Value: suppkey
*/
@Override
public void map(Record record, Collector<Record> out) throws Exception {
suppKey = record.getField(0, suppKey);
inputTuple = record.getField(1, inputTuple);
/* Project (suppkey | name, address, nationkey, phone, acctbal, comment): */
IntValue nationKey = new IntValue(Integer.parseInt(inputTuple.getStringValueAt(3)));
record.setField(0, nationKey);
record.setField(1, suppKey);
out.collect(record);
}
@Override
public void apply(
Integer integer, TimeWindow window, Iterable<Integer> values, Collector<Integer> out)
throws Exception {
for (Integer i : values) {
out.collect(i);
numElements++;
if (numElements >= failAfterElements) {
throw new Exception("Artificial Test Exception");
}
}
}
@Override
public void coGroup(
Iterable<Itinerary> itineraries, Iterable<MIDT> midts, Collector<Itinerary> out)
throws Exception {
Iterator<Itinerary> itinIter = itineraries.iterator();
Iterator<MIDT> midtIter = midts.iterator();
if (!midtIter.hasNext()) {
out.collect(itinIter.next());
} else if (!itinIter.hasNext()) {
out.collect(TAUtil.MIDTToItinerary(midtIter.next()));
} else {
MIDT midt = midtIter.next();
Itinerary itin = itinIter.next();
itin.f13 = midt.f11;
out.collect(itin);
}
if (itinIter.hasNext()) {
throw new Exception("More than one Itinerary: " + itinIter.next().toString());
}
if (midtIter.hasNext()) {
throw new Exception("More than one MIDT: " + midtIter.next().toString());
}
}
@Override
public void apply(
Integer key, TimeWindow window, Iterable<Integer> values, Collector<Integer> out)
throws Exception {
for (Integer i : values) {
// we need to update this state before emitting elements. Else, the test's main
// thread will have received all output elements before the state is updated and
// the checks may fail
state.update(state.value() + 1);
globalCounts.put(key, state.value());
out.collect(i);
}
}
@Override
public void coWindow(List<IN1> first, List<IN2> second, Collector<OUT> out) throws Exception {
Map<Object, List<IN1>> map = build(first);
for (IN2 record : second) {
Object key = keySelector2.getKey(record);
List<IN1> match = map.get(key);
if (match != null) {
for (IN1 matching : match) {
out.collect(joinFunction.join(matching, record));
}
}
}
}
@Override
public void flatMap(Flight flight, Collector<Itinerary> out) throws Exception {
if (flight.getDepartureTimestamp() > TrafficAnalysis.lastPossibleTimestamp
|| flight.getDepartureTimestamp() < TrafficAnalysis.firstPossibleTimestamp) {
return;
}
int numCountries = 2;
if (CBUtil.isDomestic(flight)) {
numCountries = 1;
}
Date date = new Date(flight.getDepartureTimestamp());
String dayString = format.format(date);
Double distance =
CBUtil.dist(
flight.getOriginLatitude(),
flight.getOriginLongitude(),
flight.getDestinationLatitude(),
flight.getDestinationLongitude());
Integer travelTime =
Math.max(
(int)
((flight.getArrivalTimestamp() - flight.getDepartureTimestamp()) / (60L * 1000L)),
1);
out.collect(
new Itinerary(
flight.getOriginAirport(),
flight.getDestinationAirport(),
dayString,
flight.getAirline() + flight.getFlightNumber(),
"",
"",
"",
"",
distance,
distance,
travelTime,
0,
flight.getLegCount(),
0,
flight.getMaxCapacity(),
-1.0,
-1.0,
-1.0,
"",
numCountries,
"",
""));
}
@Override
public void flatMap(Tuple2<Flight, Flight> flight, Collector<Itinerary> out) throws Exception {
if (flight.f0.getDepartureTimestamp() > TrafficAnalysis.lastPossibleTimestamp
|| flight.f0.getDepartureTimestamp() < TrafficAnalysis.firstPossibleTimestamp) {
return;
}
HashSet<String> countries = new HashSet<String>(3);
countries.add(flight.f0.getOriginCountry());
countries.add(flight.f0.getDestinationCountry());
countries.add(flight.f1.getOriginCountry());
countries.add(flight.f1.getDestinationCountry());
Date date = new Date(flight.f0.getDepartureTimestamp());
String dayString = format.format(date);
Double directDistance =
CBUtil.dist(
flight.f0.getOriginLatitude(),
flight.f0.getOriginLongitude(),
flight.f1.getDestinationLatitude(),
flight.f1.getDestinationLongitude());
Double travelledDistance =
CBUtil.dist(
flight.f0.getOriginLatitude(),
flight.f0.getOriginLongitude(),
flight.f0.getDestinationLatitude(),
flight.f0.getDestinationLongitude())
+ CBUtil.dist(
flight.f1.getOriginLatitude(),
flight.f1.getOriginLongitude(),
flight.f1.getDestinationLatitude(),
flight.f1.getDestinationLongitude());
Integer travelTime =
Math.max(
(int)
((flight.f1.getArrivalTimestamp() - flight.f0.getDepartureTimestamp())
/ (60L * 1000L)),
1);
Integer waitingTime =
(int)
((flight.f1.getDepartureTimestamp() - flight.f0.getArrivalTimestamp())
/ (60L * 1000L));
Integer legCount = flight.f0.getLegCount() + flight.f1.getLegCount();
Integer maxCapacity = Math.min(flight.f0.getMaxCapacity(), flight.f1.getMaxCapacity());
out.collect(
new Itinerary(
flight.f0.getOriginAirport(),
flight.f1.getDestinationAirport(),
dayString,
flight.f0.getAirline() + flight.f0.getFlightNumber(),
flight.f1.getAirline() + flight.f1.getFlightNumber(),
"",
"",
"",
directDistance,
travelledDistance,
travelTime,
waitingTime,
legCount,
0,
maxCapacity,
-1.0,
-1.0,
-1.0,
"",
Math.max(1, countries.size()),
flight.f1.getOriginAirport(),
""));
}
@Override
public void map(Record record, Collector<Record> out) throws Exception {
out.collect(record);
}
@Override
public void map(Record record, Collector<Record> out) throws Exception {
Thread.sleep(WAIT_TIME_PER_RECORD);
out.collect(record);
}
@Override
public void invoke(Collector<T> collector) throws Exception {
for (T element : iterable) {
collector.collect(element);
}
}
@Override
public void reduce(Iterable<Record> it, Collector<Record> out) {
for (Record r : it) {
out.collect(r);
}
}
@Override
public void reduce(Iterable<T> values, Collector<T> out) {
out.collect(values.iterator().next());
}
@Override
public void mapPartition(Iterable<Tuple> values, Collector<Tuple> out) throws Exception {
for (Tuple t : values) {
out.collect(t);
}
}