@Override
protected void map(
BytesWritable key,
ArrayWritable value,
Mapper<BytesWritable, ArrayWritable, BytesWritable, TextArrayWritable>.Context context)
throws IOException, InterruptedException {
Map<String, Report> reports = new LinkedHashMap<String, Report>();
Writable[] repts = value.get();
if (repts.length == 0 || !(repts[0] instanceof Text)) {
System.out.println("error: bad input.");
return; // bail out more drastically
}
Text[] repts_as_text = (Text[]) repts;
for (Text t : repts_as_text) {
Report r = Report.createFromString(t.toString());
reports.put(r.getMetadata().getOpIdString(), r);
}
Text[] indexed = indexGraph(reports);
TextArrayWritable output = new TextArrayWritable();
output.set(indexed);
context.write(key, output);
}
@Override
public void reduce(NullWritable key, Iterable<TextArrayWritable> values, Context context)
throws IOException, InterruptedException {
// TODO
// TextArrayWritable tmp;
// while(values.hasNext()){
for (TextArrayWritable value : values) {
for (Writable w : value.get()) {
String str = ((Text) w).toString();
String arrStr[] = str.split("\\|");
tree.add(new Pair(Integer.parseInt(arrStr[0]), arrStr[1]));
}
}
// reverse tree set
TreeSet<Pair> treereverse = new TreeSet<Pair>();
treereverse = (TreeSet) tree.descendingSet();
Iterator itr = treereverse.iterator();
int count = 1;
while (itr.hasNext() && count != N + 1) {
Pair p = (Pair) itr.next();
// send top N entry
context.write(
new Text(p.second.toString()), new IntWritable(Integer.parseInt(p.first.toString())));
count++;
}
}
@Override
public void reduce(NullWritable key, Iterable<TextArrayWritable> values, Context context)
throws IOException, InterruptedException {
Integer sum, mean, max, min, var;
// TODO
for (TextArrayWritable val : values) {
Text[] pair = (Text[]) val.toArray();
String title = pair[0].toString();
Integer count = Integer.parseInt(pair[1].toString());
countToTitleMap.add(new Pair<Integer, String>(count, title));
if (countToTitleMap.size() > this.N) {
countToTitleMap.remove(countToTitleMap.first());
}
}
// calculate the statistics
Integer[] counts = new Integer[this.N];
int j = 0;
for (Pair<Integer, String> pair : countToTitleMap) {
counts[j++] = pair.first;
}
sum = 0;
min = Integer.MAX_VALUE;
max = Integer.MIN_VALUE;
for (int i = 0; i < this.N; i++) {
sum += counts[i];
min = Math.min(min, counts[i]);
max = Math.max(max, counts[i]);
}
mean = sum / this.N;
var = 0;
for (int i = 0; i < this.N; i++) {
var += ((counts[i] - mean) * (counts[i] - mean));
}
var /= this.N;
context.write(new Text("Mean"), new IntWritable(mean));
context.write(new Text("Sum"), new IntWritable(sum));
context.write(new Text("Min"), new IntWritable(min));
context.write(new Text("Max"), new IntWritable(max));
context.write(new Text("Var"), new IntWritable(var));
}
@Override
public void reduce(NullWritable key, Iterable<TextArrayWritable> values, Context context)
throws IOException, InterruptedException {
TreeSet<Pair<Integer, String>> all = new TreeSet<Pair<Integer, String>>();
for (TextArrayWritable t : values) {
String[] line = t.toStrings();
Pair<Integer, String> p = Pair.of(Integer.parseInt(line[1]), line[0]);
all.add(p);
}
ArrayList<Pair<Integer, String>> result = new ArrayList<Pair<Integer, String>>(all);
for (int i = result.size() - 1; i >= result.size() - N; i--) {
context.write(new Text(result.get(i).second), new IntWritable(result.get(i).first));
}
}
@Override
public void reduce(NullWritable key, Iterable<TextArrayWritable> values, Context context)
throws IOException, InterruptedException {
int sum, mean, max, min, var;
sum = mean = max = min = var = 0;
for (TextArrayWritable val : values) {
Text[] pair = (Text[]) val.toArray();
String word = pair[0].toString();
Integer count = Integer.parseInt(pair[1].toString());
sortedWordCount.add(new Pair<Integer, String>(count, word));
if (sortedWordCount.size() > N) {
sortedWordCount.remove(sortedWordCount.first());
}
}
for (Pair<Integer, String> entry : sortedWordCount) {
// Text word = new Text(entry.second);
// IntWritable value = new IntWritable(entry.first);
// context.write(word, value);
sum += entry.first;
}
mean = (sum / sortedWordCount.size());
min = sortedWordCount.first().first;
max = sortedWordCount.last().first;
for (Pair<Integer, String> entry : sortedWordCount) {
var += ((entry.first - mean) * (entry.first - mean));
}
var = var / sortedWordCount.size();
context.write(new Text("Mean"), new IntWritable(mean));
context.write(new Text("Sum"), new IntWritable(sum));
context.write(new Text("Min"), new IntWritable(min));
context.write(new Text("Max"), new IntWritable(max));
context.write(new Text("Var"), new IntWritable(var));
}
@Override
public void reduce(NullWritable key, Iterable<TextArrayWritable> values, Context context)
throws IOException, InterruptedException {
for (TextArrayWritable val : values) {
Text[] pair = (Text[]) val.toArray();
String word = pair[0].toString();
Float avg = Float.parseFloat(pair[1].toString());
/*
Text tWord = new Text(word);
FloatWritable value = new FloatWritable(avg);
context.write(tWord, value);
*/
String[] parts = word.split("-");
String newKey = parts[0] + "-" + parts[1];
String carrier = parts[2];
if (!carrierAvgDelay.containsKey(newKey)) {
TreeSet<Pair<Float, String>> avgDelayMap = new TreeSet<Pair<Float, String>>();
carrierAvgDelay.put(newKey, avgDelayMap);
}
TreeSet<Pair<Float, String>> ts = carrierAvgDelay.get(newKey);
ts.add(new Pair<Float, String>(avg, carrier));
if (ts.size() > 10) {
ts.remove(ts.last());
}
carrierAvgDelay.put(newKey, ts);
}
Iterator it = carrierAvgDelay.entrySet().iterator();
while (it.hasNext()) {
Map.Entry pair = (Map.Entry) it.next();
TreeSet<Pair<Float, String>> ts = (TreeSet<Pair<Float, String>>) pair.getValue();
for (Pair<Float, String> item : ts) {
Text word = new Text(pair.getKey() + "-" + item.second);
FloatWritable value = new FloatWritable(item.first);
context.write(word, value);
}
}
}
@Override
public void reduce(NullWritable key, Iterable<TextArrayWritable> values, Context context)
throws IOException, InterruptedException {
for (TextArrayWritable val : values) {
Text[] pair = (Text[]) val.toArray();
String word = pair[0].toString();
Integer count = Integer.parseInt(pair[1].toString());
countToWordMap.add(new Pair<Integer, String>(count, word));
if (countToWordMap.size() > N) {
countToWordMap.remove(countToWordMap.first());
}
}
for (Pair<Integer, String> item : countToWordMap) {
Text word = new Text(item.second);
IntWritable value = new IntWritable(item.first);
context.write(word, value);
}
}
@Override
protected void reduce(
Key_IMOAndRecordTime key, Iterable<TextArrayWritable> LocationList, Context context)
throws IOException, InterruptedException {
try {
context.getCounter(Counters.VESSEL_PROCESSED).increment(1);
String IMO_str = LpadNum(key.getIMO().get(), 7);
long first_pos_time = key.getRecordTime().get();
/////////////////////////////////////////////////////////////////////////////////
// Populate newPoints with new locations
List<VesselLocation> newPoints = new ArrayList<VesselLocation>();
for (TextArrayWritable rowcontent : LocationList) {
// population location
context.getCounter(Counters.LOCATION_ROWS).increment(1);
VesselLocation newlocation = new VesselLocation();
try {
Writable[] content = rowcontent.get();
String Latitude = content[16].toString().trim();
String Longitude = content[15].toString().trim();
String Coordinates = Latitude + "," + Longitude;
String Speed = content[18].toString().trim();
String Destination = content[9].toString().trim();
String Timestamp = content[21].toString().trim().substring(0, 19);
long record_time = DateTime.parse(Timestamp, rawformatter).getMillis();
newlocation.coordinates = Coordinates;
newlocation.recordtime = record_time;
newlocation.speed = Speed;
newlocation.destination = Destination;
context.getCounter(Counters.LOCATION_VALID).increment(1);
} catch (Exception e) {
e.printStackTrace();
context.getCounter(Counters.LOCATION_ERROR).increment(1);
continue;
}
newPoints.add(newlocation);
}
/////////////////////////////////////////////////////////////////////////////////
// Get last new post time
long last_pos_time = newPoints.get(newPoints.size() - 1).recordtime;
////////////////////////////////////////////////////////////////////////////////
// Get Existing trackinfo
VesselTrackInfo VTI = getTrackInfo(TrackInfo_Table, IMO_str);
List<VesselLocation> AllBetweenPoints = new ArrayList<VesselLocation>();
String BeforeRowKey = null;
String AfterRowKey = null;
// //////////////////////////////////////////////////////////////////////////////
// Retrieve all the existing locations between the first new location and the last new
// location.
if ((VTI.FirstRecordTime != null) && (VTI.LastRecordTime != null)) {
if (last_pos_time < VTI.FirstRecordTime) {
AfterRowKey = IMO_str + LpadNum(Long.MAX_VALUE - VTI.FirstRecordTime, 19);
} else if (first_pos_time > VTI.LastRecordTime) {
BeforeRowKey = IMO_str + LpadNum(Long.MAX_VALUE - VTI.LastRecordTime, 19);
} else {
AllBetweenPoints =
ImportReducer.getLocationsBetween(
VTLocation_Table, IMO_str, first_pos_time, last_pos_time);
if (AllBetweenPoints.size() == 0) {
// Search for the first DB point before the first new point
VesselLocation BeforeLocation =
getLocationBefore(VTLocation_Table, IMO_str, first_pos_time);
BeforeRowKey = IMO_str + LpadNum(Long.MAX_VALUE - BeforeLocation.recordtime, 19);
AfterRowKey = BeforeLocation.nextlocation;
} else {
java.util.Collections.sort(AllBetweenPoints);
BeforeRowKey = AllBetweenPoints.get(0).previouslocation;
AfterRowKey = AllBetweenPoints.get(AllBetweenPoints.size() - 1).nextlocation;
}
List<Delete> deletes =
ImportReducer.GetDeleteEventsBetween(
VTEvent_Table, IMO_str, first_pos_time, last_pos_time);
ImportReducer.DeleteEvents(VTEvent, deletes);
VTEvent.flush();
}
}
// Find out the location before the first new location in
VesselLocation BeforeLocation = getLocation(VTLocation_Table, BeforeRowKey);
// Find out the location after the last new location in
VesselLocation AfterLocation = getLocation(VTLocation_Table, AfterRowKey);
Map<Integer, VesselEvent> PreviousZoneEvents = new HashMap<Integer, VesselEvent>();
;
Map<Integer, VesselEvent> AfterZoneEvents = new HashMap<Integer, VesselEvent>();
if (BeforeLocation != null) {
// Get all events with exit at last location
PreviousZoneEvents =
getAllEventsStartBeforeEndAfterBeforeLocation(VTEvent_Table, IMO_str, BeforeLocation);
}
////////////////////////////////////////////////////
// Analyze and calculate previous and next location
for (VesselLocation newlocation : newPoints) {
int index = AllBetweenPoints.indexOf(newlocation);
if (index != -1) {
VesselLocation dblocation = AllBetweenPoints.get(index);
dblocation.coordinates = newlocation.coordinates;
dblocation.destination = newlocation.destination;
dblocation.speed = newlocation.speed;
} else {
AllBetweenPoints.add(newlocation);
}
}
java.util.Collections.sort(AllBetweenPoints);
String previousRowKey = null;
for (VesselLocation location : AllBetweenPoints) {
location.previouslocation = previousRowKey;
previousRowKey = IMO_str + LpadNum(Long.MAX_VALUE - location.recordtime, 19);
}
String NextRowKey = null;
for (int i = (AllBetweenPoints.size() - 1); i >= 0; i--) {
VesselLocation location = AllBetweenPoints.get(i);
location.nextlocation = NextRowKey;
NextRowKey = IMO_str + LpadNum(Long.MAX_VALUE - location.recordtime, 19);
}
AllBetweenPoints.get(0).previouslocation = BeforeRowKey;
AllBetweenPoints.get(AllBetweenPoints.size() - 1).nextlocation = AfterRowKey;
////////////////////////////////////////////////////
// Upsert all locations
for (VesselLocation location : AllBetweenPoints) {
// population location
try {
byte[] rowkey =
Bytes.toBytes(IMO_str + LpadNum(Long.MAX_VALUE - location.recordtime, 19));
Put put = new Put(rowkey);
put.addColumn(details, speed, Bytes.toBytes(location.speed));
put.addColumn(details, destination, Bytes.toBytes(location.destination));
put.addColumn(details, coordinates, Bytes.toBytes(location.coordinates));
put.addColumn(
details,
timestamp,
Bytes.toBytes(new DateTime(location.recordtime).toString(rawformatter)));
if (location.previouslocation != null) {
put.addColumn(details, previouslocation, Bytes.toBytes(location.previouslocation));
}
if (location.nextlocation != null) {
put.addColumn(details, nextlocation, Bytes.toBytes(location.nextlocation));
}
VTLocation.mutate(put);
} catch (Exception e) {
e.printStackTrace();
context.getCounter(Counters.LOCATION_ERROR).increment(1);
continue;
}
}
// update before next location and after previous location
if (BeforeRowKey != null) {
Put BeforeLocationPut = new Put(Bytes.toBytes(BeforeRowKey));
BeforeLocationPut.addColumn(
details,
nextlocation,
Bytes.toBytes(
IMO_str + LpadNum(Long.MAX_VALUE - AllBetweenPoints.get(0).recordtime, 19)));
VTLocation.mutate(BeforeLocationPut);
}
if (AfterRowKey != null) {
Put AfterLocationPut = new Put(Bytes.toBytes(AfterRowKey));
AfterLocationPut.addColumn(
details,
previouslocation,
Bytes.toBytes(
IMO_str
+ LpadNum(
Long.MAX_VALUE
- AllBetweenPoints.get(AllBetweenPoints.size() - 1).recordtime,
19)));
VTLocation.mutate(AfterLocationPut);
}
VTLocation.flush();
/////////////////////////////////////////////////////////////////////
// Store latest location
// rowkey: global zone id (4)+ longlat22
// ((long11(sign(1)+integer(3)+digit(7)))(lat11(sign(1)+integer(3)+(7))))+imo(7)+recordtime(19)
/////////////////////////////////////////////////////////////////////
Put vessel_track_info = new Put(Bytes.toBytes(IMO_str));
if (AfterLocation == null) {
// Get the last location
VesselLocation lastLocation = AllBetweenPoints.get(AllBetweenPoints.size() - 1);
// update the last location
String[] longlat = lastLocation.coordinates.split(",");
GeometryFactory geometryFactory = JTSFactoryFinder.getGeometryFactory(null);
Coordinate coord =
new Coordinate(Double.parseDouble(longlat[1]), Double.parseDouble(longlat[0]));
Point point = geometryFactory.createPoint(coord);
Integer BelongedGlobalZoneIndex = null;
for (int i = 0; i < VesselZone.GlobalZones.length; i++) {
if (VesselZone.GlobalZones[i].covers(point)) {
BelongedGlobalZoneIndex = i;
break;
}
}
if (VTI.LastLocation != null) {
LastLocation_BM.mutate(new Delete(VTI.LastLocation));
}
byte[] lastlocationrowkey =
Bytes.toBytes(
LpadNum(BelongedGlobalZoneIndex, 4)
+ ConvertCoordinatesToStr(longlat[1])
+ ConvertCoordinatesToStr(longlat[0]));
Put lastlocation_put = new Put(lastlocationrowkey);
lastlocation_put.addColumn(details, imo, Bytes.toBytes(IMO_str));
lastlocation_put.addColumn(
details,
timestamp,
Bytes.toBytes(new DateTime(lastLocation.recordtime).toString(rawformatter)));
LastLocation_BM.mutate(lastlocation_put);
LastLocation_BM.flush();
vessel_track_info.addColumn(details, lastlocation, lastlocationrowkey);
vessel_track_info.addColumn(
details,
lastrecordtime,
Bytes.toBytes(new DateTime(lastLocation.recordtime).toString(rawformatter)));
} else {
// Get events that start before last new location and end after last new location
AfterZoneEvents =
getAllEventsStartBeforeEndAfter(VTEvent_Table, IMO_str, AfterLocation.recordtime);
}
// update firstrecordtime and lastrecordtime
if (BeforeLocation == null) {
vessel_track_info.addColumn(
details,
firstrecordtime,
Bytes.toBytes(
new DateTime(AllBetweenPoints.get(0).recordtime).toString(rawformatter)));
}
if (!vessel_track_info.isEmpty()) {
TrackInfo_BM.mutate(vessel_track_info);
TrackInfo_BM.flush();
}
////////////////////////////////////////////////////////////////////
ArrayList<VesselEvent> DerivedEventList = new ArrayList<VesselEvent>();
///////////////////////////////////////////////////////////
// Get Vessel
String VesselType = getVesselType(Vessel_Table, IMO_str);
if (VesselType == null) {
context.getCounter(Counters.VESSEL_WITHOUTTYPE).increment(1);
return;
}
// calculating event
for (VesselLocation VL : AllBetweenPoints) {
ArrayList<Integer> CurrentZones = LocateCurrentZone(VL.coordinates, VesselType, Zonemap);
Iterator<Map.Entry<Integer, VesselEvent>> it = PreviousZoneEvents.entrySet().iterator();
while (it.hasNext()) {
Map.Entry<Integer, VesselEvent> thisEntry = it.next();
int Zone_Axsmarine_id = thisEntry.getKey();
if (!CurrentZones.contains(Zone_Axsmarine_id)) {
VesselEvent PreviousEvent = thisEntry.getValue();
if (!DerivedEventList.contains(PreviousEvent)) {
DerivedEventList.add(PreviousEvent);
}
// remove close event from PreviousZoneEvents;
it.remove();
}
}
for (Integer thisZone_Axsmarine_id : CurrentZones) {
if (PreviousZoneEvents.containsKey(thisZone_Axsmarine_id)) {
//////////////////////////////////////////////////
// For current zones which both previous and current locations belong to, update exit
// point of previous open events with current locations.
//////////////////////////////////////////////////
VesselEvent PreviousEvent = PreviousZoneEvents.get(thisZone_Axsmarine_id);
PreviousEvent.exitcoordinates = VL.coordinates;
PreviousEvent.exittime = VL.recordtime;
PreviousEvent.destination = VL.destination;
if (!DerivedEventList.contains(PreviousEvent)) {
DerivedEventList.add(PreviousEvent);
}
} else {
//////////////////////////////////////////////////
// For current zones which only current locations belong to, fire new open events
//////////////////////////////////////////////////
VesselEvent NewEvent = new VesselEvent();
NewEvent.entrycoordinates = VL.coordinates;
NewEvent.entrytime = VL.recordtime;
NewEvent.exitcoordinates = VL.coordinates;
NewEvent.exittime = VL.recordtime;
NewEvent.destination = VL.destination;
NewEvent.polygonid = thisZone_Axsmarine_id;
PreviousZoneEvents.put(thisZone_Axsmarine_id, NewEvent);
DerivedEventList.add(NewEvent);
}
}
}
///////////////////////////////////////////////////////////////////////////////////////
// Merge with PreviousZoneEvents with AfterZoneEvents
Iterator<Map.Entry<Integer, VesselEvent>> it = AfterZoneEvents.entrySet().iterator();
while (it.hasNext()) {
Map.Entry<Integer, VesselEvent> thisEntry = it.next();
int Zone_Axsmarine_id = thisEntry.getKey();
VesselEvent After_VE = thisEntry.getValue();
VesselEvent Previous_VE = PreviousZoneEvents.get(Zone_Axsmarine_id);
if (Previous_VE != null) {
Previous_VE.exitcoordinates = After_VE.exitcoordinates;
Previous_VE.exittime = After_VE.exittime;
Previous_VE.destination = After_VE.destination;
if (!DerivedEventList.contains(Previous_VE)) {
DerivedEventList.add(Previous_VE);
}
} else {
VesselEvent NewEvent = new VesselEvent();
NewEvent.entrycoordinates = AfterLocation.coordinates;
NewEvent.entrytime = AfterLocation.recordtime;
NewEvent.exitcoordinates = After_VE.exitcoordinates;
NewEvent.exittime = After_VE.exittime;
NewEvent.destination = After_VE.destination;
NewEvent.polygonid = Zone_Axsmarine_id;
DerivedEventList.add(NewEvent);
}
// Delete This Event from HBase
DeleteEvent(VTEvent, IMO_str, After_VE);
}
VTEvent.flush();
// pupulate Derived Events into Hbase
for (VesselEvent newEvent : DerivedEventList) {
// rowkey: IMO(7)+timestamp(19 desc)+polygonid(8)
// qualifier:entrytime,entrycoordinates,exittime,exitcoordinates,destination
context.getCounter(Counters.EVENT_UPSERTS).increment(1);
byte[] rowkey =
Bytes.toBytes(
IMO_str
+ LpadNum(Long.MAX_VALUE - newEvent.entrytime, 19)
+ LpadNum(newEvent.polygonid, 10));
Put put = new Put(rowkey);
put.addColumn(
details,
entrytime,
Bytes.toBytes(new DateTime(newEvent.entrytime).toString(rawformatter)));
put.addColumn(details, entrycoordinates, Bytes.toBytes(newEvent.entrycoordinates));
put.addColumn(
details,
exittime,
Bytes.toBytes(new DateTime(newEvent.exittime).toString(rawformatter)));
put.addColumn(details, exitcoordinates, Bytes.toBytes(newEvent.exitcoordinates));
put.addColumn(details, destination, Bytes.toBytes(newEvent.destination));
VTEvent.mutate(put);
context.getCounter(Counters.EVENT_VALID).increment(1);
}
// VTLocation.flush(); Moved to the first step
VTEvent.flush();
} catch (RuntimeException e) {
// TODO Auto-generated catch block
System.out.println("Exception occured while loading data for:" + key.getIMO());
throw e;
}
}
