@Override
protected void reduce(Text key, Iterable<LongWritable> values, Context context)
throws IOException, InterruptedException {
/*
* Added by Xudong Zhang
*/
if (parseWeibo) {
String word = key.toString();
if (!ChineseUtils.allChinese(word)) {
context.getCounter("MyCounter", "NumWordsFilteredByWeiboParser").increment(1);
return;
}
}
long sum = 0;
for (LongWritable value : values) {
sum += value.get();
}
if (sum >= minSupport) {
context.write(key, new LongWritable(sum));
} else {
context.getCounter("MyCounter", "NumWordsLessThanMinSupport").increment(1);
}
}
@Override
protected void map(LongWritable key, Text value, Context context)
throws IOException, InterruptedException {
parser.parse(value);
if (parser.isValidTemperature()) {
int airTemperature = parser.getAirTemperature();
context.write(new Text(parser.getYear()), new IntWritable(airTemperature));
} else if (parser.isMalformedTemperature()) {
System.err.println("Ignoring possibly corrupt input: " + value);
context.getCounter(Temperature.MALFORMED).increment(1);
} else if (parser.isMissingTemperature()) {
context.getCounter(Temperature.MISSING).increment(1);
}
// dynamic counter
context.getCounter("TemperatureQuality", parser.getQuality()).increment(1);
}
/* (non-Javadoc)
* @see org.apache.hadoop.mapreduce.Mapper#cleanup(org.apache.hadoop.mapreduce.Mapper.Context)
*/
protected void cleanup(Context context) throws IOException, InterruptedException {
if (isValidationMode) {
if (neighborhood.IsInClassificationMode()) {
context.getCounter("Validation", "TruePositive").increment(confMatrix.getTruePos());
context.getCounter("Validation", "FalseNegative").increment(confMatrix.getFalseNeg());
context.getCounter("Validation", "TrueNagative").increment(confMatrix.getTrueNeg());
context.getCounter("Validation", "FalsePositive").increment(confMatrix.getFalsePos());
context.getCounter("Validation", "Accuracy").increment(confMatrix.getAccuracy());
context.getCounter("Validation", "Recall").increment(confMatrix.getRecall());
context.getCounter("Validation", "Precision").increment(confMatrix.getPrecision());
}
}
}
@Override
protected void reduce(
Key_IMOAndRecordTime key, Iterable<TextArrayWritable> LocationList, Context context)
throws IOException, InterruptedException {
try {
context.getCounter(Counters.VESSEL_PROCESSED).increment(1);
String IMO_str = LpadNum(key.getIMO().get(), 7);
long first_pos_time = key.getRecordTime().get();
/////////////////////////////////////////////////////////////////////////////////
// Populate newPoints with new locations
List<VesselLocation> newPoints = new ArrayList<VesselLocation>();
for (TextArrayWritable rowcontent : LocationList) {
// population location
context.getCounter(Counters.LOCATION_ROWS).increment(1);
VesselLocation newlocation = new VesselLocation();
try {
Writable[] content = rowcontent.get();
String Latitude = content[16].toString().trim();
String Longitude = content[15].toString().trim();
String Coordinates = Latitude + "," + Longitude;
String Speed = content[18].toString().trim();
String Destination = content[9].toString().trim();
String Timestamp = content[21].toString().trim().substring(0, 19);
long record_time = DateTime.parse(Timestamp, rawformatter).getMillis();
newlocation.coordinates = Coordinates;
newlocation.recordtime = record_time;
newlocation.speed = Speed;
newlocation.destination = Destination;
context.getCounter(Counters.LOCATION_VALID).increment(1);
} catch (Exception e) {
e.printStackTrace();
context.getCounter(Counters.LOCATION_ERROR).increment(1);
continue;
}
newPoints.add(newlocation);
}
/////////////////////////////////////////////////////////////////////////////////
// Get last new post time
long last_pos_time = newPoints.get(newPoints.size() - 1).recordtime;
////////////////////////////////////////////////////////////////////////////////
// Get Existing trackinfo
VesselTrackInfo VTI = getTrackInfo(TrackInfo_Table, IMO_str);
List<VesselLocation> AllBetweenPoints = new ArrayList<VesselLocation>();
String BeforeRowKey = null;
String AfterRowKey = null;
// //////////////////////////////////////////////////////////////////////////////
// Retrieve all the existing locations between the first new location and the last new
// location.
if ((VTI.FirstRecordTime != null) && (VTI.LastRecordTime != null)) {
if (last_pos_time < VTI.FirstRecordTime) {
AfterRowKey = IMO_str + LpadNum(Long.MAX_VALUE - VTI.FirstRecordTime, 19);
} else if (first_pos_time > VTI.LastRecordTime) {
BeforeRowKey = IMO_str + LpadNum(Long.MAX_VALUE - VTI.LastRecordTime, 19);
} else {
AllBetweenPoints =
ImportReducer.getLocationsBetween(
VTLocation_Table, IMO_str, first_pos_time, last_pos_time);
if (AllBetweenPoints.size() == 0) {
// Search for the first DB point before the first new point
VesselLocation BeforeLocation =
getLocationBefore(VTLocation_Table, IMO_str, first_pos_time);
BeforeRowKey = IMO_str + LpadNum(Long.MAX_VALUE - BeforeLocation.recordtime, 19);
AfterRowKey = BeforeLocation.nextlocation;
} else {
java.util.Collections.sort(AllBetweenPoints);
BeforeRowKey = AllBetweenPoints.get(0).previouslocation;
AfterRowKey = AllBetweenPoints.get(AllBetweenPoints.size() - 1).nextlocation;
}
List<Delete> deletes =
ImportReducer.GetDeleteEventsBetween(
VTEvent_Table, IMO_str, first_pos_time, last_pos_time);
ImportReducer.DeleteEvents(VTEvent, deletes);
VTEvent.flush();
}
}
// Find out the location before the first new location in
VesselLocation BeforeLocation = getLocation(VTLocation_Table, BeforeRowKey);
// Find out the location after the last new location in
VesselLocation AfterLocation = getLocation(VTLocation_Table, AfterRowKey);
Map<Integer, VesselEvent> PreviousZoneEvents = new HashMap<Integer, VesselEvent>();
;
Map<Integer, VesselEvent> AfterZoneEvents = new HashMap<Integer, VesselEvent>();
if (BeforeLocation != null) {
// Get all events with exit at last location
PreviousZoneEvents =
getAllEventsStartBeforeEndAfterBeforeLocation(VTEvent_Table, IMO_str, BeforeLocation);
}
////////////////////////////////////////////////////
// Analyze and calculate previous and next location
for (VesselLocation newlocation : newPoints) {
int index = AllBetweenPoints.indexOf(newlocation);
if (index != -1) {
VesselLocation dblocation = AllBetweenPoints.get(index);
dblocation.coordinates = newlocation.coordinates;
dblocation.destination = newlocation.destination;
dblocation.speed = newlocation.speed;
} else {
AllBetweenPoints.add(newlocation);
}
}
java.util.Collections.sort(AllBetweenPoints);
String previousRowKey = null;
for (VesselLocation location : AllBetweenPoints) {
location.previouslocation = previousRowKey;
previousRowKey = IMO_str + LpadNum(Long.MAX_VALUE - location.recordtime, 19);
}
String NextRowKey = null;
for (int i = (AllBetweenPoints.size() - 1); i >= 0; i--) {
VesselLocation location = AllBetweenPoints.get(i);
location.nextlocation = NextRowKey;
NextRowKey = IMO_str + LpadNum(Long.MAX_VALUE - location.recordtime, 19);
}
AllBetweenPoints.get(0).previouslocation = BeforeRowKey;
AllBetweenPoints.get(AllBetweenPoints.size() - 1).nextlocation = AfterRowKey;
////////////////////////////////////////////////////
// Upsert all locations
for (VesselLocation location : AllBetweenPoints) {
// population location
try {
byte[] rowkey =
Bytes.toBytes(IMO_str + LpadNum(Long.MAX_VALUE - location.recordtime, 19));
Put put = new Put(rowkey);
put.addColumn(details, speed, Bytes.toBytes(location.speed));
put.addColumn(details, destination, Bytes.toBytes(location.destination));
put.addColumn(details, coordinates, Bytes.toBytes(location.coordinates));
put.addColumn(
details,
timestamp,
Bytes.toBytes(new DateTime(location.recordtime).toString(rawformatter)));
if (location.previouslocation != null) {
put.addColumn(details, previouslocation, Bytes.toBytes(location.previouslocation));
}
if (location.nextlocation != null) {
put.addColumn(details, nextlocation, Bytes.toBytes(location.nextlocation));
}
VTLocation.mutate(put);
} catch (Exception e) {
e.printStackTrace();
context.getCounter(Counters.LOCATION_ERROR).increment(1);
continue;
}
}
// update before next location and after previous location
if (BeforeRowKey != null) {
Put BeforeLocationPut = new Put(Bytes.toBytes(BeforeRowKey));
BeforeLocationPut.addColumn(
details,
nextlocation,
Bytes.toBytes(
IMO_str + LpadNum(Long.MAX_VALUE - AllBetweenPoints.get(0).recordtime, 19)));
VTLocation.mutate(BeforeLocationPut);
}
if (AfterRowKey != null) {
Put AfterLocationPut = new Put(Bytes.toBytes(AfterRowKey));
AfterLocationPut.addColumn(
details,
previouslocation,
Bytes.toBytes(
IMO_str
+ LpadNum(
Long.MAX_VALUE
- AllBetweenPoints.get(AllBetweenPoints.size() - 1).recordtime,
19)));
VTLocation.mutate(AfterLocationPut);
}
VTLocation.flush();
/////////////////////////////////////////////////////////////////////
// Store latest location
// rowkey: global zone id (4)+ longlat22
// ((long11(sign(1)+integer(3)+digit(7)))(lat11(sign(1)+integer(3)+(7))))+imo(7)+recordtime(19)
/////////////////////////////////////////////////////////////////////
Put vessel_track_info = new Put(Bytes.toBytes(IMO_str));
if (AfterLocation == null) {
// Get the last location
VesselLocation lastLocation = AllBetweenPoints.get(AllBetweenPoints.size() - 1);
// update the last location
String[] longlat = lastLocation.coordinates.split(",");
GeometryFactory geometryFactory = JTSFactoryFinder.getGeometryFactory(null);
Coordinate coord =
new Coordinate(Double.parseDouble(longlat[1]), Double.parseDouble(longlat[0]));
Point point = geometryFactory.createPoint(coord);
Integer BelongedGlobalZoneIndex = null;
for (int i = 0; i < VesselZone.GlobalZones.length; i++) {
if (VesselZone.GlobalZones[i].covers(point)) {
BelongedGlobalZoneIndex = i;
break;
}
}
if (VTI.LastLocation != null) {
LastLocation_BM.mutate(new Delete(VTI.LastLocation));
}
byte[] lastlocationrowkey =
Bytes.toBytes(
LpadNum(BelongedGlobalZoneIndex, 4)
+ ConvertCoordinatesToStr(longlat[1])
+ ConvertCoordinatesToStr(longlat[0]));
Put lastlocation_put = new Put(lastlocationrowkey);
lastlocation_put.addColumn(details, imo, Bytes.toBytes(IMO_str));
lastlocation_put.addColumn(
details,
timestamp,
Bytes.toBytes(new DateTime(lastLocation.recordtime).toString(rawformatter)));
LastLocation_BM.mutate(lastlocation_put);
LastLocation_BM.flush();
vessel_track_info.addColumn(details, lastlocation, lastlocationrowkey);
vessel_track_info.addColumn(
details,
lastrecordtime,
Bytes.toBytes(new DateTime(lastLocation.recordtime).toString(rawformatter)));
} else {
// Get events that start before last new location and end after last new location
AfterZoneEvents =
getAllEventsStartBeforeEndAfter(VTEvent_Table, IMO_str, AfterLocation.recordtime);
}
// update firstrecordtime and lastrecordtime
if (BeforeLocation == null) {
vessel_track_info.addColumn(
details,
firstrecordtime,
Bytes.toBytes(
new DateTime(AllBetweenPoints.get(0).recordtime).toString(rawformatter)));
}
if (!vessel_track_info.isEmpty()) {
TrackInfo_BM.mutate(vessel_track_info);
TrackInfo_BM.flush();
}
////////////////////////////////////////////////////////////////////
ArrayList<VesselEvent> DerivedEventList = new ArrayList<VesselEvent>();
///////////////////////////////////////////////////////////
// Get Vessel
String VesselType = getVesselType(Vessel_Table, IMO_str);
if (VesselType == null) {
context.getCounter(Counters.VESSEL_WITHOUTTYPE).increment(1);
return;
}
// calculating event
for (VesselLocation VL : AllBetweenPoints) {
ArrayList<Integer> CurrentZones = LocateCurrentZone(VL.coordinates, VesselType, Zonemap);
Iterator<Map.Entry<Integer, VesselEvent>> it = PreviousZoneEvents.entrySet().iterator();
while (it.hasNext()) {
Map.Entry<Integer, VesselEvent> thisEntry = it.next();
int Zone_Axsmarine_id = thisEntry.getKey();
if (!CurrentZones.contains(Zone_Axsmarine_id)) {
VesselEvent PreviousEvent = thisEntry.getValue();
if (!DerivedEventList.contains(PreviousEvent)) {
DerivedEventList.add(PreviousEvent);
}
// remove close event from PreviousZoneEvents;
it.remove();
}
}
for (Integer thisZone_Axsmarine_id : CurrentZones) {
if (PreviousZoneEvents.containsKey(thisZone_Axsmarine_id)) {
//////////////////////////////////////////////////
// For current zones which both previous and current locations belong to, update exit
// point of previous open events with current locations.
//////////////////////////////////////////////////
VesselEvent PreviousEvent = PreviousZoneEvents.get(thisZone_Axsmarine_id);
PreviousEvent.exitcoordinates = VL.coordinates;
PreviousEvent.exittime = VL.recordtime;
PreviousEvent.destination = VL.destination;
if (!DerivedEventList.contains(PreviousEvent)) {
DerivedEventList.add(PreviousEvent);
}
} else {
//////////////////////////////////////////////////
// For current zones which only current locations belong to, fire new open events
//////////////////////////////////////////////////
VesselEvent NewEvent = new VesselEvent();
NewEvent.entrycoordinates = VL.coordinates;
NewEvent.entrytime = VL.recordtime;
NewEvent.exitcoordinates = VL.coordinates;
NewEvent.exittime = VL.recordtime;
NewEvent.destination = VL.destination;
NewEvent.polygonid = thisZone_Axsmarine_id;
PreviousZoneEvents.put(thisZone_Axsmarine_id, NewEvent);
DerivedEventList.add(NewEvent);
}
}
}
///////////////////////////////////////////////////////////////////////////////////////
// Merge with PreviousZoneEvents with AfterZoneEvents
Iterator<Map.Entry<Integer, VesselEvent>> it = AfterZoneEvents.entrySet().iterator();
while (it.hasNext()) {
Map.Entry<Integer, VesselEvent> thisEntry = it.next();
int Zone_Axsmarine_id = thisEntry.getKey();
VesselEvent After_VE = thisEntry.getValue();
VesselEvent Previous_VE = PreviousZoneEvents.get(Zone_Axsmarine_id);
if (Previous_VE != null) {
Previous_VE.exitcoordinates = After_VE.exitcoordinates;
Previous_VE.exittime = After_VE.exittime;
Previous_VE.destination = After_VE.destination;
if (!DerivedEventList.contains(Previous_VE)) {
DerivedEventList.add(Previous_VE);
}
} else {
VesselEvent NewEvent = new VesselEvent();
NewEvent.entrycoordinates = AfterLocation.coordinates;
NewEvent.entrytime = AfterLocation.recordtime;
NewEvent.exitcoordinates = After_VE.exitcoordinates;
NewEvent.exittime = After_VE.exittime;
NewEvent.destination = After_VE.destination;
NewEvent.polygonid = Zone_Axsmarine_id;
DerivedEventList.add(NewEvent);
}
// Delete This Event from HBase
DeleteEvent(VTEvent, IMO_str, After_VE);
}
VTEvent.flush();
// pupulate Derived Events into Hbase
for (VesselEvent newEvent : DerivedEventList) {
// rowkey: IMO(7)+timestamp(19 desc)+polygonid(8)
// qualifier:entrytime,entrycoordinates,exittime,exitcoordinates,destination
context.getCounter(Counters.EVENT_UPSERTS).increment(1);
byte[] rowkey =
Bytes.toBytes(
IMO_str
+ LpadNum(Long.MAX_VALUE - newEvent.entrytime, 19)
+ LpadNum(newEvent.polygonid, 10));
Put put = new Put(rowkey);
put.addColumn(
details,
entrytime,
Bytes.toBytes(new DateTime(newEvent.entrytime).toString(rawformatter)));
put.addColumn(details, entrycoordinates, Bytes.toBytes(newEvent.entrycoordinates));
put.addColumn(
details,
exittime,
Bytes.toBytes(new DateTime(newEvent.exittime).toString(rawformatter)));
put.addColumn(details, exitcoordinates, Bytes.toBytes(newEvent.exitcoordinates));
put.addColumn(details, destination, Bytes.toBytes(newEvent.destination));
VTEvent.mutate(put);
context.getCounter(Counters.EVENT_VALID).increment(1);
}
// VTLocation.flush(); Moved to the first step
VTEvent.flush();
} catch (RuntimeException e) {
// TODO Auto-generated catch block
System.out.println("Exception occured while loading data for:" + key.getIMO());
throw e;
}
}