@Override protected void map( BytesWritable key, ArrayWritable value, Mapper<BytesWritable, ArrayWritable, BytesWritable, TextArrayWritable>.Context context) throws IOException, InterruptedException { Map<String, Report> reports = new LinkedHashMap<String, Report>(); Writable[] repts = value.get(); if (repts.length == 0 || !(repts[0] instanceof Text)) { System.out.println("error: bad input."); return; // bail out more drastically } Text[] repts_as_text = (Text[]) repts; for (Text t : repts_as_text) { Report r = Report.createFromString(t.toString()); reports.put(r.getMetadata().getOpIdString(), r); } Text[] indexed = indexGraph(reports); TextArrayWritable output = new TextArrayWritable(); output.set(indexed); context.write(key, output); }
@Override public void reduce(NullWritable key, Iterable<TextArrayWritable> values, Context context) throws IOException, InterruptedException { // TODO // TextArrayWritable tmp; // while(values.hasNext()){ for (TextArrayWritable value : values) { for (Writable w : value.get()) { String str = ((Text) w).toString(); String arrStr[] = str.split("\\|"); tree.add(new Pair(Integer.parseInt(arrStr[0]), arrStr[1])); } } // reverse tree set TreeSet<Pair> treereverse = new TreeSet<Pair>(); treereverse = (TreeSet) tree.descendingSet(); Iterator itr = treereverse.iterator(); int count = 1; while (itr.hasNext() && count != N + 1) { Pair p = (Pair) itr.next(); // send top N entry context.write( new Text(p.second.toString()), new IntWritable(Integer.parseInt(p.first.toString()))); count++; } }
@Override public void reduce(NullWritable key, Iterable<TextArrayWritable> values, Context context) throws IOException, InterruptedException { Integer sum, mean, max, min, var; // TODO for (TextArrayWritable val : values) { Text[] pair = (Text[]) val.toArray(); String title = pair[0].toString(); Integer count = Integer.parseInt(pair[1].toString()); countToTitleMap.add(new Pair<Integer, String>(count, title)); if (countToTitleMap.size() > this.N) { countToTitleMap.remove(countToTitleMap.first()); } } // calculate the statistics Integer[] counts = new Integer[this.N]; int j = 0; for (Pair<Integer, String> pair : countToTitleMap) { counts[j++] = pair.first; } sum = 0; min = Integer.MAX_VALUE; max = Integer.MIN_VALUE; for (int i = 0; i < this.N; i++) { sum += counts[i]; min = Math.min(min, counts[i]); max = Math.max(max, counts[i]); } mean = sum / this.N; var = 0; for (int i = 0; i < this.N; i++) { var += ((counts[i] - mean) * (counts[i] - mean)); } var /= this.N; context.write(new Text("Mean"), new IntWritable(mean)); context.write(new Text("Sum"), new IntWritable(sum)); context.write(new Text("Min"), new IntWritable(min)); context.write(new Text("Max"), new IntWritable(max)); context.write(new Text("Var"), new IntWritable(var)); }
@Override public void reduce(NullWritable key, Iterable<TextArrayWritable> values, Context context) throws IOException, InterruptedException { TreeSet<Pair<Integer, String>> all = new TreeSet<Pair<Integer, String>>(); for (TextArrayWritable t : values) { String[] line = t.toStrings(); Pair<Integer, String> p = Pair.of(Integer.parseInt(line[1]), line[0]); all.add(p); } ArrayList<Pair<Integer, String>> result = new ArrayList<Pair<Integer, String>>(all); for (int i = result.size() - 1; i >= result.size() - N; i--) { context.write(new Text(result.get(i).second), new IntWritable(result.get(i).first)); } }
@Override public void reduce(NullWritable key, Iterable<TextArrayWritable> values, Context context) throws IOException, InterruptedException { int sum, mean, max, min, var; sum = mean = max = min = var = 0; for (TextArrayWritable val : values) { Text[] pair = (Text[]) val.toArray(); String word = pair[0].toString(); Integer count = Integer.parseInt(pair[1].toString()); sortedWordCount.add(new Pair<Integer, String>(count, word)); if (sortedWordCount.size() > N) { sortedWordCount.remove(sortedWordCount.first()); } } for (Pair<Integer, String> entry : sortedWordCount) { // Text word = new Text(entry.second); // IntWritable value = new IntWritable(entry.first); // context.write(word, value); sum += entry.first; } mean = (sum / sortedWordCount.size()); min = sortedWordCount.first().first; max = sortedWordCount.last().first; for (Pair<Integer, String> entry : sortedWordCount) { var += ((entry.first - mean) * (entry.first - mean)); } var = var / sortedWordCount.size(); context.write(new Text("Mean"), new IntWritable(mean)); context.write(new Text("Sum"), new IntWritable(sum)); context.write(new Text("Min"), new IntWritable(min)); context.write(new Text("Max"), new IntWritable(max)); context.write(new Text("Var"), new IntWritable(var)); }
@Override public void reduce(NullWritable key, Iterable<TextArrayWritable> values, Context context) throws IOException, InterruptedException { for (TextArrayWritable val : values) { Text[] pair = (Text[]) val.toArray(); String word = pair[0].toString(); Float avg = Float.parseFloat(pair[1].toString()); /* Text tWord = new Text(word); FloatWritable value = new FloatWritable(avg); context.write(tWord, value); */ String[] parts = word.split("-"); String newKey = parts[0] + "-" + parts[1]; String carrier = parts[2]; if (!carrierAvgDelay.containsKey(newKey)) { TreeSet<Pair<Float, String>> avgDelayMap = new TreeSet<Pair<Float, String>>(); carrierAvgDelay.put(newKey, avgDelayMap); } TreeSet<Pair<Float, String>> ts = carrierAvgDelay.get(newKey); ts.add(new Pair<Float, String>(avg, carrier)); if (ts.size() > 10) { ts.remove(ts.last()); } carrierAvgDelay.put(newKey, ts); } Iterator it = carrierAvgDelay.entrySet().iterator(); while (it.hasNext()) { Map.Entry pair = (Map.Entry) it.next(); TreeSet<Pair<Float, String>> ts = (TreeSet<Pair<Float, String>>) pair.getValue(); for (Pair<Float, String> item : ts) { Text word = new Text(pair.getKey() + "-" + item.second); FloatWritable value = new FloatWritable(item.first); context.write(word, value); } } }
@Override public void reduce(NullWritable key, Iterable<TextArrayWritable> values, Context context) throws IOException, InterruptedException { for (TextArrayWritable val : values) { Text[] pair = (Text[]) val.toArray(); String word = pair[0].toString(); Integer count = Integer.parseInt(pair[1].toString()); countToWordMap.add(new Pair<Integer, String>(count, word)); if (countToWordMap.size() > N) { countToWordMap.remove(countToWordMap.first()); } } for (Pair<Integer, String> item : countToWordMap) { Text word = new Text(item.second); IntWritable value = new IntWritable(item.first); context.write(word, value); } }
@Override protected void reduce( Key_IMOAndRecordTime key, Iterable<TextArrayWritable> LocationList, Context context) throws IOException, InterruptedException { try { context.getCounter(Counters.VESSEL_PROCESSED).increment(1); String IMO_str = LpadNum(key.getIMO().get(), 7); long first_pos_time = key.getRecordTime().get(); ///////////////////////////////////////////////////////////////////////////////// // Populate newPoints with new locations List<VesselLocation> newPoints = new ArrayList<VesselLocation>(); for (TextArrayWritable rowcontent : LocationList) { // population location context.getCounter(Counters.LOCATION_ROWS).increment(1); VesselLocation newlocation = new VesselLocation(); try { Writable[] content = rowcontent.get(); String Latitude = content[16].toString().trim(); String Longitude = content[15].toString().trim(); String Coordinates = Latitude + "," + Longitude; String Speed = content[18].toString().trim(); String Destination = content[9].toString().trim(); String Timestamp = content[21].toString().trim().substring(0, 19); long record_time = DateTime.parse(Timestamp, rawformatter).getMillis(); newlocation.coordinates = Coordinates; newlocation.recordtime = record_time; newlocation.speed = Speed; newlocation.destination = Destination; context.getCounter(Counters.LOCATION_VALID).increment(1); } catch (Exception e) { e.printStackTrace(); context.getCounter(Counters.LOCATION_ERROR).increment(1); continue; } newPoints.add(newlocation); } ///////////////////////////////////////////////////////////////////////////////// // Get last new post time long last_pos_time = newPoints.get(newPoints.size() - 1).recordtime; //////////////////////////////////////////////////////////////////////////////// // Get Existing trackinfo VesselTrackInfo VTI = getTrackInfo(TrackInfo_Table, IMO_str); List<VesselLocation> AllBetweenPoints = new ArrayList<VesselLocation>(); String BeforeRowKey = null; String AfterRowKey = null; // ////////////////////////////////////////////////////////////////////////////// // Retrieve all the existing locations between the first new location and the last new // location. if ((VTI.FirstRecordTime != null) && (VTI.LastRecordTime != null)) { if (last_pos_time < VTI.FirstRecordTime) { AfterRowKey = IMO_str + LpadNum(Long.MAX_VALUE - VTI.FirstRecordTime, 19); } else if (first_pos_time > VTI.LastRecordTime) { BeforeRowKey = IMO_str + LpadNum(Long.MAX_VALUE - VTI.LastRecordTime, 19); } else { AllBetweenPoints = ImportReducer.getLocationsBetween( VTLocation_Table, IMO_str, first_pos_time, last_pos_time); if (AllBetweenPoints.size() == 0) { // Search for the first DB point before the first new point VesselLocation BeforeLocation = getLocationBefore(VTLocation_Table, IMO_str, first_pos_time); BeforeRowKey = IMO_str + LpadNum(Long.MAX_VALUE - BeforeLocation.recordtime, 19); AfterRowKey = BeforeLocation.nextlocation; } else { java.util.Collections.sort(AllBetweenPoints); BeforeRowKey = AllBetweenPoints.get(0).previouslocation; AfterRowKey = AllBetweenPoints.get(AllBetweenPoints.size() - 1).nextlocation; } List<Delete> deletes = ImportReducer.GetDeleteEventsBetween( VTEvent_Table, IMO_str, first_pos_time, last_pos_time); ImportReducer.DeleteEvents(VTEvent, deletes); VTEvent.flush(); } } // Find out the location before the first new location in VesselLocation BeforeLocation = getLocation(VTLocation_Table, BeforeRowKey); // Find out the location after the last new location in VesselLocation AfterLocation = getLocation(VTLocation_Table, AfterRowKey); Map<Integer, VesselEvent> PreviousZoneEvents = new HashMap<Integer, VesselEvent>(); ; Map<Integer, VesselEvent> AfterZoneEvents = new HashMap<Integer, VesselEvent>(); if (BeforeLocation != null) { // Get all events with exit at last location PreviousZoneEvents = getAllEventsStartBeforeEndAfterBeforeLocation(VTEvent_Table, IMO_str, BeforeLocation); } //////////////////////////////////////////////////// // Analyze and calculate previous and next location for (VesselLocation newlocation : newPoints) { int index = AllBetweenPoints.indexOf(newlocation); if (index != -1) { VesselLocation dblocation = AllBetweenPoints.get(index); dblocation.coordinates = newlocation.coordinates; dblocation.destination = newlocation.destination; dblocation.speed = newlocation.speed; } else { AllBetweenPoints.add(newlocation); } } java.util.Collections.sort(AllBetweenPoints); String previousRowKey = null; for (VesselLocation location : AllBetweenPoints) { location.previouslocation = previousRowKey; previousRowKey = IMO_str + LpadNum(Long.MAX_VALUE - location.recordtime, 19); } String NextRowKey = null; for (int i = (AllBetweenPoints.size() - 1); i >= 0; i--) { VesselLocation location = AllBetweenPoints.get(i); location.nextlocation = NextRowKey; NextRowKey = IMO_str + LpadNum(Long.MAX_VALUE - location.recordtime, 19); } AllBetweenPoints.get(0).previouslocation = BeforeRowKey; AllBetweenPoints.get(AllBetweenPoints.size() - 1).nextlocation = AfterRowKey; //////////////////////////////////////////////////// // Upsert all locations for (VesselLocation location : AllBetweenPoints) { // population location try { byte[] rowkey = Bytes.toBytes(IMO_str + LpadNum(Long.MAX_VALUE - location.recordtime, 19)); Put put = new Put(rowkey); put.addColumn(details, speed, Bytes.toBytes(location.speed)); put.addColumn(details, destination, Bytes.toBytes(location.destination)); put.addColumn(details, coordinates, Bytes.toBytes(location.coordinates)); put.addColumn( details, timestamp, Bytes.toBytes(new DateTime(location.recordtime).toString(rawformatter))); if (location.previouslocation != null) { put.addColumn(details, previouslocation, Bytes.toBytes(location.previouslocation)); } if (location.nextlocation != null) { put.addColumn(details, nextlocation, Bytes.toBytes(location.nextlocation)); } VTLocation.mutate(put); } catch (Exception e) { e.printStackTrace(); context.getCounter(Counters.LOCATION_ERROR).increment(1); continue; } } // update before next location and after previous location if (BeforeRowKey != null) { Put BeforeLocationPut = new Put(Bytes.toBytes(BeforeRowKey)); BeforeLocationPut.addColumn( details, nextlocation, Bytes.toBytes( IMO_str + LpadNum(Long.MAX_VALUE - AllBetweenPoints.get(0).recordtime, 19))); VTLocation.mutate(BeforeLocationPut); } if (AfterRowKey != null) { Put AfterLocationPut = new Put(Bytes.toBytes(AfterRowKey)); AfterLocationPut.addColumn( details, previouslocation, Bytes.toBytes( IMO_str + LpadNum( Long.MAX_VALUE - AllBetweenPoints.get(AllBetweenPoints.size() - 1).recordtime, 19))); VTLocation.mutate(AfterLocationPut); } VTLocation.flush(); ///////////////////////////////////////////////////////////////////// // Store latest location // rowkey: global zone id (4)+ longlat22 // ((long11(sign(1)+integer(3)+digit(7)))(lat11(sign(1)+integer(3)+(7))))+imo(7)+recordtime(19) ///////////////////////////////////////////////////////////////////// Put vessel_track_info = new Put(Bytes.toBytes(IMO_str)); if (AfterLocation == null) { // Get the last location VesselLocation lastLocation = AllBetweenPoints.get(AllBetweenPoints.size() - 1); // update the last location String[] longlat = lastLocation.coordinates.split(","); GeometryFactory geometryFactory = JTSFactoryFinder.getGeometryFactory(null); Coordinate coord = new Coordinate(Double.parseDouble(longlat[1]), Double.parseDouble(longlat[0])); Point point = geometryFactory.createPoint(coord); Integer BelongedGlobalZoneIndex = null; for (int i = 0; i < VesselZone.GlobalZones.length; i++) { if (VesselZone.GlobalZones[i].covers(point)) { BelongedGlobalZoneIndex = i; break; } } if (VTI.LastLocation != null) { LastLocation_BM.mutate(new Delete(VTI.LastLocation)); } byte[] lastlocationrowkey = Bytes.toBytes( LpadNum(BelongedGlobalZoneIndex, 4) + ConvertCoordinatesToStr(longlat[1]) + ConvertCoordinatesToStr(longlat[0])); Put lastlocation_put = new Put(lastlocationrowkey); lastlocation_put.addColumn(details, imo, Bytes.toBytes(IMO_str)); lastlocation_put.addColumn( details, timestamp, Bytes.toBytes(new DateTime(lastLocation.recordtime).toString(rawformatter))); LastLocation_BM.mutate(lastlocation_put); LastLocation_BM.flush(); vessel_track_info.addColumn(details, lastlocation, lastlocationrowkey); vessel_track_info.addColumn( details, lastrecordtime, Bytes.toBytes(new DateTime(lastLocation.recordtime).toString(rawformatter))); } else { // Get events that start before last new location and end after last new location AfterZoneEvents = getAllEventsStartBeforeEndAfter(VTEvent_Table, IMO_str, AfterLocation.recordtime); } // update firstrecordtime and lastrecordtime if (BeforeLocation == null) { vessel_track_info.addColumn( details, firstrecordtime, Bytes.toBytes( new DateTime(AllBetweenPoints.get(0).recordtime).toString(rawformatter))); } if (!vessel_track_info.isEmpty()) { TrackInfo_BM.mutate(vessel_track_info); TrackInfo_BM.flush(); } //////////////////////////////////////////////////////////////////// ArrayList<VesselEvent> DerivedEventList = new ArrayList<VesselEvent>(); /////////////////////////////////////////////////////////// // Get Vessel String VesselType = getVesselType(Vessel_Table, IMO_str); if (VesselType == null) { context.getCounter(Counters.VESSEL_WITHOUTTYPE).increment(1); return; } // calculating event for (VesselLocation VL : AllBetweenPoints) { ArrayList<Integer> CurrentZones = LocateCurrentZone(VL.coordinates, VesselType, Zonemap); Iterator<Map.Entry<Integer, VesselEvent>> it = PreviousZoneEvents.entrySet().iterator(); while (it.hasNext()) { Map.Entry<Integer, VesselEvent> thisEntry = it.next(); int Zone_Axsmarine_id = thisEntry.getKey(); if (!CurrentZones.contains(Zone_Axsmarine_id)) { VesselEvent PreviousEvent = thisEntry.getValue(); if (!DerivedEventList.contains(PreviousEvent)) { DerivedEventList.add(PreviousEvent); } // remove close event from PreviousZoneEvents; it.remove(); } } for (Integer thisZone_Axsmarine_id : CurrentZones) { if (PreviousZoneEvents.containsKey(thisZone_Axsmarine_id)) { ////////////////////////////////////////////////// // For current zones which both previous and current locations belong to, update exit // point of previous open events with current locations. ////////////////////////////////////////////////// VesselEvent PreviousEvent = PreviousZoneEvents.get(thisZone_Axsmarine_id); PreviousEvent.exitcoordinates = VL.coordinates; PreviousEvent.exittime = VL.recordtime; PreviousEvent.destination = VL.destination; if (!DerivedEventList.contains(PreviousEvent)) { DerivedEventList.add(PreviousEvent); } } else { ////////////////////////////////////////////////// // For current zones which only current locations belong to, fire new open events ////////////////////////////////////////////////// VesselEvent NewEvent = new VesselEvent(); NewEvent.entrycoordinates = VL.coordinates; NewEvent.entrytime = VL.recordtime; NewEvent.exitcoordinates = VL.coordinates; NewEvent.exittime = VL.recordtime; NewEvent.destination = VL.destination; NewEvent.polygonid = thisZone_Axsmarine_id; PreviousZoneEvents.put(thisZone_Axsmarine_id, NewEvent); DerivedEventList.add(NewEvent); } } } /////////////////////////////////////////////////////////////////////////////////////// // Merge with PreviousZoneEvents with AfterZoneEvents Iterator<Map.Entry<Integer, VesselEvent>> it = AfterZoneEvents.entrySet().iterator(); while (it.hasNext()) { Map.Entry<Integer, VesselEvent> thisEntry = it.next(); int Zone_Axsmarine_id = thisEntry.getKey(); VesselEvent After_VE = thisEntry.getValue(); VesselEvent Previous_VE = PreviousZoneEvents.get(Zone_Axsmarine_id); if (Previous_VE != null) { Previous_VE.exitcoordinates = After_VE.exitcoordinates; Previous_VE.exittime = After_VE.exittime; Previous_VE.destination = After_VE.destination; if (!DerivedEventList.contains(Previous_VE)) { DerivedEventList.add(Previous_VE); } } else { VesselEvent NewEvent = new VesselEvent(); NewEvent.entrycoordinates = AfterLocation.coordinates; NewEvent.entrytime = AfterLocation.recordtime; NewEvent.exitcoordinates = After_VE.exitcoordinates; NewEvent.exittime = After_VE.exittime; NewEvent.destination = After_VE.destination; NewEvent.polygonid = Zone_Axsmarine_id; DerivedEventList.add(NewEvent); } // Delete This Event from HBase DeleteEvent(VTEvent, IMO_str, After_VE); } VTEvent.flush(); // pupulate Derived Events into Hbase for (VesselEvent newEvent : DerivedEventList) { // rowkey: IMO(7)+timestamp(19 desc)+polygonid(8) // qualifier:entrytime,entrycoordinates,exittime,exitcoordinates,destination context.getCounter(Counters.EVENT_UPSERTS).increment(1); byte[] rowkey = Bytes.toBytes( IMO_str + LpadNum(Long.MAX_VALUE - newEvent.entrytime, 19) + LpadNum(newEvent.polygonid, 10)); Put put = new Put(rowkey); put.addColumn( details, entrytime, Bytes.toBytes(new DateTime(newEvent.entrytime).toString(rawformatter))); put.addColumn(details, entrycoordinates, Bytes.toBytes(newEvent.entrycoordinates)); put.addColumn( details, exittime, Bytes.toBytes(new DateTime(newEvent.exittime).toString(rawformatter))); put.addColumn(details, exitcoordinates, Bytes.toBytes(newEvent.exitcoordinates)); put.addColumn(details, destination, Bytes.toBytes(newEvent.destination)); VTEvent.mutate(put); context.getCounter(Counters.EVENT_VALID).increment(1); } // VTLocation.flush(); Moved to the first step VTEvent.flush(); } catch (RuntimeException e) { // TODO Auto-generated catch block System.out.println("Exception occured while loading data for:" + key.getIMO()); throw e; } }