@Override
  protected void reduce(Text key, Iterable<LongWritable> values, Context context)
      throws IOException, InterruptedException {

    /*
     * Added by Xudong Zhang
     */
    if (parseWeibo) {
      String word = key.toString();
      if (!ChineseUtils.allChinese(word)) {
        context.getCounter("MyCounter", "NumWordsFilteredByWeiboParser").increment(1);
        return;
      }
    }

    long sum = 0;
    for (LongWritable value : values) {
      sum += value.get();
    }
    if (sum >= minSupport) {
      context.write(key, new LongWritable(sum));
    } else {
      context.getCounter("MyCounter", "NumWordsLessThanMinSupport").increment(1);
    }
  }
    @Override
    protected void map(LongWritable key, Text value, Context context)
        throws IOException, InterruptedException {

      parser.parse(value);
      if (parser.isValidTemperature()) {
        int airTemperature = parser.getAirTemperature();
        context.write(new Text(parser.getYear()), new IntWritable(airTemperature));
      } else if (parser.isMalformedTemperature()) {
        System.err.println("Ignoring possibly corrupt input: " + value);
        context.getCounter(Temperature.MALFORMED).increment(1);
      } else if (parser.isMissingTemperature()) {
        context.getCounter(Temperature.MISSING).increment(1);
      }

      // dynamic counter
      context.getCounter("TemperatureQuality", parser.getQuality()).increment(1);
    }
Beispiel #3
0
 /* (non-Javadoc)
  * @see org.apache.hadoop.mapreduce.Mapper#cleanup(org.apache.hadoop.mapreduce.Mapper.Context)
  */
 protected void cleanup(Context context) throws IOException, InterruptedException {
   if (isValidationMode) {
     if (neighborhood.IsInClassificationMode()) {
       context.getCounter("Validation", "TruePositive").increment(confMatrix.getTruePos());
       context.getCounter("Validation", "FalseNegative").increment(confMatrix.getFalseNeg());
       context.getCounter("Validation", "TrueNagative").increment(confMatrix.getTrueNeg());
       context.getCounter("Validation", "FalsePositive").increment(confMatrix.getFalsePos());
       context.getCounter("Validation", "Accuracy").increment(confMatrix.getAccuracy());
       context.getCounter("Validation", "Recall").increment(confMatrix.getRecall());
       context.getCounter("Validation", "Precision").increment(confMatrix.getPrecision());
     }
   }
 }
    @Override
    protected void reduce(
        Key_IMOAndRecordTime key, Iterable<TextArrayWritable> LocationList, Context context)
        throws IOException, InterruptedException {

      try {
        context.getCounter(Counters.VESSEL_PROCESSED).increment(1);

        String IMO_str = LpadNum(key.getIMO().get(), 7);
        long first_pos_time = key.getRecordTime().get();

        /////////////////////////////////////////////////////////////////////////////////
        // Populate newPoints with new locations
        List<VesselLocation> newPoints = new ArrayList<VesselLocation>();

        for (TextArrayWritable rowcontent : LocationList) {
          // population location
          context.getCounter(Counters.LOCATION_ROWS).increment(1);
          VesselLocation newlocation = new VesselLocation();

          try {

            Writable[] content = rowcontent.get();
            String Latitude = content[16].toString().trim();
            String Longitude = content[15].toString().trim();
            String Coordinates = Latitude + "," + Longitude;
            String Speed = content[18].toString().trim();
            String Destination = content[9].toString().trim();
            String Timestamp = content[21].toString().trim().substring(0, 19);

            long record_time = DateTime.parse(Timestamp, rawformatter).getMillis();
            newlocation.coordinates = Coordinates;
            newlocation.recordtime = record_time;
            newlocation.speed = Speed;
            newlocation.destination = Destination;
            context.getCounter(Counters.LOCATION_VALID).increment(1);

          } catch (Exception e) {
            e.printStackTrace();
            context.getCounter(Counters.LOCATION_ERROR).increment(1);
            continue;
          }

          newPoints.add(newlocation);
        }

        /////////////////////////////////////////////////////////////////////////////////
        // Get last new post time
        long last_pos_time = newPoints.get(newPoints.size() - 1).recordtime;

        ////////////////////////////////////////////////////////////////////////////////
        // Get Existing trackinfo
        VesselTrackInfo VTI = getTrackInfo(TrackInfo_Table, IMO_str);

        List<VesselLocation> AllBetweenPoints = new ArrayList<VesselLocation>();

        String BeforeRowKey = null;
        String AfterRowKey = null;

        // //////////////////////////////////////////////////////////////////////////////
        // Retrieve all the existing locations between the first new location and the last new
        // location.
        if ((VTI.FirstRecordTime != null) && (VTI.LastRecordTime != null)) {

          if (last_pos_time < VTI.FirstRecordTime) {
            AfterRowKey = IMO_str + LpadNum(Long.MAX_VALUE - VTI.FirstRecordTime, 19);
          } else if (first_pos_time > VTI.LastRecordTime) {
            BeforeRowKey = IMO_str + LpadNum(Long.MAX_VALUE - VTI.LastRecordTime, 19);
          } else {
            AllBetweenPoints =
                ImportReducer.getLocationsBetween(
                    VTLocation_Table, IMO_str, first_pos_time, last_pos_time);

            if (AllBetweenPoints.size() == 0) {
              // Search for the first DB point before the first new point
              VesselLocation BeforeLocation =
                  getLocationBefore(VTLocation_Table, IMO_str, first_pos_time);
              BeforeRowKey = IMO_str + LpadNum(Long.MAX_VALUE - BeforeLocation.recordtime, 19);
              AfterRowKey = BeforeLocation.nextlocation;

            } else {
              java.util.Collections.sort(AllBetweenPoints);
              BeforeRowKey = AllBetweenPoints.get(0).previouslocation;
              AfterRowKey = AllBetweenPoints.get(AllBetweenPoints.size() - 1).nextlocation;
            }

            List<Delete> deletes =
                ImportReducer.GetDeleteEventsBetween(
                    VTEvent_Table, IMO_str, first_pos_time, last_pos_time);
            ImportReducer.DeleteEvents(VTEvent, deletes);
            VTEvent.flush();
          }
        }

        // Find out the location before the first new location in
        VesselLocation BeforeLocation = getLocation(VTLocation_Table, BeforeRowKey);

        // Find out the location after the last new location in
        VesselLocation AfterLocation = getLocation(VTLocation_Table, AfterRowKey);

        Map<Integer, VesselEvent> PreviousZoneEvents = new HashMap<Integer, VesselEvent>();
        ;
        Map<Integer, VesselEvent> AfterZoneEvents = new HashMap<Integer, VesselEvent>();

        if (BeforeLocation != null) {
          // Get all events with exit at last location
          PreviousZoneEvents =
              getAllEventsStartBeforeEndAfterBeforeLocation(VTEvent_Table, IMO_str, BeforeLocation);
        }

        ////////////////////////////////////////////////////
        // Analyze and calculate previous and next location
        for (VesselLocation newlocation : newPoints) {

          int index = AllBetweenPoints.indexOf(newlocation);
          if (index != -1) {
            VesselLocation dblocation = AllBetweenPoints.get(index);
            dblocation.coordinates = newlocation.coordinates;
            dblocation.destination = newlocation.destination;
            dblocation.speed = newlocation.speed;
          } else {
            AllBetweenPoints.add(newlocation);
          }
        }

        java.util.Collections.sort(AllBetweenPoints);

        String previousRowKey = null;

        for (VesselLocation location : AllBetweenPoints) {
          location.previouslocation = previousRowKey;
          previousRowKey = IMO_str + LpadNum(Long.MAX_VALUE - location.recordtime, 19);
        }

        String NextRowKey = null;

        for (int i = (AllBetweenPoints.size() - 1); i >= 0; i--) {
          VesselLocation location = AllBetweenPoints.get(i);
          location.nextlocation = NextRowKey;
          NextRowKey = IMO_str + LpadNum(Long.MAX_VALUE - location.recordtime, 19);
        }

        AllBetweenPoints.get(0).previouslocation = BeforeRowKey;
        AllBetweenPoints.get(AllBetweenPoints.size() - 1).nextlocation = AfterRowKey;

        ////////////////////////////////////////////////////
        // Upsert all locations

        for (VesselLocation location : AllBetweenPoints) {
          // population location
          try {

            byte[] rowkey =
                Bytes.toBytes(IMO_str + LpadNum(Long.MAX_VALUE - location.recordtime, 19));
            Put put = new Put(rowkey);

            put.addColumn(details, speed, Bytes.toBytes(location.speed));
            put.addColumn(details, destination, Bytes.toBytes(location.destination));
            put.addColumn(details, coordinates, Bytes.toBytes(location.coordinates));

            put.addColumn(
                details,
                timestamp,
                Bytes.toBytes(new DateTime(location.recordtime).toString(rawformatter)));

            if (location.previouslocation != null) {
              put.addColumn(details, previouslocation, Bytes.toBytes(location.previouslocation));
            }

            if (location.nextlocation != null) {
              put.addColumn(details, nextlocation, Bytes.toBytes(location.nextlocation));
            }

            VTLocation.mutate(put);

          } catch (Exception e) {
            e.printStackTrace();
            context.getCounter(Counters.LOCATION_ERROR).increment(1);
            continue;
          }
        }

        // update before next location and after previous location

        if (BeforeRowKey != null) {
          Put BeforeLocationPut = new Put(Bytes.toBytes(BeforeRowKey));
          BeforeLocationPut.addColumn(
              details,
              nextlocation,
              Bytes.toBytes(
                  IMO_str + LpadNum(Long.MAX_VALUE - AllBetweenPoints.get(0).recordtime, 19)));
          VTLocation.mutate(BeforeLocationPut);
        }

        if (AfterRowKey != null) {

          Put AfterLocationPut = new Put(Bytes.toBytes(AfterRowKey));
          AfterLocationPut.addColumn(
              details,
              previouslocation,
              Bytes.toBytes(
                  IMO_str
                      + LpadNum(
                          Long.MAX_VALUE
                              - AllBetweenPoints.get(AllBetweenPoints.size() - 1).recordtime,
                          19)));
          VTLocation.mutate(AfterLocationPut);
        }

        VTLocation.flush();

        /////////////////////////////////////////////////////////////////////
        // Store latest location
        // rowkey: global zone id (4)+ longlat22
        // ((long11(sign(1)+integer(3)+digit(7)))(lat11(sign(1)+integer(3)+(7))))+imo(7)+recordtime(19)
        /////////////////////////////////////////////////////////////////////

        Put vessel_track_info = new Put(Bytes.toBytes(IMO_str));

        if (AfterLocation == null) {
          // Get the last location
          VesselLocation lastLocation = AllBetweenPoints.get(AllBetweenPoints.size() - 1);
          // update the last location
          String[] longlat = lastLocation.coordinates.split(",");
          GeometryFactory geometryFactory = JTSFactoryFinder.getGeometryFactory(null);
          Coordinate coord =
              new Coordinate(Double.parseDouble(longlat[1]), Double.parseDouble(longlat[0]));
          Point point = geometryFactory.createPoint(coord);

          Integer BelongedGlobalZoneIndex = null;

          for (int i = 0; i < VesselZone.GlobalZones.length; i++) {
            if (VesselZone.GlobalZones[i].covers(point)) {
              BelongedGlobalZoneIndex = i;
              break;
            }
          }

          if (VTI.LastLocation != null) {
            LastLocation_BM.mutate(new Delete(VTI.LastLocation));
          }

          byte[] lastlocationrowkey =
              Bytes.toBytes(
                  LpadNum(BelongedGlobalZoneIndex, 4)
                      + ConvertCoordinatesToStr(longlat[1])
                      + ConvertCoordinatesToStr(longlat[0]));
          Put lastlocation_put = new Put(lastlocationrowkey);
          lastlocation_put.addColumn(details, imo, Bytes.toBytes(IMO_str));
          lastlocation_put.addColumn(
              details,
              timestamp,
              Bytes.toBytes(new DateTime(lastLocation.recordtime).toString(rawformatter)));
          LastLocation_BM.mutate(lastlocation_put);

          LastLocation_BM.flush();

          vessel_track_info.addColumn(details, lastlocation, lastlocationrowkey);
          vessel_track_info.addColumn(
              details,
              lastrecordtime,
              Bytes.toBytes(new DateTime(lastLocation.recordtime).toString(rawformatter)));

        } else {
          // Get events that start before last new location and end after last new location
          AfterZoneEvents =
              getAllEventsStartBeforeEndAfter(VTEvent_Table, IMO_str, AfterLocation.recordtime);
        }

        // update firstrecordtime and lastrecordtime
        if (BeforeLocation == null) {
          vessel_track_info.addColumn(
              details,
              firstrecordtime,
              Bytes.toBytes(
                  new DateTime(AllBetweenPoints.get(0).recordtime).toString(rawformatter)));
        }

        if (!vessel_track_info.isEmpty()) {
          TrackInfo_BM.mutate(vessel_track_info);
          TrackInfo_BM.flush();
        }

        ////////////////////////////////////////////////////////////////////

        ArrayList<VesselEvent> DerivedEventList = new ArrayList<VesselEvent>();

        ///////////////////////////////////////////////////////////
        // Get Vessel
        String VesselType = getVesselType(Vessel_Table, IMO_str);

        if (VesselType == null) {
          context.getCounter(Counters.VESSEL_WITHOUTTYPE).increment(1);
          return;
        }

        // calculating event
        for (VesselLocation VL : AllBetweenPoints) {
          ArrayList<Integer> CurrentZones = LocateCurrentZone(VL.coordinates, VesselType, Zonemap);

          Iterator<Map.Entry<Integer, VesselEvent>> it = PreviousZoneEvents.entrySet().iterator();

          while (it.hasNext()) {
            Map.Entry<Integer, VesselEvent> thisEntry = it.next();
            int Zone_Axsmarine_id = thisEntry.getKey();
            if (!CurrentZones.contains(Zone_Axsmarine_id)) {
              VesselEvent PreviousEvent = thisEntry.getValue();

              if (!DerivedEventList.contains(PreviousEvent)) {
                DerivedEventList.add(PreviousEvent);
              }
              // remove close event from PreviousZoneEvents;
              it.remove();
            }
          }

          for (Integer thisZone_Axsmarine_id : CurrentZones) {

            if (PreviousZoneEvents.containsKey(thisZone_Axsmarine_id)) {
              //////////////////////////////////////////////////
              // For current zones which both previous and current locations belong to, update exit
              // point of previous open events with current locations.
              //////////////////////////////////////////////////
              VesselEvent PreviousEvent = PreviousZoneEvents.get(thisZone_Axsmarine_id);
              PreviousEvent.exitcoordinates = VL.coordinates;
              PreviousEvent.exittime = VL.recordtime;
              PreviousEvent.destination = VL.destination;

              if (!DerivedEventList.contains(PreviousEvent)) {
                DerivedEventList.add(PreviousEvent);
              }
            } else {
              //////////////////////////////////////////////////
              // For current zones which only current locations belong to, fire new open events
              //////////////////////////////////////////////////
              VesselEvent NewEvent = new VesselEvent();
              NewEvent.entrycoordinates = VL.coordinates;
              NewEvent.entrytime = VL.recordtime;
              NewEvent.exitcoordinates = VL.coordinates;
              NewEvent.exittime = VL.recordtime;
              NewEvent.destination = VL.destination;
              NewEvent.polygonid = thisZone_Axsmarine_id;

              PreviousZoneEvents.put(thisZone_Axsmarine_id, NewEvent);

              DerivedEventList.add(NewEvent);
            }
          }
        }

        ///////////////////////////////////////////////////////////////////////////////////////
        // Merge with PreviousZoneEvents with AfterZoneEvents

        Iterator<Map.Entry<Integer, VesselEvent>> it = AfterZoneEvents.entrySet().iterator();
        while (it.hasNext()) {
          Map.Entry<Integer, VesselEvent> thisEntry = it.next();
          int Zone_Axsmarine_id = thisEntry.getKey();
          VesselEvent After_VE = thisEntry.getValue();

          VesselEvent Previous_VE = PreviousZoneEvents.get(Zone_Axsmarine_id);

          if (Previous_VE != null) {
            Previous_VE.exitcoordinates = After_VE.exitcoordinates;
            Previous_VE.exittime = After_VE.exittime;
            Previous_VE.destination = After_VE.destination;
            if (!DerivedEventList.contains(Previous_VE)) {
              DerivedEventList.add(Previous_VE);
            }

          } else {
            VesselEvent NewEvent = new VesselEvent();
            NewEvent.entrycoordinates = AfterLocation.coordinates;
            NewEvent.entrytime = AfterLocation.recordtime;
            NewEvent.exitcoordinates = After_VE.exitcoordinates;
            NewEvent.exittime = After_VE.exittime;
            NewEvent.destination = After_VE.destination;
            NewEvent.polygonid = Zone_Axsmarine_id;
            DerivedEventList.add(NewEvent);
          }
          // Delete This Event from HBase
          DeleteEvent(VTEvent, IMO_str, After_VE);
        }

        VTEvent.flush();

        // pupulate Derived Events into Hbase

        for (VesselEvent newEvent : DerivedEventList) {
          // rowkey: IMO(7)+timestamp(19 desc)+polygonid(8)
          // qualifier:entrytime,entrycoordinates,exittime,exitcoordinates,destination

          context.getCounter(Counters.EVENT_UPSERTS).increment(1);

          byte[] rowkey =
              Bytes.toBytes(
                  IMO_str
                      + LpadNum(Long.MAX_VALUE - newEvent.entrytime, 19)
                      + LpadNum(newEvent.polygonid, 10));
          Put put = new Put(rowkey);

          put.addColumn(
              details,
              entrytime,
              Bytes.toBytes(new DateTime(newEvent.entrytime).toString(rawformatter)));
          put.addColumn(details, entrycoordinates, Bytes.toBytes(newEvent.entrycoordinates));
          put.addColumn(
              details,
              exittime,
              Bytes.toBytes(new DateTime(newEvent.exittime).toString(rawformatter)));
          put.addColumn(details, exitcoordinates, Bytes.toBytes(newEvent.exitcoordinates));
          put.addColumn(details, destination, Bytes.toBytes(newEvent.destination));

          VTEvent.mutate(put);
          context.getCounter(Counters.EVENT_VALID).increment(1);
        }

        // VTLocation.flush(); Moved to the first step
        VTEvent.flush();
      } catch (RuntimeException e) {
        // TODO Auto-generated catch block
        System.out.println("Exception occured while loading data for:" + key.getIMO());
        throw e;
      }
    }