コード例 #1
0
 /**
  * make a general method that takes a pair of lat/lon and a radius and give a boolean whether it
  * was in or out.
  *
  * @throws IOException
  */
 @Override
 public Map<String, Integer> getAvailableBikesFromAPoint(
     double lat, double lon, double radius, Get get) throws IOException {
   Result r = ((RegionCoprocessorEnvironment) getEnvironment()).getRegion().get(get, null);
   log.debug("r is " + r);
   log.debug(r.getMap().toString());
   Map<String, Integer> result = new HashMap<String, Integer>();
   try {
     String s = null, latStr = null, lonStr = null;
     for (KeyValue kv : r.raw()) {
       s = Bytes.toString(kv.getValue());
       log.debug("cell value is: " + s);
       String[] sArr = s.split(BIXI_DELIMITER); // array of key=value pairs
       latStr = sArr[3];
       lonStr = sArr[4];
       latStr = latStr.substring(latStr.indexOf("=") + 1);
       lonStr = lonStr.substring(lonStr.indexOf("=") + 1);
       log.debug("lon/lat values are: " + lonStr + "; " + latStr);
       double distance =
           giveDistance(Double.parseDouble(latStr), Double.parseDouble(lonStr), lat, lon) - radius;
       log.debug("distance is : " + distance);
       if (distance < 0) { // add it
         result.put(sArr[0], getFreeBikes(kv));
       }
     }
   } finally {
   }
   return result;
 }
コード例 #2
0
  public static int getDynamicTable(Configuration config) {
    /** Connection to the cluster. A single connection shared by all application threads. */
    Connection connection = null;
    /** A lightweight handle to a specific table. Used from a single thread. */
    Table table = null;

    try {
      connection = ConnectionFactory.createConnection(config);
      table = connection.getTable(TABLE_NAME1);
      Get get = new Get(Bytes.toBytes("cloudera"));
      get.addFamily(CF);
      get.setMaxVersions(Integer.MAX_VALUE);
      Result result = table.get(get);

      NavigableMap<byte[], NavigableMap<byte[], NavigableMap<Long, byte[]>>> map = result.getMap();
      for (Entry<byte[], NavigableMap<byte[], NavigableMap<Long, byte[]>>> columnFamilyEntry :
          map.entrySet()) {
        NavigableMap<byte[], NavigableMap<Long, byte[]>> columnMap = columnFamilyEntry.getValue();
        for (Entry<byte[], NavigableMap<Long, byte[]>> columnEntry : columnMap.entrySet()) {
          NavigableMap<Long, byte[]> cellMap = columnEntry.getValue();
          for (Entry<Long, byte[]> cellEntry : cellMap.entrySet()) {
            System.out.println(
                String.format(
                    "Key : %s, Value :%s",
                    Bytes.toString(columnEntry.getKey()), Bytes.toString(cellEntry.getValue())));
          }
        }
      }
    } catch (IOException e) {
      e.printStackTrace();
    }

    return 0;
  }
コード例 #3
0
 public void map(ImmutableBytesWritable row, Result value, Context context)
     throws InterruptedException, IOException {
   // go through the column family
   for (Map.Entry<byte[], NavigableMap<byte[], NavigableMap<Long, byte[]>>> columnFamilyMap :
       value.getMap().entrySet()) {
     // go through the column
     for (Map.Entry<byte[], NavigableMap<Long, byte[]>> entryVersion :
         columnFamilyMap.getValue().entrySet()) {}
   }
 }
コード例 #4
0
ファイル: TransactionalTable.java プロジェクト: rainyin/omid
  /**
   * Transactional version of {@link HTable#delete(Delete)}
   *
   * @param transactionState Identifier of the transaction
   * @see HTable#delete(Delete)
   * @throws IOException
   */
  public void delete(TransactionState transactionState, Delete delete) throws IOException {
    final long startTimestamp = transactionState.getStartTimestamp();
    boolean issueGet = false;

    final Put deleteP = new Put(delete.getRow(), startTimestamp);
    final Get deleteG = new Get(delete.getRow());
    Map<byte[], List<KeyValue>> fmap = delete.getFamilyMap();
    if (fmap.isEmpty()) {
      issueGet = true;
    }
    for (List<KeyValue> kvl : fmap.values()) {
      for (KeyValue kv : kvl) {
        switch (KeyValue.Type.codeToType(kv.getType())) {
          case DeleteColumn:
            deleteP.add(kv.getFamily(), kv.getQualifier(), startTimestamp, null);
            break;
          case DeleteFamily:
            deleteG.addFamily(kv.getFamily());
            issueGet = true;
            break;
          case Delete:
            if (kv.getTimestamp() == HConstants.LATEST_TIMESTAMP) {
              deleteP.add(kv.getFamily(), kv.getQualifier(), startTimestamp, null);
              break;
            } else {
              throw new UnsupportedOperationException(
                  "Cannot delete specific versions on Snapshot Isolation.");
            }
        }
      }
    }
    if (issueGet) {
      Result result = this.get(deleteG);
      for (Entry<byte[], NavigableMap<byte[], NavigableMap<Long, byte[]>>> entryF :
          result.getMap().entrySet()) {
        byte[] family = entryF.getKey();
        for (Entry<byte[], NavigableMap<Long, byte[]>> entryQ : entryF.getValue().entrySet()) {
          byte[] qualifier = entryQ.getKey();
          deleteP.add(family, qualifier, null);
        }
      }
    }

    transactionState.addRow(
        new RowKeyFamily(delete.getRow(), getTableName(), deleteP.getFamilyMap()));

    put(deleteP);
  }
コード例 #5
0
 /**
  * Pass the key, and reversed value to reduce
  *
  * @param key
  * @param value
  * @param context
  * @throws IOException
  */
 public void map(ImmutableBytesWritable key, Result value, Context context)
     throws IOException, InterruptedException {
   if (value.size() != 1) {
     throw new IOException("There should only be one input column");
   }
   Map<byte[], NavigableMap<byte[], NavigableMap<Long, byte[]>>> cf = value.getMap();
   if (!cf.containsKey(INPUT_FAMILY)) {
     throw new IOException(
         "Wrong input columns. Missing: '" + Bytes.toString(INPUT_FAMILY) + "'.");
   }
   // Get the original value and reverse it
   String originalValue = Bytes.toString(value.getValue(INPUT_FAMILY, null));
   StringBuilder newValue = new StringBuilder(originalValue);
   newValue.reverse();
   // Now set the value to be collected
   Put outval = new Put(key.get());
   outval.add(OUTPUT_FAMILY, null, Bytes.toBytes(newValue.toString()));
   context.write(key, outval);
 }
コード例 #6
0
ファイル: MultiThreadedAction.java プロジェクト: enis/hbase
  /**
   * Verifies the result from get or scan using the dataGenerator (that was presumably also used to
   * generate said result).
   *
   * @param verifyValues verify that values in the result make sense for row/cf/column combination
   * @param verifyCfAndColumnIntegrity verify that cf/column set in the result is complete. Note
   *     that to use this multiPut should be used, or verification has to happen after writes,
   *     otherwise there can be races.
   * @return
   */
  public boolean verifyResultAgainstDataGenerator(
      Result result, boolean verifyValues, boolean verifyCfAndColumnIntegrity) {
    String rowKeyStr = Bytes.toString(result.getRow());
    // See if we have any data at all.
    if (result.isEmpty()) {
      LOG.error("Error checking data for key [" + rowKeyStr + "], no data returned");
      printLocations(result);
      return false;
    }

    if (!verifyValues && !verifyCfAndColumnIntegrity) {
      return true; // as long as we have something, we are good.
    }

    // See if we have all the CFs.
    byte[][] expectedCfs = dataGenerator.getColumnFamilies();
    if (verifyCfAndColumnIntegrity && (expectedCfs.length != result.getMap().size())) {
      LOG.error(
          "Error checking data for key ["
              + rowKeyStr
              + "], bad family count: "
              + result.getMap().size());
      printLocations(result);
      return false;
    }

    // Verify each column family from get in the result.
    for (byte[] cf : result.getMap().keySet()) {
      String cfStr = Bytes.toString(cf);
      Map<byte[], byte[]> columnValues = result.getFamilyMap(cf);
      if (columnValues == null) {
        LOG.error(
            "Error checking data for key [" + rowKeyStr + "], no data for family [" + cfStr + "]]");
        printLocations(result);
        return false;
      }

      Map<String, MutationType> mutateInfo = null;
      if (verifyCfAndColumnIntegrity || verifyValues) {
        if (!columnValues.containsKey(MUTATE_INFO)) {
          LOG.error(
              "Error checking data for key ["
                  + rowKeyStr
                  + "], column family ["
                  + cfStr
                  + "], column ["
                  + Bytes.toString(MUTATE_INFO)
                  + "]; value is not found");
          printLocations(result);
          return false;
        }

        long cfHash = Arrays.hashCode(cf);
        // Verify deleted columns, and make up column counts if deleted
        byte[] mutateInfoValue = columnValues.remove(MUTATE_INFO);
        mutateInfo = parseMutateInfo(mutateInfoValue);
        for (Map.Entry<String, MutationType> mutate : mutateInfo.entrySet()) {
          if (mutate.getValue() == MutationType.DELETE) {
            byte[] column = Bytes.toBytes(mutate.getKey());
            long columnHash = Arrays.hashCode(column);
            long hashCode = cfHash + columnHash;
            if (hashCode % 2 == 0) {
              if (columnValues.containsKey(column)) {
                LOG.error(
                    "Error checking data for key ["
                        + rowKeyStr
                        + "], column family ["
                        + cfStr
                        + "], column ["
                        + mutate.getKey()
                        + "]; should be deleted");
                printLocations(result);
                return false;
              }
              byte[] hashCodeBytes = Bytes.toBytes(hashCode);
              columnValues.put(column, hashCodeBytes);
            }
          }
        }

        // Verify increment
        if (!columnValues.containsKey(INCREMENT)) {
          LOG.error(
              "Error checking data for key ["
                  + rowKeyStr
                  + "], column family ["
                  + cfStr
                  + "], column ["
                  + Bytes.toString(INCREMENT)
                  + "]; value is not found");
          printLocations(result);
          return false;
        }
        long currentValue = Bytes.toLong(columnValues.remove(INCREMENT));
        if (verifyValues) {
          long amount = mutateInfo.isEmpty() ? 0 : cfHash;
          long originalValue = Arrays.hashCode(result.getRow());
          long extra = currentValue - originalValue;
          if (extra != 0 && (amount == 0 || extra % amount != 0)) {
            LOG.error(
                "Error checking data for key ["
                    + rowKeyStr
                    + "], column family ["
                    + cfStr
                    + "], column [increment], extra ["
                    + extra
                    + "], amount ["
                    + amount
                    + "]");
            printLocations(result);
            return false;
          }
          if (amount != 0 && extra != amount) {
            LOG.warn(
                "Warning checking data for key ["
                    + rowKeyStr
                    + "], column family ["
                    + cfStr
                    + "], column [increment], incremented ["
                    + (extra / amount)
                    + "] times");
          }
        }

        // See if we have correct columns.
        if (verifyCfAndColumnIntegrity
            && !dataGenerator.verify(result.getRow(), cf, columnValues.keySet())) {
          String colsStr = "";
          for (byte[] col : columnValues.keySet()) {
            if (colsStr.length() > 0) {
              colsStr += ", ";
            }
            colsStr += "[" + Bytes.toString(col) + "]";
          }
          LOG.error(
              "Error checking data for key ["
                  + rowKeyStr
                  + "], bad columns for family ["
                  + cfStr
                  + "]: "
                  + colsStr);
          printLocations(result);
          return false;
        }
        // See if values check out.
        if (verifyValues) {
          for (Map.Entry<byte[], byte[]> kv : columnValues.entrySet()) {
            String column = Bytes.toString(kv.getKey());
            MutationType mutation = mutateInfo.get(column);
            boolean verificationNeeded = true;
            byte[] bytes = kv.getValue();
            if (mutation != null) {
              boolean mutationVerified = true;
              long columnHash = Arrays.hashCode(kv.getKey());
              long hashCode = cfHash + columnHash;
              byte[] hashCodeBytes = Bytes.toBytes(hashCode);
              if (mutation == MutationType.APPEND) {
                int offset = bytes.length - hashCodeBytes.length;
                mutationVerified =
                    offset > 0
                        && Bytes.equals(
                            hashCodeBytes,
                            0,
                            hashCodeBytes.length,
                            bytes,
                            offset,
                            hashCodeBytes.length);
                if (mutationVerified) {
                  int n = 1;
                  while (true) {
                    int newOffset = offset - hashCodeBytes.length;
                    if (newOffset < 0
                        || !Bytes.equals(
                            hashCodeBytes,
                            0,
                            hashCodeBytes.length,
                            bytes,
                            newOffset,
                            hashCodeBytes.length)) {
                      break;
                    }
                    offset = newOffset;
                    n++;
                  }
                  if (n > 1) {
                    LOG.warn(
                        "Warning checking data for key ["
                            + rowKeyStr
                            + "], column family ["
                            + cfStr
                            + "], column ["
                            + column
                            + "], appended ["
                            + n
                            + "] times");
                  }
                  byte[] dest = new byte[offset];
                  System.arraycopy(bytes, 0, dest, 0, offset);
                  bytes = dest;
                }
              } else if (hashCode % 2 == 0) { // checkAndPut
                mutationVerified = Bytes.equals(bytes, hashCodeBytes);
                verificationNeeded = false;
              }
              if (!mutationVerified) {
                LOG.error(
                    "Error checking data for key ["
                        + rowKeyStr
                        + "], mutation checking failed for column family ["
                        + cfStr
                        + "], column ["
                        + column
                        + "]; mutation ["
                        + mutation
                        + "], hashCode ["
                        + hashCode
                        + "], verificationNeeded ["
                        + verificationNeeded
                        + "]");
                printLocations(result);
                return false;
              }
            } // end of mutation checking
            if (verificationNeeded
                && !dataGenerator.verify(result.getRow(), cf, kv.getKey(), bytes)) {
              LOG.error(
                  "Error checking data for key ["
                      + rowKeyStr
                      + "], column family ["
                      + cfStr
                      + "], column ["
                      + column
                      + "], mutation ["
                      + mutation
                      + "]; value of length "
                      + bytes.length);
              printLocations(result);
              return false;
            }
          }
        }
      }
    }
    return true;
  }
コード例 #7
0
ファイル: EnNiuHBaseProxy.java プロジェクト: joezxh/DATAX-UI
 public boolean fetchLine(Line line) throws IOException {
   if (null == this.rs) {
     throw new IllegalStateException("HBase Client try to fetch data failed .");
   }
   Result result = this.rs.next();
   if (null == result) {
     return false;
   }
   if (this.maxversion == -1) { // 多版本记录,按照每个字段的设置获取不同版本数据
     NavigableMap<byte[], NavigableMap<byte[], NavigableMap<Long, byte[]>>> nMap = result.getMap();
     this.p = new Put(result.getRow());
     this.p.setDurability(Durability.SKIP_WAL);
     for (int i = 0; i < this.families.length; i++) {
       NavigableMap<Long, byte[]> vmap =
           nMap.get(this.families[i].getBytes()).get(this.columns[i].getBytes());
       byte[] value = null;
       byte[] firstValue = null;
       if (vmap == null || vmap.size() == 0) { // 无记录,判断后获取替换的字段值
         line.addField(null);
         continue;
       } else if (vmap.size() > 1) { // 多个版本
         if ("1".equalsIgnoreCase(this.column_version[i])) {
           Iterator<Map.Entry<Long, byte[]>> iter = vmap.entrySet().iterator();
           int id = 0;
           while (iter.hasNext()) {
             if (id == 0) {
               firstValue = iter.next().getValue();
               value = firstValue;
             } else {
               value = iter.next().getValue();
             }
             id++;
           }
           if (id > 0) {
             this.p.addColumn(families[i].getBytes(), this.columns[i].getBytes(), value);
           }
         } else { // 取第一个最新版本的值
           value = vmap.entrySet().iterator().next().getValue();
         }
       } else { // 单个版本
         value = vmap.entrySet().iterator().next().getValue();
       }
       if (null == value) {
         line.addField(null);
       } else {
         line.addField(new String(value, encode));
       }
     }
     // 判断是否将hbase值替换和修改
     if (ETLStringUtils.isNotEmpty(this.columnProcRule) && this.partRuleId == 1) { // 需要替换字段
       if (ETLStringUtils.isEmpty(line.getField(this.partColumnIdx))) { // 需要替换的字段为空,则替换字段
         line.addField(line.getField(this.bakPartColumnIdx), this.partColumnIdx);
         this.p.addColumn(
             families[partColumnIdx].getBytes(),
             this.columns[partColumnIdx].getBytes(),
             line.getField(this.bakPartColumnIdx).getBytes());
       }
     }
     if (this.p.size() > 0) {
       buffer.add(this.p);
     }
     if (buffer.size() >= BUFFER_LINE) {
       htable.put(buffer);
       htable.flushCommits();
       buffer.clear();
     }
   } else {
     for (int i = 0; i < this.families.length; i++) {
       byte[] value = result.getValue(this.families[i].getBytes(), this.columns[i].getBytes());
       if (null == value) {
         line.addField(null);
       } else {
         line.addField(new String(value, encode));
       }
     }
   }
   //
   line.addField(new String(result.getRow(), encode));
   return true;
 }
コード例 #8
0
  public List<Point2D.Double> debugColumnVersion(
      String timestamp, double latitude, double longitude, double radius) {

    this.getStatLog(this.STAT_FILE_NAME);
    long sTime = System.currentTimeMillis();
    // build up a raster
    XRaster raster = new XRaster(this.space, this.min_size_of_height, this.max_num_of_column);
    Point2D.Double point = new Point2D.Double(latitude, longitude);
    ResultScanner rScanner = null;
    // return result
    HashMap<String, String> results = new HashMap<String, String>();
    ArrayList<Point2D.Double> returnPoints = new ArrayList<Point2D.Double>();
    try {
      // match rect to find the subspace it belongs to
      XBox[] match_boxes = raster.match(latitude, longitude, radius);
      String[] rowRange = new String[2];
      rowRange[0] = match_boxes[0].getRow();
      rowRange[1] = match_boxes[1].getRow() + "0";

      String[] c = raster.getColumns(match_boxes[0], match_boxes[1]);
      // the version here is harded coded, because i cannot get how many
      // objects in one cell now
      FilterList fList = new FilterList();
      fList.addFilter(this.hbaseUtil.getInclusiveFilter(rowRange[1]));
      rScanner = this.hbaseUtil.getResultSet(rowRange, fList, this.familyName, c, 1000000);

      BixiReader reader = new BixiReader();
      int count = 0;
      int accepted = 0;

      int max_column = 0;
      int min_column = 10000;
      int max_version = 0;
      int min_version = 10000;
      int row_count = 0;
      int byte_lenght = 0;

      for (Result r : rScanner) {
        byte_lenght = r.getBytes().getLength();
        row_count++;

        NavigableMap<byte[], NavigableMap<byte[], NavigableMap<Long, byte[]>>> resultMap =
            r.getMap();

        int count_column = 0;
        for (byte[] family : resultMap.keySet()) {
          NavigableMap<byte[], NavigableMap<Long, byte[]>> columns = resultMap.get(family);
          count_column = 0;
          for (byte[] col : columns.keySet()) {
            NavigableMap<Long, byte[]> values = columns.get(col);
            count_column++;
            if (values.values().size() > max_version) {
              max_version = values.values().size();
            }
            if (values.values().size() < min_version) {
              min_version = values.values().size();
            }

            for (Long version : values.keySet()) {
              count++;
              // get the distance between this point and the given
              // point
              XStation station = reader.getStationFromJson(Bytes.toString(values.get(version)));

              Point2D.Double resPoint =
                  new Point2D.Double(station.getLatitude(), station.getlongitude());
              double distance = resPoint.distance(point);

              if (Bytes.toString(col).equals("0011")) {
                /*								System.out.println("!!!! key=>"+Bytes.toString(r.getRow())+
                ";column=>"+Bytes.toString(col)+
                ";version=>"+version+
                ";point=>"+resPoint.toString());*/
              }
              if (distance <= radius) {
                returnPoints.add(resPoint);
                // System.out.println("row=>"+Bytes.toString(r.getRow())
                // +
                // ";colum=>"+Bytes.toString(col)+";version=>"+version+
                // ";station=>"+station.getId()+";distance=>"+distance);
                accepted++;
                results.put(station.getId(), String.valueOf(distance));
              }
            }
          }
          if (count_column > max_column) max_column = count_column;
          if (count_column < min_column) min_column = count_column;
        }
      }
      System.out.println("byte_length=>" + byte_lenght + ";row_count=>" + row_count);
      System.out.println(
          "max_column=>"
              + max_column
              + ";min_column=>"
              + min_column
              + ";max_version=>"
              + max_version
              + ";min_version=>"
              + min_version);
      long eTime = System.currentTimeMillis();
      System.out.println(
          "count=>" + count + ";accepted=>" + accepted + ";time=>" + (eTime - sTime));
      String outStr =
          "radius=>"
              + radius
              + ";count=>"
              + count
              + ";accepted=>"
              + accepted
              + ";time=>"
              + (eTime - sTime)
              + ";row_stride=>"
              + this.min_size_of_height
              + ";columns=>"
              + this.max_num_of_column;
      this.writeStat(outStr);

    } catch (Exception e) {
      e.printStackTrace();
    } finally {
      this.hbaseUtil.closeTableHandler();
      this.closeStatLog();
    }
    return returnPoints;
  }
コード例 #9
0
  @Override
  public TreeMap<Double, String> scanQueryAvailableKNN(
      String timestamp, double latitude, double longitude, int k) {
    this.getStatLog(STAT_FILE_NAME);
    long sTime = System.currentTimeMillis();
    // Step1: estimate the window circle for the first time
    int total_points =
        Integer.valueOf(this.conf.getProperty("total_num_of_points")); // 1000000000;//
    double areaOfMBB = this.space.width * this.space.height;
    double DensityOfMBB = total_points / areaOfMBB;
    double init_radius = Math.sqrt(k / DensityOfMBB);
    XRaster raster = new XRaster(this.space, this.min_size_of_height, this.max_num_of_column);

    longitude = Math.abs(longitude);
    int count = 0;
    int accepted = 0;
    ResultScanner rScanner = null;
    List<String> resultsList = new ArrayList<String>();
    BixiReader reader = new BixiReader();
    TreeMap<Double, String> sorted = null;
    try {
      // Step2: trigger a scan to get the points based on the above window
      int iteration = 1;

      long match_s = System.currentTimeMillis();
      double radius =
          (init_radius > this.min_size_of_height) ? init_radius : this.min_size_of_height;

      do {
        String str = "iteration" + iteration + "; count=>" + count + ";radius=>" + radius;
        this.writeStat(str);
        // match rect to find the subspace it belongs to
        XBox[] match_boxes = raster.match(latitude, longitude, radius);

        String[] rowRange = new String[2];
        rowRange[0] = match_boxes[0].getRow();
        rowRange[1] = match_boxes[1].getRow() + "-*";

        String[] c = raster.getColumns(match_boxes[0], match_boxes[1]);
        rScanner = this.hbaseUtil.getResultSet(rowRange, null, this.familyName, c, 1000000);

        count = 0;
        resultsList.clear();
        for (Result r : rScanner) {
          NavigableMap<byte[], NavigableMap<byte[], NavigableMap<Long, byte[]>>> resultMap =
              r.getMap();

          for (byte[] family : resultMap.keySet()) {
            NavigableMap<byte[], NavigableMap<Long, byte[]>> columns = resultMap.get(family);
            for (byte[] col : columns.keySet()) {
              NavigableMap<Long, byte[]> values = columns.get(col);
              for (Long version : values.keySet()) {
                resultsList.add(Bytes.toString(values.get(version)));
                count++;
              }
            }
          }
        }
        radius = init_radius * (iteration + 1);

        // Step3: get the result,estimate the window circle next depending on the previous step
        // result, util we got the K nodes
        /*				if(count == 0 && iteration ==1){ // when the first time count == 0
        	radius = radius * 2;
        }else if(count > 0 && iteration >0){ // when the first time count >0 && count < k
        	areaOfMBB = radius * radius;
        	DensityOfMBB = count / areaOfMBB;
        	radius = Math.sqrt(k / DensityOfMBB);
        }*/

      } while (count < k && (++iteration > 0));
      String str = "iteration" + iteration + "; count=>" + count + ";radius=>" + radius;
      this.writeStat(str);

      long match_time = System.currentTimeMillis() - match_s;

      // Step4: get all possible points and sort them by the distance and get the top K
      Point2D.Double point = new Point2D.Double(latitude, longitude);
      // result container
      HashMap<Double, String> distanceMap = new HashMap<Double, String>();

      for (String value : resultsList) {

        XStation station = reader.getStationFromJson(value);

        Point2D.Double resPoint = new Point2D.Double(station.getLatitude(), station.getlongitude());
        double distance = resPoint.distance(point);

        distanceMap.put(distance, station.getId());
      }

      sorted = new TreeMap<Double, String>(distanceMap);

      long eTime = System.currentTimeMillis();

      String outStr =
          "q=knn;m=>scan;"
              + ";count=>"
              + count
              + ";accepted=>"
              + accepted
              + ";time=>"
              + (eTime - sTime)
              + ";k=>"
              + k;
      this.writeStat(outStr);

    } catch (Exception e) {
      e.printStackTrace();
    } finally {
      this.hbaseUtil.closeTableHandler();
      this.closeStatLog();
    }
    return sorted;
  }
コード例 #10
0
  @Override
  public String scanQueryPoint(double latitude, double longitude) {
    this.getStatLog(STAT_FILE_NAME);

    long sTime = System.currentTimeMillis();
    // build up a raster
    XRaster raster = new XRaster(this.space, this.min_size_of_height, this.max_num_of_column);
    ResultScanner rScanner = null;

    try {
      // match rect to find the subspace it belongs to
      XBox match_box = raster.locate(latitude, Math.abs(longitude));
      System.out.println("match_box is : " + match_box.toString());
      String[] rowRange = new String[2];
      rowRange[0] = match_box.getRow();
      rowRange[1] = match_box.getRow() + "-*";

      // the version here is harded coded, because i cannot get how many
      // objects in one cell now
      rScanner =
          this.hbaseUtil.getResultSet(
              rowRange, null, this.familyName, new String[] {match_box.getColumn()}, 1000000);
      BixiReader reader = new BixiReader();
      int count = 0;
      int row = 0;
      String stationName = null;
      for (Result r : rScanner) {
        NavigableMap<byte[], NavigableMap<byte[], NavigableMap<Long, byte[]>>> resultMap =
            r.getMap();
        row++;
        for (byte[] family : resultMap.keySet()) {
          NavigableMap<byte[], NavigableMap<Long, byte[]>> columns = resultMap.get(family);
          for (byte[] col : columns.keySet()) {
            NavigableMap<Long, byte[]> values = columns.get(col);
            for (Long version : values.keySet()) {
              count++;
              // get the distance between this point and the given
              // point
              XStation station = reader.getStationFromJson(Bytes.toString(values.get(version)));

              if ((station.getLatitude() == latitude && station.getlongitude() == longitude)) {
                stationName = station.getId();
                break;
              }
            }
            if (stationName != null) break;
          }

          if (stationName != null) break;
        }
        if (stationName != null) break;
      }
      long eTime = System.currentTimeMillis();
      System.out.println(
          "count=>" + count + ";time=>" + (eTime - sTime) + ";result=>" + stationName);
      String outStr =
          "q=point;m=scan;count=>"
              + count
              + ";time=>"
              + (eTime - sTime)
              + ";row=>"
              + row
              + ";result=>"
              + stationName;
      this.writeStat(outStr);

      return stationName;

    } catch (Exception e) {
      e.printStackTrace();
    } finally {
      this.hbaseUtil.closeTableHandler();
      this.closeStatLog();
    }
    return null;
  }
コード例 #11
0
  @Override
  public HashMap<String, String> scanQueryAvailableNear(
      String timestamp, double latitude, double longitude, double radius) {

    this.getStatLog(STAT_FILE_NAME);
    this.getCSVLog(FILE_NAME_PREFIX, 0);
    this.getCSVLog(FILE_NAME_PREFIX, 2);
    this.timePhase.clear();
    long sTime = System.currentTimeMillis();
    this.timePhase.add(sTime);
    // build up a raster
    XRaster raster = new XRaster(this.space, this.min_size_of_height, this.max_num_of_column);
    Point2D.Double point = new Point2D.Double(latitude, longitude);
    ResultScanner rScanner = null;
    // return result
    HashMap<String, String> results = new HashMap<String, String>();
    try {
      // match rect to find the subspace it belongs to
      long match_s = System.currentTimeMillis();
      XBox[] match_boxes = raster.match(latitude, longitude, radius);
      long match_time = System.currentTimeMillis() - match_s;
      String[] rowRange = new String[2];
      rowRange[0] = match_boxes[0].getRow();
      rowRange[1] = match_boxes[1].getRow() + "-*";

      String[] c = raster.getColumns(match_boxes[0], match_boxes[1]);
      /*
       * System.out.println(rowRange[0]+":"+rowRange[1]); for(int
       * i=0;i<c.length;i++){ System.out.print(c[i]+";"); }
       * System.out.println();
       */
      // the version here is harded coded, because i cannot get how many
      // objects in one cell now
      this.timePhase.add(System.currentTimeMillis());
      rScanner = this.hbaseUtil.getResultSet(rowRange, null, this.familyName, c, 1000000);
      BixiReader reader = new BixiReader();
      int count = 0;
      int row = 0;
      int accepted = 0;

      for (Result r : rScanner) {

        NavigableMap<byte[], NavigableMap<byte[], NavigableMap<Long, byte[]>>> resultMap =
            r.getMap();

        for (byte[] family : resultMap.keySet()) {
          NavigableMap<byte[], NavigableMap<Long, byte[]>> columns = resultMap.get(family);
          for (byte[] col : columns.keySet()) {
            NavigableMap<Long, byte[]> values = columns.get(col);
            for (Long version : values.keySet()) {
              count++;
              // get the distance between this point and the given
              // point
              XStation station = reader.getStationFromJson(Bytes.toString(values.get(version)));

              Point2D.Double resPoint =
                  new Point2D.Double(station.getLatitude(), station.getlongitude());
              double distance = resPoint.distance(point);

              if (distance <= radius) {
                // System.out.println("row=>"+Bytes.toString(r.getRow())
                // +
                // ";colum=>"+Bytes.toString(col)+";version=>"+version+
                // ";station=>"+station.getId()+";distance=>"+distance);
                accepted++;
                results.put(station.getId(), String.valueOf(distance));
              }
            }
          }
        }
      }
      long eTime = System.currentTimeMillis();
      this.timePhase.add(eTime);
      String outStr =
          "m=scan;"
              + "radius=>"
              + radius
              + ";count=>"
              + count
              + ";accepted=>"
              + accepted
              + ";time=>"
              + (eTime - sTime)
              + ";row=>"
              + row
              + "row_stride=>"
              + this.min_size_of_height
              + ";columns=>"
              + this.max_num_of_column;
      ;
      this.writeStat(outStr);

      // write to csv file
      outStr = "";
      outStr +=
          "within,"
              + "scan,"
              + count
              + ","
              + accepted
              + ","
              + (eTime - sTime)
              + ","
              + row
              + ","
              + match_time
              + ","
              + this.min_size_of_height
              + ","
              + radius;

      this.writeCSVLog(outStr, 0);

      String timeStr = "within,scan," + radius + ",";
      for (int i = 0; i < this.timePhase.size(); i++) {
        timeStr += this.timePhase.get(i) + ",";
      }
      this.writeCSVLog(timeStr, 2);

    } catch (Exception e) {
      e.printStackTrace();
    } finally {
      this.hbaseUtil.closeTableHandler();
      this.closeStatLog();
      this.closeCSVLog();
    }
    return results;
  }