public List<String> getStationsNearPoint_Schema2(double lat, double lon) throws IOException {
   Scan scan = new Scan();
   scan.addFamily(BixiConstant.SCHEMA2_CLUSTER_FAMILY_NAME.getBytes());
   InternalScanner scanner =
       ((RegionCoprocessorEnvironment) getEnvironment()).getRegion().getScanner(scan);
   boolean hasMoreResult = false;
   List<KeyValue> res = new ArrayList<KeyValue>();
   List<String> result = new ArrayList<String>();
   try {
     do {
       hasMoreResult = scanner.next(res);
       for (KeyValue kv : res) {
         String clusterId = Bytes.toString(kv.getRow());
         String[] parts = clusterId.split(":");
         double cLat = Double.parseDouble(parts[0]);
         double cLon = Double.parseDouble(parts[1]);
         double dx = Double.parseDouble(parts[2]);
         double dy = Double.parseDouble(parts[3]);
         double distx = lat - cLat;
         double disty = lon - cLon;
         if (distx >= 0 && distx <= dx && disty >= 0 && disty <= dy) {
           // get stations in cluster
           result.add(Bytes.toString(kv.getQualifier()));
         }
       }
       res.clear();
     } while (hasMoreResult);
   } finally {
     scanner.close();
   }
   return result;
 }
  @Override
  public Map<String, Integer> giveAvailableBikes(
      long milliseconds, List<String> stationIds, Scan scan) throws IOException {
    // scan has set the time stamp accordingly, i.e., the start and end row of
    // the scan.

    for (String qualifier : stationIds) {
      log.debug("adding qualifier: " + qualifier);
      scan.addColumn(colFamily, qualifier.getBytes());
    }
    InternalScanner scanner =
        ((RegionCoprocessorEnvironment) getEnvironment()).getRegion().getScanner(scan);
    List<KeyValue> res = new ArrayList<KeyValue>();
    Map<String, Integer> result = new HashMap<String, Integer>();
    boolean hasMoreResult = false;
    try {
      do {
        hasMoreResult = scanner.next(res);
        for (KeyValue kv : res) {
          // log.debug("got a kv: " + kv);
          int availBikes = getFreeBikes(kv);
          String id = Bytes.toString(kv.getQualifier());
          // log.debug("result to be added is: " + availBikes + " id: " + id);
          result.put(id, availBikes);
        }
        res.clear();
      } while (hasMoreResult);
    } finally {
      scanner.close();
    }
    return result;
  }
  @Override
  public Map<String, TotalNum> getTotalUsage_Schema2(Scan scan) throws IOException {

    // System.err.println("scanning");
    scan.addFamily(colFamilyStat);

    InternalScanner scanner =
        ((RegionCoprocessorEnvironment) getEnvironment()).getRegion().getScanner(scan);
    List<KeyValue> res = new ArrayList<KeyValue>();
    Map<String, TotalNum> result = new HashMap<String, TotalNum>();
    boolean hasMoreResult = false;
    try {
      do {
        hasMoreResult = scanner.next(res);
        for (KeyValue kv : res) {
          String stationId = Bytes.toString(kv.getRow()).split("-")[1];
          String value = new String(kv.getValue());
          Long usage = Long.parseLong(value.split(";")[1]);
          if (result.containsKey(stationId)) {
            TotalNum tn = result.get(stationId);
            tn.add(usage);
            result.put(stationId, tn);
          } else {
            TotalNum tn = new TotalNum();
            tn.add(usage);
            result.put(stationId, tn);
          }
        }
        res.clear();
      } while (hasMoreResult);
    } finally {
      scanner.close();
    }
    return result;
  }
 @Override
 public Map<String, Integer> getAvailableBikesFromAPoint_Schema2(Scan scan) throws IOException {
   scan.addFamily(colFamilyStat);
   InternalScanner scanner =
       ((RegionCoprocessorEnvironment) getEnvironment()).getRegion().getScanner(scan);
   Map<String, Integer> result = new HashMap<String, Integer>();
   boolean hasMoreResult = false;
   List<KeyValue> res = new ArrayList<KeyValue>();
   try {
     do {
       hasMoreResult = scanner.next(res);
       for (KeyValue kv : res) {
         String stationId = Bytes.toString(kv.getRow()).split("-")[1];
         String value = new String(kv.getValue());
         Integer free = Integer.parseInt(value.split(";")[0]);
         result.put(stationId, free);
         /*if(result.containsKey(stationId)){
         	result.put(stationId, free + result.get(stationId));
         }else{
         	result.put(stationId, free);
         }*/
       }
       res.clear();
     } while (hasMoreResult);
   } finally {
     scanner.close();
   }
   return result;
 }
 public static void doScan(HRegion region, Scan scan, List<Cell> result) throws IOException {
   InternalScanner scanner = null;
   try {
     scan.setIsolationLevel(IsolationLevel.READ_UNCOMMITTED);
     scanner = region.getScanner(scan);
     result.clear();
     scanner.next(result);
   } finally {
     if (scanner != null) scanner.close();
   }
 }
  public Result get(final long transactionId, final Get get) throws IOException {
    Scan scan = new Scan(get);
    List<KeyValue> results = new ArrayList<KeyValue>();

    InternalScanner scanner = null;
    try {
      scanner = getScanner(transactionId, scan);
      scanner.next(results);
    } finally {
      if (scanner != null) {
        scanner.close();
      }
    }
    return new Result(results);
  }
Exemple #7
0
    OfflineMerger(Configuration conf, FileSystem fs) throws IOException {
      super(conf, fs, HConstants.META_TABLE_NAME);

      Path rootTableDir =
          HTableDescriptor.getTableDir(
              fs.makeQualified(new Path(conf.get(HConstants.HBASE_DIR))),
              HConstants.ROOT_TABLE_NAME);

      // Scan root region to find all the meta regions

      root =
          HRegion.newHRegion(
              rootTableDir,
              hlog,
              fs,
              conf,
              HRegionInfo.ROOT_REGIONINFO,
              HTableDescriptor.ROOT_TABLEDESC,
              null);
      root.initialize();

      Scan scan = new Scan();
      scan.addColumn(HConstants.CATALOG_FAMILY, HConstants.REGIONINFO_QUALIFIER);
      InternalScanner rootScanner = root.getScanner(scan);

      try {
        List<KeyValue> results = new ArrayList<KeyValue>();
        boolean hasMore;
        do {
          hasMore = rootScanner.next(results);
          for (KeyValue kv : results) {
            HRegionInfo info = Writables.getHRegionInfoOrNull(kv.getValue());
            if (info != null) {
              metaRegions.add(info);
            }
          }
        } while (hasMore);
      } finally {
        rootScanner.close();
        try {
          root.close();

        } catch (IOException e) {
          LOG.error(e);
        }
      }
    }
 @Override
 public Map<String, TotalNum> giveTotalUsage(List<String> stationIds, Scan scan)
     throws IOException {
   for (String qualifier : stationIds) {
     log.debug("adding qualifier: " + qualifier);
     String colName = Integer.toString(Integer.parseInt(qualifier));
     scan.addColumn(colFamily, colName.getBytes());
   }
   InternalScanner scanner =
       ((RegionCoprocessorEnvironment) getEnvironment()).getRegion().getScanner(scan);
   List<KeyValue> res = new ArrayList<KeyValue>();
   Map<String, TotalNum> result = new HashMap<String, TotalNum>();
   boolean hasMoreResult = false;
   try {
     do {
       hasMoreResult = scanner.next(res);
       for (KeyValue kv : res) {
         // log.debug("got a kv: " + kv);
         long emptyDocks = getEmptyDocks(kv);
         String id = Bytes.toString(kv.getQualifier());
         TotalNum tn;
         if (result.containsKey(id)) {
           tn = result.get(id);
         } else {
           tn = new TotalNum();
         }
         tn.add(emptyDocks);
         // emptyDocks = emptyDocks + (prevVal != null ? prevVal.intValue() : 0);
         // log.debug("result to be added is: " + emptyDocks + " id: " + id);
         result.put(id, tn);
       }
       res.clear();
     } while (hasMoreResult);
   } finally {
     scanner.close();
   }
   return result;
 }
  /**
   * Asynchronous HBase scan read as RAW qualifier
   *
   * @param scan
   * @param listener
   * @throws Exception
   */
  protected InternalReadReport asyncStreamRead(
      EntityDefinition ed, Scan scan, QualifierCreationListener listener) throws IOException {
    //		_init();
    long counter = 0;
    long startTimestamp = 0;
    long stopTimestamp = 0;
    InternalScanner scanner = this.getCurrentRegion().getScanner(scan);
    List<Cell> results = new ArrayList<Cell>();
    try {
      boolean hasMoreRows; // false by default
      do {
        hasMoreRows = scanner.next(results);
        Map<String, byte[]> kvMap = new HashMap<String, byte[]>();
        if (!results.isEmpty()) {
          counter++;
          byte[] row = results.get(0).getRow();
          //					if(ed.isTimeSeries()){
          long timestamp = RowkeyBuilder.getTimestamp(row, ed);
          // Min
          if (startTimestamp == 0 || startTimestamp > timestamp) {
            startTimestamp = timestamp;
          }
          // Max
          if (stopTimestamp == 0 || stopTimestamp < timestamp) {
            stopTimestamp = timestamp;
          }
          //					}

          for (Cell kv : results) {
            String qualifierName = Bytes.toString(kv.getQualifier());
            Qualifier qualifier = null;
            if (!ed.isTag(qualifierName)) {
              qualifier = ed.getQualifierNameMap().get(qualifierName);
              if (qualifier == null) {
                LOG.error("qualifier for field " + qualifierName + " not exist");
                throw new IOException(
                    new NullPointerException("qualifier for field " + qualifierName + " is null"));
              }
              qualifierName = qualifier.getDisplayName();
            }
            if (kv.getValue() != null) kvMap.put(qualifierName, kv.getValue());
          }

          //					LOG.info("DEBUG: timestamp="+timestamp+",
          // keys=["+StringUtils.join(kvMap.keySet(),",")+"]");

          if (!kvMap.isEmpty()) listener.qualifierCreated(kvMap);
          results.clear();
        } else {
          if (LOG.isDebugEnabled()) LOG.warn("Empty batch of KeyValue");
        }
      } while (hasMoreRows);
    } catch (IOException ex) {
      LOG.error(ex.getMessage(), ex);
      throw ex;
    } finally {
      if (scanner != null) {
        scanner.close();
      }
    }

    return new InternalReadReport(counter, startTimestamp, stopTimestamp);
  }
  /**
   * Asynchronous HBase scan read as entity
   *
   * @param scan
   * @throws java.io.IOException
   */
  protected InternalReadReport asyncStreamRead(
      EntityDefinition ed, Scan scan, EntityCreationListener listener) throws IOException {
    //		_init();
    long counter = 0;
    long startTimestamp = 0;
    long stopTimestamp = 0;

    InternalScanner scanner = this.getCurrentRegion().getScanner(scan);
    List<Cell> results = new ArrayList<Cell>();
    try {
      boolean hasMoreRows;
      GenericMetricShadowEntity singleMetricEntity = null;
      do {
        hasMoreRows = scanner.next(results);
        Map<String, byte[]> kvMap = new HashMap<String, byte[]>();
        if (!results.isEmpty()) {
          counter++;
          byte[] row = results.get(0).getRow();
          long timestamp = RowkeyBuilder.getTimestamp(row, ed);

          // Min
          if (startTimestamp == 0 || startTimestamp > timestamp) {
            startTimestamp = timestamp;
          }

          // Max
          if (stopTimestamp == 0 || stopTimestamp < timestamp) {
            stopTimestamp = timestamp;
          }

          for (Cell kv : results) {
            String qualifierName = Bytes.toString(kv.getQualifier());
            //						Qualifier qualifier = null;
            //						if(!ed.isTag(qualifierName)){
            //							qualifier = ed.getQualifierNameMap().get(qualifierName);
            //							if(qualifier == null){
            //								LOG.error("qualifier for   " + qualifierName + " not exist");
            //								throw new NullPointerException("qualifier for field "+qualifierName+" not
            // exist");
            //							}
            //						}
            if (kv.getValue() != null) kvMap.put(qualifierName, kv.getValue());
          }

          // LOG.info("DEBUG: timestamp="+timestamp+",
          // keys=["+StringUtils.join(kvMap.keySet(),",")+"]");

          InternalLog internalLog = HBaseInternalLogHelper.buildObject(ed, row, timestamp, kvMap);
          if (internalLog != null) {
            TaggedLogAPIEntity logAPIEntity = null;
            try {
              logAPIEntity = HBaseInternalLogHelper.buildEntity(internalLog, ed);
              if (logAPIEntity instanceof GenericMetricEntity) {
                if (singleMetricEntity == null)
                  singleMetricEntity = new GenericMetricShadowEntity();
                GenericMetricEntity e = (GenericMetricEntity) logAPIEntity;
                if (e.getValue() != null) {
                  int count = e.getValue().length;
                  @SuppressWarnings("unused")
                  Class<?> cls = ed.getMetricDefinition().getSingleTimestampEntityClass();
                  for (int i = 0; i < count; i++) {
                    long ts =
                        logAPIEntity.getTimestamp() + i * ed.getMetricDefinition().getInterval();
                    // exclude those entity which is not within the time range in search condition.
                    // [start, end)
                    singleMetricEntity.setTimestamp(ts);
                    singleMetricEntity.setTags(e.getTags());
                    singleMetricEntity.setValue(e.getValue()[i]);
                    // Min
                    if (startTimestamp == 0 || startTimestamp > ts) startTimestamp = ts;
                    // Max
                    if (stopTimestamp == 0 || stopTimestamp < ts) stopTimestamp = ts;
                    listener.entityCreated(singleMetricEntity);
                  }
                }
              } else {
                // LOG.info("DEBUG: rowKey="+logAPIEntity.getEncodedRowkey());
                listener.entityCreated(logAPIEntity);
              }
            } catch (Exception e) {
              if (internalLog != null) {
                LOG.error(
                    "Got exception to handle " + internalLog.toString() + ": " + e.getMessage(), e);
              }
              throw new IOException(e);
            }
          } else {
            LOG.error(
                "Got null to parse internal log for row: " + row.length + " with fields: " + kvMap);
          }
          results.clear();
        } else {
          if (LOG.isDebugEnabled()) LOG.warn("Empty batch of KeyValue");
        }
      } while (hasMoreRows);
    } catch (IOException ex) {
      LOG.error(ex.getMessage(), ex);
      throw ex;
    } finally {
      if (scanner != null) {
        scanner.close();
      }
    }
    return new InternalReadReport(counter, startTimestamp, stopTimestamp);
  }
  @SuppressWarnings("unchecked")
  public <T extends Number, S extends Number> Map<BytesWritable, GroupByStatsValues<T, S>> getStats(
      Scan scan, List<byte[][]> groupByTuples, byte[][] statsTuple, ColumnInterpreter<T, S> statsCI)
      throws IOException {
    long bt = System.currentTimeMillis();
    long et;
    long nt = 0, ntemp;
    long octime = 0;
    long nrecords = 0;
    double ntmillis, ocmillis;
    log.debug("getStats: start.");

    byte[] sfamily = statsTuple[0];
    byte[] squalifier = statsTuple[1];

    List<KeyValue> results = new ArrayList<KeyValue>();

    Map<Writable, Writable> facets = new MapWritable();

    ntemp = System.nanoTime();
    InternalScanner scanner =
        ((RegionCoprocessorEnvironment) getEnvironment()).getRegion().getScanner(scan);
    octime += System.nanoTime() - ntemp;

    try {
      KeyValue statsFieldKV;
      boolean hasMoreRows = false;

      do {
        ntemp = System.nanoTime();
        hasMoreRows = scanner.next(results);
        nt += System.nanoTime() - ntemp;
        ++nrecords;

        ByteArrayOutputStream gbkey = new ByteArrayOutputStream(128);
        BytesWritable gbkeyt;
        int countOfKVs = 0;

        statsFieldKV = null;

        for (byte[][] gbtuple : groupByTuples) {
          byte[] gbfamily = gbtuple[0];
          byte[] gbqualifier = gbtuple[1];

          for (KeyValue kv : results) {
            if (Bytes.equals(kv.getFamily(), gbfamily)
                && Bytes.equals(kv.getQualifier(), gbqualifier)) {
              if (countOfKVs > 0) {
                gbkey.write(GROUP_BY_SEPARATOR);
              }
              gbkey.write(kv.getValue());
              ++countOfKVs;
            }
            if (Bytes.equals(kv.getFamily(), sfamily)
                && Bytes.equals(kv.getQualifier(), squalifier)) {
              statsFieldKV = kv;
            }
          }
        }

        gbkeyt = new BytesWritable(gbkey.toByteArray());

        if (countOfKVs == groupByTuples.size()) {
          if (statsFieldKV != null) {
            T statsFieldValue = statsCI.getValue(sfamily, squalifier, statsFieldKV);

            if (facets.containsKey(gbkeyt)) {
              GroupByStatsValues<T, S> svs = (GroupByStatsValues<T, S>) facets.get(gbkeyt);
              svs.accumulate(statsFieldValue);
            } else {
              GroupByStatsValues<T, S> svs = new GroupByStatsValues<T, S>(statsCI);
              svs.accumulate(statsFieldValue);
              facets.put(gbkeyt, svs);
            }
          } else {
            if (facets.containsKey(gbkeyt)) {
              GroupByStatsValues<T, S> svs = (GroupByStatsValues<T, S>) facets.get(gbkeyt);
              svs.addMissing(1);
            } else {
              GroupByStatsValues<T, S> svs = new GroupByStatsValues<T, S>(statsCI);
              svs.addMissing(1);
              facets.put(gbkeyt, svs);
            }
          }
        }
        results.clear();
      } while (hasMoreRows);
    } finally {
      ntemp = System.nanoTime();
      scanner.close();
      octime += System.nanoTime() - ntemp;
    }
    Map<? extends Writable, ? extends Writable> derefmap = facets;
    Map<BytesWritable, GroupByStatsValues<T, S>> retmap =
        (Map<BytesWritable, GroupByStatsValues<T, S>>) derefmap;

    et = System.currentTimeMillis();
    ntmillis = nt / 1000000.0;
    ocmillis = octime / 1000000.0;

    log.debug("getStats(): InternalScanner next took " + ntmillis + " ms.");
    log.debug("getStats(): InternalScanner open/close tool " + ocmillis + " ms.");
    log.debug("getStats(): aggregations took " + ((et - bt) - ntmillis) + " ms.");
    log.debug("getStats(): number of values aggregated is " + nrecords);
    log.debug(
        "getStats(): constructed GroupByImplementation HashMap of size " + retmap.size() + ".");
    log.debug("getStats(): finished at " + (et - bt) + " ms.");

    return retmap;
  }