コード例 #1
0
  /**
   * Given a scan and a key range, return a new Scan whose range is truncated to only include keys
   * in that range. Returns null if the Scan does not overlap the given range.
   */
  private static final Scan truncateScan(Scan scan, byte[] rangeStart, byte[] rangeEnd) {
    byte[] scanStart = scan.getStartRow();
    byte[] scanEnd = scan.getStopRow();

    if (scanEnd.length > 0 && bytesCompare(scanEnd, rangeStart) <= 0) {
      // The entire scan range is before the entire cube key range
      return null;
    } else if (scanStart.length > 0 && bytesCompare(scanStart, rangeEnd) >= 0) {
      // The entire scan range is after the entire cube key range
      return null;
    } else {
      // Now we now that the scan range at least partially overlaps the cube key range.
      Scan truncated;
      try {
        truncated = new Scan(scan); // make a copy, don't modify input scan
      } catch (IOException e) {
        throw new RuntimeException(); // This is not plausible
      }

      if (scanStart.length == 0 || bytesCompare(rangeStart, scanStart) > 0) {
        // The scan includes extra keys at the beginning that are not part of the cube. Move
        // the scan start point so that it only touches keys belonging to the cube.
        truncated.setStartRow(rangeStart);
      }
      if (scanEnd.length == 0 || bytesCompare(rangeEnd, scanEnd) < 0) {
        // The scan includes extra keys at the end that are not part of the cube. Move the
        // scan end point so it only touches keys belonging to the cube.
        truncated.setStopRow(rangeEnd);
      }
      return truncated;
    }
  }
コード例 #2
0
 /**
  * @param scan the scan specification
  * @throws Exception
  */
 public static ScannerModel fromScan(Scan scan) throws Exception {
   ScannerModel model = new ScannerModel();
   model.setStartRow(scan.getStartRow());
   model.setEndRow(scan.getStopRow());
   Map<byte[], NavigableSet<byte[]>> families = scan.getFamilyMap();
   if (families != null) {
     for (Map.Entry<byte[], NavigableSet<byte[]>> entry : families.entrySet()) {
       if (entry.getValue() != null) {
         for (byte[] qualifier : entry.getValue()) {
           model.addColumn(Bytes.add(entry.getKey(), COLUMN_DIVIDER, qualifier));
         }
       } else {
         model.addColumn(entry.getKey());
       }
     }
   }
   model.setStartTime(scan.getTimeRange().getMin());
   model.setEndTime(scan.getTimeRange().getMax());
   int caching = scan.getCaching();
   if (caching > 0) {
     model.setBatch(caching);
   }
   int maxVersions = scan.getMaxVersions();
   if (maxVersions > 0) {
     model.setMaxVersions(maxVersions);
   }
   Filter filter = scan.getFilter();
   if (filter != null) {
     model.setFilter(stringifyFilter(filter));
   }
   return model;
 }
コード例 #3
0
 protected boolean checkScanStopRow(final byte[] endKey) {
   if (this.scan.getStopRow().length > 0) {
     byte[] stopRow = scan.getStopRow();
     int cmp = Bytes.compareTo(stopRow, 0, stopRow.length, endKey, 0, endKey.length);
     if (cmp <= 0) {
       return true;
     }
   }
   return false;
 }
コード例 #4
0
  @Override
  public Builder adapt(Scan scan, ReadHooks readHooks) {
    throwIfUnsupportedScan(scan);
    // For gets, startRow == stopRow.  There's no need to create a new ByteString for stopRow
    RowFilter filter = buildFilter(scan, readHooks);

    return ReadRowsRequest.newBuilder()
        .setFilter(filter)
        .setRowRange(
            RowRange.newBuilder()
                .setStartKey(ByteString.copyFrom(scan.getStartRow()))
                .setEndKey(ByteString.copyFrom(scan.getStopRow())));
  }
コード例 #5
0
ファイル: ScanUtil.java プロジェクト: hzg0102/phoenix
 // Start/stop row must be swapped if scan is being done in reverse
 public static void setupReverseScan(Scan scan) {
   if (isReversed(scan)) {
     byte[] startRow = scan.getStartRow();
     byte[] stopRow = scan.getStopRow();
     byte[] newStartRow = startRow;
     byte[] newStopRow = stopRow;
     if (startRow.length != 0) {
       /*
        * Must get previous key because this is going from an inclusive start key to an exclusive stop key, and
        * we need the start key to be included. We get the previous key by decrementing the last byte by one.
        * However, with variable length data types, we need to fill with the max byte value, otherwise, if the
        * start key is 'ab', we lower it to 'aa' which would cause 'aab' to be included (which isn't correct).
        * So we fill with a 0xFF byte to prevent this. A single 0xFF would be enough for our primitive types (as
        * that byte wouldn't occur), but for an arbitrary VARBINARY key we can't know how many bytes to tack
        * on. It's lame of HBase to force us to do this.
        */
       newStartRow =
           Arrays.copyOf(startRow, startRow.length + MAX_FILL_LENGTH_FOR_PREVIOUS_KEY.length);
       if (ByteUtil.previousKey(newStartRow, startRow.length)) {
         System.arraycopy(
             MAX_FILL_LENGTH_FOR_PREVIOUS_KEY,
             0,
             newStartRow,
             startRow.length,
             MAX_FILL_LENGTH_FOR_PREVIOUS_KEY.length);
       } else {
         newStartRow = HConstants.EMPTY_START_ROW;
       }
     }
     if (stopRow.length != 0) {
       // Must add null byte because we need the start to be exclusive while it was inclusive
       newStopRow = ByteUtil.concat(stopRow, QueryConstants.SEPARATOR_BYTE_ARRAY);
     }
     scan.setStartRow(newStopRow);
     scan.setStopRow(newStartRow);
     scan.setReversed(true);
   }
 }
コード例 #6
0
ファイル: ScanUtil.java プロジェクト: hzg0102/phoenix
  public static boolean intersectScanRange(
      Scan scan, byte[] startKey, byte[] stopKey, boolean useSkipScan) {
    boolean mayHaveRows = false;
    int offset = 0;
    if (ScanUtil.isLocalIndex(scan)) {
      offset = startKey.length != 0 ? startKey.length : stopKey.length;
    }
    byte[] existingStartKey = scan.getStartRow();
    byte[] existingStopKey = scan.getStopRow();
    if (existingStartKey.length > 0) {
      if (startKey.length == 0 || Bytes.compareTo(existingStartKey, startKey) > 0) {
        startKey = existingStartKey;
      }
    } else {
      mayHaveRows = true;
    }
    if (existingStopKey.length > 0) {
      if (stopKey.length == 0 || Bytes.compareTo(existingStopKey, stopKey) < 0) {
        stopKey = existingStopKey;
      }
    } else {
      mayHaveRows = true;
    }
    scan.setStartRow(startKey);
    scan.setStopRow(stopKey);
    if (offset > 0 && useSkipScan) {
      byte[] temp = null;
      if (startKey.length != 0) {
        temp = new byte[startKey.length - offset];
        System.arraycopy(startKey, offset, temp, 0, startKey.length - offset);
        startKey = temp;
      }
      if (stopKey.length != 0) {
        temp = new byte[stopKey.length - offset];
        System.arraycopy(stopKey, offset, temp, 0, stopKey.length - offset);
        stopKey = temp;
      }
    }
    mayHaveRows = mayHaveRows || Bytes.compareTo(scan.getStartRow(), scan.getStopRow()) < 0;

    // If the scan is using skip scan filter, intersect and replace the filter.
    if (mayHaveRows && useSkipScan) {
      Filter filter = scan.getFilter();
      if (filter instanceof SkipScanFilter) {
        SkipScanFilter oldFilter = (SkipScanFilter) filter;
        SkipScanFilter newFilter = oldFilter.intersect(startKey, stopKey);
        if (newFilter == null) {
          return false;
        }
        // Intersect found: replace skip scan with intersected one
        scan.setFilter(newFilter);
      } else if (filter instanceof FilterList) {
        FilterList oldList = (FilterList) filter;
        FilterList newList = new FilterList(FilterList.Operator.MUST_PASS_ALL);
        for (Filter f : oldList.getFilters()) {
          if (f instanceof SkipScanFilter) {
            SkipScanFilter newFilter = ((SkipScanFilter) f).intersect(startKey, stopKey);
            if (newFilter == null) {
              return false;
            }
            newList.addFilter(newFilter);
          } else {
            newList.addFilter(f);
          }
        }
        scan.setFilter(newList);
      }
    }
    return mayHaveRows;
  }
コード例 #7
0
  @Override
  public List<String> copQueryAvailableNear(
      String timestamp, final double latitude, final double longitude, final double radius) {
    this.getStatLog(STAT_FILE_NAME);
    this.getCSVLog(this.FILE_NAME_PREFIX, 0);
    this.getCSVLog(this.FILE_NAME_PREFIX, 1);
    this.getCSVLog(this.FILE_NAME_PREFIX, 2);
    this.timePhase.clear();

    try {
      /** Step1** Call back class definition * */
      class BixiCallBack implements Batch.Callback<RCopResult> {
        RCopResult res = new RCopResult();
        int count = 0;
        QueryAbstraction query = null;

        public BixiCallBack(QueryAbstraction query) {
          this.query = query;
        }

        @Override
        public void update(byte[] region, byte[] row, RCopResult result) {
          long current = System.currentTimeMillis();
          count++;
          res.getRes().addAll(result.getRes()); // to verify the error when large data
          res.setStart(result.getStart());
          res.setEnd(result.getEnd());
          res.setRows((res.getRows() + result.getRows()));
          res.setCells(res.getCells() + result.getCells());
          // System.out.println((count++) + ": come back region: "+ Bytes.toString(region) + ";
          // result: "+ result.size());
          /*					String outStr="count=>"+count+";start=>"+result.getStart()+";end=>"+result.getEnd()+";process=>"+(result.getEnd()-result.getStart())+
          		";tranmission=>"+(System.currentTimeMillis()-result.getEnd())+
          		";region=>"+Bytes.toString(region)+
          		";row=>"+result.getRows()+";result=>"+result.getRes().size();
          this.query.writeStat(outStr);
            	outStr = query.regions.get(Bytes.toString(region)).toString();
            	this.query.writeStat(outStr);	*/
          // write them into csv file
          String outStr = "";
          outStr +=
              "within,"
                  + "cop,"
                  + result.getParameter()
                  + ","
                  + result.getStart()
                  + ","
                  + result.getEnd()
                  + ","
                  + current
                  + ","
                  + result.getRows()
                  + ","
                  + result.getCells()
                  + ","
                  + ","
                  + result.getKvLength()
                  + result.getRes().size()
                  + ","
                  + this.query.regionAndRS.get(Bytes.toString(region))
                  + ","
                  + Bytes.toString(region);
          this.query.writeCSVLog(outStr, 1);
        }
      }
      BixiCallBack callBack = new BixiCallBack(this);

      /** Step2** generate the scan ********** */
      long s_time = System.currentTimeMillis();
      this.timePhase.add(s_time);
      // build up the Raster
      XRaster raster = new XRaster(this.space, this.min_size_of_height, this.max_num_of_column);
      // match the query area in Raster to get the row range and column
      // range
      long match_s = System.currentTimeMillis();
      XBox[] match_boxes = raster.match(latitude, longitude, radius);
      long match_time = System.currentTimeMillis() - match_s;
      String[] rowRange = new String[2];
      rowRange[0] = match_boxes[0].getRow();
      rowRange[1] = match_boxes[1].getRow() + "-*";

      String[] c = raster.getColumns(match_boxes[0], match_boxes[1]);
      // generate the scan
      final Scan scan =
          hbaseUtil.generateScan(
              rowRange, null, new String[] {BixiConstant.LOCATION_FAMILY_NAME}, c, 1000000);

      /** Step3** send out the query to trigger the corresponding function in Coprocessor*** */
      this.timePhase.add(System.currentTimeMillis());
      hbaseUtil
          .getHTable()
          .coprocessorExec(
              BixiProtocol.class,
              scan.getStartRow(),
              scan.getStopRow(),
              new Batch.Call<BixiProtocol, RCopResult>() {

                public RCopResult call(BixiProtocol instance) throws IOException {
                  return instance.copQueryNeighbor4LS2(scan, latitude, longitude, radius);
                };
              },
              callBack);

      long e_time = System.currentTimeMillis();
      this.timePhase.add(e_time);

      long exe_time = e_time - s_time;

      /*			System.out.println("exe_time=>"+exe_time+";result=>"+callBack.res.getRes().size());
      String outStr = "m=>cop;"+"radius=>"+radius+";exe_time=>"+exe_time+";result=>"+callBack.res.getRes().size();
      this.writeStat(outStr);*/
      // write to csv file
      String outStr = "";
      outStr +=
          "within,"
              + "cop,"
              + callBack.res.getCells()
              + ","
              + callBack.res.getRes().size()
              + ","
              + exe_time
              + ","
              + callBack.res.getRows()
              + ","
              + match_time
              + ","
              + this.min_size_of_height
              + ","
              + radius;

      this.writeCSVLog(outStr, 0);

      String timeStr = "within,cop," + radius + ",";
      for (int i = 0; i < this.timePhase.size(); i++) {
        timeStr += this.timePhase.get(i) + ",";
      }
      this.writeCSVLog(timeStr, 2);

      return callBack.res.getRes();

    } catch (Exception e) {
      e.printStackTrace();
    } catch (Throwable ee) {
      ee.printStackTrace();
    } finally {
      hbaseUtil.closeTableHandler();
      this.closeStatLog();
      this.closeCSVLog();
    }
    return null;
  }
コード例 #8
0
  @Override
  public String copQueryPoint(final double latitude, final double longitude) {

    this.getStatLog(STAT_FILE_NAME);

    long s_time = System.currentTimeMillis();
    try {
      /** Step1** Call back class definition * */
      class BixiCallBack implements Batch.Callback<String> {
        String res = null;
        int count = 0;
        QueryAbstraction query = null;

        public BixiCallBack(QueryAbstraction query) {
          this.query = query;
        }

        @Override
        public void update(byte[] region, byte[] row, String result) {
          count++;
          String outStr =
              "endTime=>"
                  + System.currentTimeMillis()
                  + ";region=>"
                  + Bytes.toString(region)
                  + ";count=>"
                  + count
                  + ";result=>"
                  + result;
          this.query.writeStat(outStr);
          outStr = query.regions.get(Bytes.toString(region)).toString();
          this.query.writeStat(outStr);
          res = result;
        }
      }
      BixiCallBack callBack = new BixiCallBack(this);

      /** Step2** generate the scan ********** */
      // build up the Raster
      XRaster raster = new XRaster(this.space, this.min_size_of_height, this.max_num_of_column);
      XBox match_box = raster.locate(latitude, Math.abs(longitude));
      System.out.println("match_box is : " + match_box.toString());
      String[] rowRange = new String[2];
      rowRange[0] = match_box.getRow();
      rowRange[1] = match_box.getRow() + "0";

      // generate the scan
      final Scan scan =
          hbaseUtil.generateScan(
              rowRange,
              null,
              new String[] {BixiConstant.LOCATION_FAMILY_NAME},
              new String[] {match_box.getColumn()},
              100000);

      /** Step3** send out the query to trigger the corresponding function in Coprocessor*** */
      hbaseUtil
          .getHTable()
          .coprocessorExec(
              BixiProtocol.class,
              scan.getStartRow(),
              scan.getStopRow(),
              new Batch.Call<BixiProtocol, String>() {

                public String call(BixiProtocol instance) throws IOException {

                  return instance.copQueryPoint4LS2(scan, latitude, longitude);
                };
              },
              callBack);

      long e_time = System.currentTimeMillis();

      long exe_time = e_time - s_time;
      // TODO store the time into database
      System.out.println("exe_time=>" + exe_time + ";result=>" + callBack.res);
      String outStr = "q=point;m=>cop;" + ";exe_time=>" + exe_time + ";result=>" + callBack.res;
      this.writeStat(outStr);

      return callBack.res;

    } catch (Exception e) {
      e.printStackTrace();
    } catch (Throwable ee) {
      ee.printStackTrace();
    } finally {
      hbaseUtil.closeTableHandler();
      this.closeStatLog();
    }
    return null;
  }
コード例 #9
0
ファイル: ScanRanges.java プロジェクト: zxs/phoenix
  public Scan intersectScan(
      Scan scan,
      final byte[] originalStartKey,
      final byte[] originalStopKey,
      final int keyOffset,
      boolean crossesRegionBoundary) {
    byte[] startKey = originalStartKey;
    byte[] stopKey = originalStopKey;
    if (stopKey.length > 0 && Bytes.compareTo(startKey, stopKey) >= 0) {
      return null;
    }
    boolean mayHaveRows = false;
    // Keep the keys as they are if we have a point lookup, as we've already resolved the
    // salt bytes in that case.
    final int scanKeyOffset =
        this.isSalted && !this.isPointLookup ? SaltingUtil.NUM_SALTING_BYTES : 0;
    assert (scanKeyOffset == 0 || keyOffset == 0);
    // Total offset for startKey/stopKey. Either 1 for salted tables or the prefix length
    // of the current region for local indexes. We'll never have a case where a table is
    // both salted and local.
    final int totalKeyOffset = scanKeyOffset + keyOffset;
    byte[] prefixBytes = ByteUtil.EMPTY_BYTE_ARRAY;
    if (totalKeyOffset > 0) {
      prefixBytes = ScanUtil.getPrefix(startKey, totalKeyOffset);
      /*
       * If our startKey to stopKey crosses a region boundary consider everything after the startKey as our scan
       * is always done within a single region. This prevents us from having to prefix the key prior to knowing
       * whether or not there may be an intersection. We can't calculate whether or not we've crossed a region
       * boundary for local indexes, because we don't know the key offset of the next region, but only for the
       * current one (which is the one passed in). If the next prefix happened to be a subset of the previous
       * prefix, then this wouldn't detect that we crossed a region boundary.
       */
      if (crossesRegionBoundary) {
        stopKey = ByteUtil.EMPTY_BYTE_ARRAY;
      }
    }
    int scanStartKeyOffset = scanKeyOffset;
    byte[] scanStartKey = scan == null ? ByteUtil.EMPTY_BYTE_ARRAY : scan.getStartRow();
    // Compare ignoring key prefix and salt byte
    if (scanStartKey.length > 0) {
      if (startKey.length > 0
          && Bytes.compareTo(
                  scanStartKey,
                  scanKeyOffset,
                  scanStartKey.length - scanKeyOffset,
                  startKey,
                  totalKeyOffset,
                  startKey.length - totalKeyOffset)
              < 0) {
        scanStartKey = startKey;
        scanStartKeyOffset = totalKeyOffset;
      }
    } else {
      scanStartKey = startKey;
      scanStartKeyOffset = totalKeyOffset;
      mayHaveRows = true;
    }
    int scanStopKeyOffset = scanKeyOffset;
    byte[] scanStopKey = scan == null ? ByteUtil.EMPTY_BYTE_ARRAY : scan.getStopRow();
    if (scanStopKey.length > 0) {
      if (stopKey.length > 0
          && Bytes.compareTo(
                  scanStopKey,
                  scanKeyOffset,
                  scanStopKey.length - scanKeyOffset,
                  stopKey,
                  totalKeyOffset,
                  stopKey.length - totalKeyOffset)
              > 0) {
        scanStopKey = stopKey;
        scanStopKeyOffset = totalKeyOffset;
      }
    } else {
      scanStopKey = stopKey;
      scanStopKeyOffset = totalKeyOffset;
      mayHaveRows = true;
    }
    mayHaveRows =
        mayHaveRows
            || Bytes.compareTo(
                    scanStartKey,
                    scanStartKeyOffset,
                    scanStartKey.length - scanStartKeyOffset,
                    scanStopKey,
                    scanStopKeyOffset,
                    scanStopKey.length - scanStopKeyOffset)
                < 0;

    if (!mayHaveRows) {
      return null;
    }
    if (originalStopKey.length != 0 && scanStopKey.length == 0) {
      scanStopKey = originalStopKey;
    }
    Filter newFilter = null;
    // If the scan is using skip scan filter, intersect and replace the filter.
    if (scan == null || this.useSkipScanFilter()) {
      byte[] skipScanStartKey = scanStartKey;
      byte[] skipScanStopKey = scanStopKey;
      // If we have a keyOffset and we've used the startKey/stopKey that
      // were passed in (which have the prefix) for the above range check,
      // we need to remove the prefix before running our intersect method.
      // TODO: we could use skipScanFilter.setOffset(keyOffset) if both
      // the startKey and stopKey were used above *and* our intersect
      // method honored the skipScanFilter.offset variable.
      if (scanKeyOffset > 0) {
        if (skipScanStartKey != originalStartKey) { // original already has correct salt byte
          skipScanStartKey = replaceSaltByte(skipScanStartKey, prefixBytes);
        }
        if (skipScanStopKey != originalStopKey) {
          skipScanStopKey = replaceSaltByte(skipScanStopKey, prefixBytes);
        }
      } else if (keyOffset > 0) {
        if (skipScanStartKey == originalStartKey) {
          skipScanStartKey = stripPrefix(skipScanStartKey, keyOffset);
        }
        if (skipScanStopKey == originalStopKey) {
          skipScanStopKey = stripPrefix(skipScanStopKey, keyOffset);
        }
      }
      if (scan == null) {
        return filter.hasIntersect(skipScanStartKey, skipScanStopKey) ? HAS_INTERSECTION : null;
      }
      Filter filter = scan.getFilter();
      SkipScanFilter newSkipScanFilter = null;
      if (filter instanceof SkipScanFilter) {
        SkipScanFilter oldSkipScanFilter = (SkipScanFilter) filter;
        newFilter =
            newSkipScanFilter = oldSkipScanFilter.intersect(skipScanStartKey, skipScanStopKey);
        if (newFilter == null) {
          return null;
        }
      } else if (filter instanceof FilterList) {
        FilterList oldList = (FilterList) filter;
        FilterList newList = new FilterList(FilterList.Operator.MUST_PASS_ALL);
        newFilter = newList;
        for (Filter f : oldList.getFilters()) {
          if (f instanceof SkipScanFilter) {
            newSkipScanFilter = ((SkipScanFilter) f).intersect(skipScanStartKey, skipScanStopKey);
            if (newSkipScanFilter == null) {
              return null;
            }
            newList.addFilter(newSkipScanFilter);
          } else {
            newList.addFilter(f);
          }
        }
      }
      // TODO: it seems that our SkipScanFilter or HBase runs into problems if we don't
      // have an enclosing range when we do a point lookup.
      if (isPointLookup) {
        scanStartKey = ScanUtil.getMinKey(schema, newSkipScanFilter.getSlots(), slotSpan);
        scanStopKey = ScanUtil.getMaxKey(schema, newSkipScanFilter.getSlots(), slotSpan);
      }
    }
    if (newFilter == null) {
      newFilter = scan.getFilter();
    }
    Scan newScan = ScanUtil.newScan(scan);
    newScan.setFilter(newFilter);
    // If we have an offset (salted table or local index), we need to make sure to
    // prefix our scan start/stop row by the prefix of the startKey or stopKey that
    // were passed in. Our scan either doesn't have the prefix or has a placeholder
    // for it.
    if (totalKeyOffset > 0) {
      if (scanStartKey != originalStartKey) {
        scanStartKey = prefixKey(scanStartKey, scanKeyOffset, prefixBytes, keyOffset);
      }
      if (scanStopKey != originalStopKey) {
        scanStopKey = prefixKey(scanStopKey, scanKeyOffset, prefixBytes, keyOffset);
      }
    }
    // Don't let the stopRow of the scan go beyond the originalStopKey
    if (originalStopKey.length > 0 && Bytes.compareTo(scanStopKey, originalStopKey) > 0) {
      scanStopKey = originalStopKey;
    }
    if (scanStopKey.length > 0 && Bytes.compareTo(scanStartKey, scanStopKey) >= 0) {
      return null;
    }
    newScan.setAttribute(SCAN_ACTUAL_START_ROW, scanStartKey);
    newScan.setStartRow(scanStartKey);
    newScan.setStopRow(scanStopKey);
    if (keyOffset > 0) {
      newScan.setAttribute(STARTKEY_OFFSET, Bytes.toBytes(keyOffset));
    }

    return newScan;
  }