private Scan createScan(String applicationName, Range range, boolean scanBackward) {
    Scan scan = new Scan();
    scan.setCaching(this.scanCacheSize);

    byte[] bApplicationName = Bytes.toBytes(applicationName);
    byte[] traceIndexStartKey = SpanUtils.getTraceIndexRowKey(bApplicationName, range.getFrom());
    byte[] traceIndexEndKey = SpanUtils.getTraceIndexRowKey(bApplicationName, range.getTo());

    if (scanBackward) {
      // start key is replaced by end key because key has been reversed
      scan.setStartRow(traceIndexEndKey);
      scan.setStopRow(traceIndexStartKey);
    } else {
      scan.setReversed(true);
      scan.setStartRow(traceIndexStartKey);
      scan.setStopRow(traceIndexEndKey);
    }

    scan.addFamily(HBaseTables.APPLICATION_TRACE_INDEX_CF_TRACE);
    scan.setId("ApplicationTraceIndexScan");

    // toString() method of Scan converts a message to json format so it is slow for the first time.
    logger.trace("create scan:{}", scan);
    return scan;
  }
  public static ResultScanner getAllFamilyList(
      String startRowRange, String stopRowRange, byte[][] cfs, String ctableName) {

    Scan scan = new Scan();

    scan.setStartRow(Bytes.toBytes(startRowRange));

    for (byte[] family : cfs) {
      scan.addFamily(family);
    }

    if (stopRowRange != null) {
      scan.setStopRow(Bytes.toBytes(stopRowRange));
    }

    ResultScanner resultScanner = null;

    try {
      resultScanner = tblMngr.getTable(ctableName).getScanner(scan);

    } catch (Exception e) {

    }
    return resultScanner;
  }
Example #3
0
 /**
  * Performs a full scan of a catalog table.
  *
  * @param catalogTracker
  * @param visitor Visitor invoked against each row.
  * @param startrow Where to start the scan. Pass null if want to begin scan at first row.
  * @param scanRoot True if we are to scan <code>-ROOT-</code> rather than <code>.META.</code>, the
  *     default (pass false to scan .META.)
  * @throws IOException
  */
 static void fullScan(
     CatalogTracker catalogTracker,
     final Visitor visitor,
     final byte[] startrow,
     final boolean scanRoot)
     throws IOException {
   Scan scan = new Scan();
   if (startrow != null) scan.setStartRow(startrow);
   if (startrow == null && !scanRoot) {
     int caching =
         catalogTracker
             .getConnection()
             .getConfiguration()
             .getInt(HConstants.HBASE_META_SCANNER_CACHING, 100);
     scan.setCaching(caching);
   }
   scan.addFamily(HConstants.CATALOG_FAMILY);
   HTable metaTable = scanRoot ? getRootHTable(catalogTracker) : getMetaHTable(catalogTracker);
   ResultScanner scanner = metaTable.getScanner(scan);
   try {
     Result data;
     while ((data = scanner.next()) != null) {
       if (data.isEmpty()) continue;
       // Break if visit returns false.
       if (!visitor.visit(data)) break;
     }
   } finally {
     scanner.close();
     metaTable.close();
   }
   return;
 }
  /**
   * 遍历多行
   *
   * @param tableName 表名
   * @param start_rowkey 开始行键
   * @param stop_rowkey 结束行键
   * @return 行列表
   */
  public ArrayList<HbaseRow> scanRows(String tableName, String start_rowkey, String stop_rowkey) {
    ResultScanner rowstmp = null;
    ArrayList<HbaseRow> rows = null;

    try {
      Scan scan = new Scan();
      scan.setStartRow(Bytes.toBytes(start_rowkey));
      scan.setStopRow(Bytes.toBytes(stop_rowkey));
      HTable table = new HTable(conf, Bytes.toBytes(tableName));
      rowstmp = table.getScanner(scan);
      rows = new ArrayList<>();

      for (Result rowtmp : rowstmp) {
        HbaseRow row = new HbaseRow();
        row.rowkey = Bytes.toString(rowtmp.getRow());
        for (Cell cell : rowtmp.listCells()) {
          HbaseColumn col = new HbaseColumn(cell);
          row.cols.add(col);
        }
        rows.add(row);
      }
    } catch (Exception e) {
      logger.error("scanRows failed", e);
    } finally {
      rowstmp.close();
    }
    return rows;
  }
Example #5
0
  /**
   * Gets an hbase scanner.
   *
   * @return An hbase scanner.
   * @throws IOException Error accessing hbase.
   */
  @Override
  protected Scan getScanner() throws IOException {
    Scan scan = super.getScanner();

    if (this.query != null) {
      if (this.query.getStartKey() != null) {
        scan.setStartRow(this.query.getStartKey());
      }

      if (this.query.getEndKey() != null) {
        scan.setStopRow(padWithMaxUnicode(this.query.getEndKey()));
      }

      if (this.query.getStartDate() != null && this.query.getEndDate() != null) {
        scan.setTimeRange(this.query.getStartDate().getTime(), this.query.getEndDate().getTime());
      }

      if (this.query.getWord() != null || this.query.getOperator().isUnary()) {
        WritableByteArrayComparable comparator;

        switch (this.query.getOperator()) {
          case Contains:
            comparator = new SubstringComparator(this.query.getWord());
            break;
          case StartsWith:
            comparator = new RegexStringComparator(String.format("^%s.*", this.query.getWord()));
            break;
          case EndsWith:
            comparator = new RegexStringComparator(String.format(".*%s$", this.query.getWord()));
            break;
          case Less:
          case LessOrEqual:
          case Equal:
          case NotEqual:
          case GreaterOrEqual:
          case Greater:
            comparator = new BinaryComparator(this.query.getWordAsByteArray());
            break;
          case IsNull:
          case IsNotNull:
            comparator = new BinaryComparator(EMPTY_BYTES_ARRAY);
            break;
          default:
            throw new IllegalArgumentException(
                String.format(
                    "The specified operator type '%s' is not supported.",
                    this.query.getOperator()));
        }

        scan.setFilter(
            new SingleColumnValueFilter(
                Bytes.toBytesBinary(this.query.getFamily()),
                Bytes.toBytesBinary(this.query.getColumn()),
                this.query.getOperator().toFilter(),
                comparator));
      }
    }

    return scan;
  }
  /**
   * Given a scan and a key range, return a new Scan whose range is truncated to only include keys
   * in that range. Returns null if the Scan does not overlap the given range.
   */
  private static final Scan truncateScan(Scan scan, byte[] rangeStart, byte[] rangeEnd) {
    byte[] scanStart = scan.getStartRow();
    byte[] scanEnd = scan.getStopRow();

    if (scanEnd.length > 0 && bytesCompare(scanEnd, rangeStart) <= 0) {
      // The entire scan range is before the entire cube key range
      return null;
    } else if (scanStart.length > 0 && bytesCompare(scanStart, rangeEnd) >= 0) {
      // The entire scan range is after the entire cube key range
      return null;
    } else {
      // Now we now that the scan range at least partially overlaps the cube key range.
      Scan truncated;
      try {
        truncated = new Scan(scan); // make a copy, don't modify input scan
      } catch (IOException e) {
        throw new RuntimeException(); // This is not plausible
      }

      if (scanStart.length == 0 || bytesCompare(rangeStart, scanStart) > 0) {
        // The scan includes extra keys at the beginning that are not part of the cube. Move
        // the scan start point so that it only touches keys belonging to the cube.
        truncated.setStartRow(rangeStart);
      }
      if (scanEnd.length == 0 || bytesCompare(rangeEnd, scanEnd) < 0) {
        // The scan includes extra keys at the end that are not part of the cube. Move the
        // scan end point so it only touches keys belonging to the cube.
        truncated.setStopRow(rangeEnd);
      }
      return truncated;
    }
  }
  /**
   * Get a collection of Scans, one per region, that cover the range of the table having the given
   * key prefix. Thes will be used as the map task input splits.
   */
  public static List<Scan> scansThisCubeOnly(byte[] keyPrefix, byte[][] splitKeys)
      throws IOException {
    Scan copyScan = new Scan();
    copyScan.setCaching(5000);
    copyScan.setCacheBlocks(false);

    // Hack: generate a key that probably comes after all this cube's keys but doesn't include any
    // keys not belonging to this cube.
    byte[] keyAfterCube = ArrayUtils.addAll(keyPrefix, fiftyBytesFF);

    List<Scan> scans = new ArrayList<Scan>();
    Scan scanUnderConstruction = new Scan(copyScan);

    for (byte[] splitKey : splitKeys) {
      scanUnderConstruction.setStopRow(splitKey);

      // Coerce scan to only touch keys belonging to this cube
      Scan truncated = truncateScan(scanUnderConstruction, keyPrefix, keyAfterCube);
      if (truncated != null) {
        scans.add(truncated);
      }

      scanUnderConstruction = new Scan(copyScan);
      scanUnderConstruction.setStartRow(splitKey);
    }

    // There's another region from last split key to the end of the table.
    Scan truncated = truncateScan(scanUnderConstruction, keyPrefix, keyAfterCube);
    if (truncated != null) {
      scans.add(truncated);
    }

    return scans;
  }
    public static List<Delete> GetDeleteEventsBetween(
        Table VTEvent_Table, String imo_str, long first_timestamp, long last_timestamp)
        throws IOException {
      // scan
      // 'cdb_vessel:vessel_event',{FILTER=>"(PrefixFilter('0000003162')"}
      Scan GetEventsBetween = new Scan();
      GetEventsBetween.setStartRow(
              Bytes.toBytes(imo_str + LpadNum(Long.MAX_VALUE - last_timestamp, 19) + "0000000000"))
          .setStopRow(
              Bytes.toBytes(
                  imo_str + LpadNum(Long.MAX_VALUE - first_timestamp + 1, 19) + "9999999999"))
          .addColumn(details, exittime);
      GetEventsBetween.setCaching(100);

      Filter ExistTimeValuefilter =
          new ValueFilter(
              CompareFilter.CompareOp.LESS_OR_EQUAL,
              new BinaryComparator(
                  Bytes.toBytes(new DateTime(last_timestamp).toString(rawformatter))));
      GetEventsBetween.setFilter(ExistTimeValuefilter);

      ResultScanner Result_ExistingEvents = VTEvent_Table.getScanner(GetEventsBetween);
      List<Delete> deletes = new ArrayList<Delete>();

      for (Result res : Result_ExistingEvents) {
        deletes.add(new Delete(res.getRow()));
      }

      Result_ExistingEvents.close();
      return deletes;
    }
  public static ResultScanner getList(
      String startRowRange,
      String stopRowRange,
      byte[] cf1,
      byte[] cf2,
      long limit,
      FilterList filterList,
      String ctableName) {

    Scan scan = new Scan();
    scan.addFamily(cf1);
    scan.setStartRow(Bytes.toBytes(startRowRange));

    if (stopRowRange != null) {
      scan.setStopRow(Bytes.toBytes(stopRowRange));
    }
    if (limit != 0) {

      filterList.addFilter(new PageFilter(limit));
    } else {
      filterList.addFilter(new PageFilter(100));
    }
    scan.setFilter(filterList);
    ResultScanner resultScanner = null;

    try {
      resultScanner = tblMngr.getTable(ctableName).getScanner(scan);

    } catch (Exception e) {

    }
    return resultScanner;
  }
Example #10
0
  public Scan generateScan(
      String[] rowRange, FilterList filterList, String[] family, String[] columns, int maxVersion)
      throws Exception {
    if (table == null) throw new Exception("No table handler");
    if (cacheSize < 0) throw new Exception("should set cache size before scanning");

    Scan scan = null;

    try {
      scan = new Scan();
      scan.setCaching(this.cacheSize);
      scan.setCacheBlocks(this.blockCached);
      scan.setFilter(filterList);
      if (maxVersion > 0) scan.setMaxVersions(maxVersion);
      if (rowRange != null) {
        scan.setStartRow(rowRange[0].getBytes());
        if (rowRange.length == 2) scan.setStopRow(rowRange[1].getBytes());
      }

      if (columns != null) {
        for (int i = 0; i < columns.length; i++) {
          scan.addColumn(family[0].getBytes(), columns[i].getBytes());
          // System.out.println(family[i]+";"+columns[i]);
        }
      } else {
        scan.addFamily(family[0].getBytes());
      }

    } catch (Exception e) {
      e.printStackTrace();
    }

    return scan;
  }
Example #11
0
 // 获取扫描器对象
 private static Scan getScan(String startRow, String stopRow) {
   Scan scan = new Scan();
   if (startRow != null && stopRow != null) {
     scan.setStartRow(getBytes(startRow));
     scan.setStopRow(getBytes(stopRow));
   }
   return scan;
 }
    // Get all events with exit at last location
    public static Map<Integer, VesselEvent> getAllEventsStartBeforeEndAfterBeforeLocation(
        Table VTEvent_Table, String IMO_str, VesselLocation location) throws IOException {
      Scan getAllEventsWithExistAtLastLocation = new Scan();
      getAllEventsWithExistAtLastLocation
          .setStartRow(
              Bytes.toBytes(
                  IMO_str + LpadNum(Long.MAX_VALUE - location.recordtime, 19) + "0000000000"))
          .setStopRow(Bytes.toBytes(IMO_str + LpadNum(Long.MAX_VALUE, 19) + "9999999999"))
          .addColumn(details, exittime);
      getAllEventsWithExistAtLastLocation.setCaching(100);

      Filter ExistTimeValuefilter =
          new ValueFilter(
              CompareFilter.CompareOp.GREATER_OR_EQUAL,
              new BinaryComparator(
                  Bytes.toBytes(new DateTime(location.recordtime).toString(rawformatter))));
      getAllEventsWithExistAtLastLocation.setFilter(ExistTimeValuefilter);

      ResultScanner Result_event = VTEvent_Table.getScanner(getAllEventsWithExistAtLastLocation);

      Map<Integer, VesselEvent> events = new HashMap<Integer, VesselEvent>();

      for (Result res : Result_event) {

        Get get = new Get(res.getRow());
        get.addColumn(details, entrytime);
        get.addColumn(details, entrycoordinates);

        Result result = VTEvent_Table.get(get);
        String rowkey = Bytes.toString(result.getRow());
        String polygonid = rowkey.substring(26);

        VesselEvent VE = new VesselEvent();
        VE.exittime = location.recordtime;
        VE.exitcoordinates = location.coordinates;
        VE.destination = location.destination;
        VE.polygonid = Integer.parseInt(polygonid);

        for (Cell cell : result.rawCells()) {
          String Qualifier = Bytes.toString(CellUtil.cloneQualifier(cell));
          String Value = Bytes.toString(CellUtil.cloneValue(cell));

          if (Qualifier.equals("entertime")) {
            VE.entrytime = DateTime.parse(Value, rawformatter).getMillis();
          } else if (Qualifier.equals("entercoordinates")) {
            VE.entrycoordinates = Value;
          }
        }

        events.put(VE.polygonid, VE);
      }

      Result_event.close();
      return events;
    }
  public HBaseSecondaryIndexCursor(
      HBaseSecondaryIndex index, TableFilter filter, byte[] startKey, byte[] endKey) {
    defaultColumnFamilyName =
        Bytes.toBytes(((HBaseTable) filter.getTable()).getDefaultColumnFamilyName());
    secondaryIndex = index;
    session = (HBaseSession) filter.getSession();
    Prepared p = filter.getPrepared();
    if (p instanceof WithWhereClause) {
      regionName = Bytes.toBytes(((WithWhereClause) p).getWhereClauseSupport().getRegionName());
    }

    if (regionName == null) throw new RuntimeException("regionName is null");

    fetchSize = filter.getPrepared().getFetchSize();
    // 非查询的操作一般不设置fetchSize,此时fetchSize为0,所以要设置一个默认值
    // if (fetchSize < 1 && !filter.getPrepared().isQuery())
    if (fetchSize < 1) fetchSize = SysProperties.SERVER_RESULT_SET_FETCH_SIZE;

    if (filter.getSelect() != null) columns = filter.getSelect().getColumns(filter);
    else columns = Arrays.asList(filter.getTable().getColumns());

    if (startKey == null) startKey = HConstants.EMPTY_BYTE_ARRAY;
    if (endKey == null) endKey = HConstants.EMPTY_BYTE_ARRAY;

    Scan scan = new Scan();
    scan.setMaxVersions(1);
    try {
      HRegionInfo info = session.getRegionServer().getRegionInfo(regionName);
      if (Bytes.compareTo(startKey, info.getStartKey()) >= 0) scan.setStartRow(startKey);
      else scan.setStartRow(info.getStartKey());

      if (Bytes.equals(endKey, HConstants.EMPTY_BYTE_ARRAY)) scan.setStopRow(info.getEndKey());
      else if (Bytes.compareTo(endKey, info.getEndKey()) < 0) scan.setStopRow(endKey);
      else scan.setStopRow(info.getEndKey());

      scan.addColumn(HBaseSecondaryIndex.PSEUDO_FAMILY, HBaseSecondaryIndex.PSEUDO_COLUMN);

      scannerId = session.getRegionServer().openScanner(regionName, scan);
    } catch (Exception e) {
      throw new RuntimeException(e);
    }
  }
 @Test
 public void testStdWithValidRange2WithNoCQ() throws Throwable {
   AggregationClient aClient = new AggregationClient(conf);
   Scan scan = new Scan();
   scan.addFamily(TEST_FAMILY);
   scan.setStartRow(ROWS[6]);
   scan.setStopRow(ROWS[7]);
   final ColumnInterpreter<Long, Long, EmptyMsg, LongMsg, LongMsg> ci =
       new LongColumnInterpreter();
   double std = aClient.std(TEST_TABLE, ci, scan);
   assertEquals(0, std, 0);
 }
 /** @throws Throwable */
 @Test
 public void testStdWithValidRange2() throws Throwable {
   AggregationClient aClient = new AggregationClient(conf);
   Scan scan = new Scan();
   scan.addColumn(TEST_FAMILY, TEST_QUALIFIER);
   scan.setStartRow(ROWS[5]);
   scan.setStopRow(ROWS[15]);
   final ColumnInterpreter<Long, Long, EmptyMsg, LongMsg, LongMsg> ci =
       new LongColumnInterpreter();
   double std = aClient.std(TEST_TABLE, ci, scan);
   assertEquals(2.87, std, 0.05d);
 }
 @Test
 public void testMinWithValidRangeWithNoCQ() throws Throwable {
   AggregationClient aClient = new AggregationClient(conf);
   Scan scan = new Scan();
   scan.addFamily(TEST_FAMILY);
   scan.setStartRow(HConstants.EMPTY_START_ROW);
   scan.setStopRow(HConstants.EMPTY_END_ROW);
   final ColumnInterpreter<Long, Long, EmptyMsg, LongMsg, LongMsg> ci =
       new LongColumnInterpreter();
   long min = aClient.min(TEST_TABLE, ci, scan);
   assertEquals(0, min);
 }
Example #17
0
  @Override
  public void tweet(User user, String tweetText) throws IOException {
    final long epoch = System.currentTimeMillis();
    final long tranposeEpoch = Long.MAX_VALUE - epoch;
    final byte[] epochBytes = Bytes.toBytes(epoch);

    final byte[] tweetBytes = Bytes.toBytes(tweetText);
    byte[] nameBytes = Bytes.toBytes(user.getName());

    /** put tweet into tweets */
    Put tweetRowPut = new Put(generateTweetId(user));

    tweetRowPut.add(_DEFAULT, _NAME, nameBytes);
    tweetRowPut.add(_DEFAULT, _MAIL, Bytes.toBytes(user.getEmail()));
    tweetRowPut.add(_DEFAULT, _TWEET, tweetBytes);
    tweetRowPut.add(_DEFAULT, _TIME, epochBytes);

    tweetsTable.put(tweetRowPut);

    /** put tweets for followers */
    Scan followerScan = new Scan();
    followerScan.setStartRow(Bytes.toBytes(user.getUserId() + "-"));
    followerScan.setStopRow(Bytes.toBytes((user.getUserId() + 1) + "-"));

    ResultScanner followerRS = followersTable.getScanner(followerScan);

    /** put users on tweet to her own tweetline */
    Put put =
        new Put(Bytes.toBytes(user.getUserId() + "-" + tranposeEpoch + "-" + user.getUserId()));
    put.add(_DEFAULT, _NAME, nameBytes);
    put.add(_DEFAULT, _TWEET, tweetBytes);
    put.add(_DEFAULT, _TIME, epochBytes);

    List<Row> puts = new ArrayList<Row>();
    puts.add(put);
    for (Result result : followerRS) {

      Long followerid = Bytes.toLong(result.getColumnLatest(_DEFAULT, _USERID).getValue());

      put = new Put(Bytes.toBytes(followerid + "-" + tranposeEpoch + "-" + user.getUserId()));
      put.add(_DEFAULT, _NAME, nameBytes);
      put.add(_DEFAULT, _TWEET, tweetBytes);
      put.add(_DEFAULT, _TIME, epochBytes);

      puts.add(put);
    }
    followerRS.close();
    try {
      tweetlineTable.batch(puts);
    } catch (InterruptedException e) {
      e.printStackTrace(); // @TODO log and handle properly.
    }
  }
 /**
  * This will test rowcount with a valid range, i.e., a subset of rows. It will be the most common
  * use case.
  *
  * @throws Throwable
  */
 @Test
 public void testRowCountWithValidRange() throws Throwable {
   AggregationClient aClient = new AggregationClient(conf);
   Scan scan = new Scan();
   scan.addColumn(TEST_FAMILY, TEST_QUALIFIER);
   scan.setStartRow(ROWS[2]);
   scan.setStopRow(ROWS[14]);
   final ColumnInterpreter<Long, Long, EmptyMsg, LongMsg, LongMsg> ci =
       new LongColumnInterpreter();
   long rowCount = aClient.rowCount(TEST_TABLE, ci, scan);
   assertEquals(12, rowCount);
 }
 /**
  * Configure the scan according to the row options.
  *
  * @param scan The HBase scan descriptor to configure.
  * @throws IOException If there is an error.
  */
 private void configureScanWithRowOptions(Scan scan) throws IOException {
   if (null != mRowOptions.getStartRow()) {
     scan.setStartRow(mRowOptions.getStartRow().getHBaseRowKey());
   }
   if (null != mRowOptions.getLimitRow()) {
     scan.setStopRow(mRowOptions.getLimitRow().getHBaseRowKey());
   }
   if (null != mRowOptions.getRowFilter()) {
     KijiTableLayout tableLayout = mInputTable.getLayout();
     KijiSchemaTable schemaTable = mInputTable.getKiji().getSchemaTable();
     new KijiRowFilterApplicator(mRowOptions.getRowFilter(), tableLayout, schemaTable)
         .applyTo(scan);
   }
 }
 public <V> List<V> list(
     String tableName, byte[] startKey, byte[] stopKey, Filter filter, HbaseMapper<V> mapper)
     throws Exception {
   HTableInterface htable = dataSource.getConnection(tableName);
   Scan scan = new Scan();
   scan.setStartRow(startKey);
   scan.setStopRow(stopKey);
   scan.setFilter(filter);
   log.info("scan is -[" + scan + "]");
   ResultScanner resultScanner = htable.getScanner(scan);
   List<V> result = trasfer(resultScanner, mapper);
   htable.close();
   log.info("list " + result.size() + " objects from " + tableName);
   return result;
 }
 /** This should return a 0 */
 @Test
 public void testRowCountWithNullCF() {
   AggregationClient aClient = new AggregationClient(conf);
   Scan scan = new Scan();
   scan.setStartRow(ROWS[5]);
   scan.setStopRow(ROWS[15]);
   final ColumnInterpreter<Long, Long, EmptyMsg, LongMsg, LongMsg> ci =
       new LongColumnInterpreter();
   long rowCount = -1;
   try {
     rowCount = aClient.rowCount(TEST_TABLE, ci, scan);
   } catch (Throwable e) {
     rowCount = 0;
   }
   assertEquals(0, rowCount);
 }
 @Test
 public void testAvgWithInvalidRange() {
   AggregationClient aClient = new AggregationClient(conf);
   Scan scan = new Scan();
   scan.addColumn(TEST_FAMILY, TEST_QUALIFIER);
   scan.setStartRow(ROWS[5]);
   scan.setStopRow(ROWS[1]);
   final ColumnInterpreter<Long, Long, EmptyMsg, LongMsg, LongMsg> ci =
       new LongColumnInterpreter();
   Double avg = null;
   try {
     avg = aClient.avg(TEST_TABLE, ci, scan);
   } catch (Throwable e) {
   }
   assertEquals(null, avg); // control should go to the catch block
 }
 @Test
 public void testStdWithValidRangeWithNullCF() {
   AggregationClient aClient = new AggregationClient(conf);
   Scan scan = new Scan();
   scan.setStartRow(ROWS[6]);
   scan.setStopRow(ROWS[17]);
   final ColumnInterpreter<Long, Long, EmptyMsg, LongMsg, LongMsg> ci =
       new LongColumnInterpreter();
   Double std = null;
   try {
     std = aClient.std(TEST_TABLE, ci, scan);
   } catch (Throwable e) {
   }
   assertEquals(null, std); // CP will throw an IOException about the
   // null column family, and max will be set to 0
 }
 @Test
 public void testStdWithInvalidRange() {
   AggregationClient aClient = new AggregationClient(conf);
   Scan scan = new Scan();
   scan.addFamily(TEST_FAMILY);
   scan.setStartRow(ROWS[6]);
   scan.setStopRow(ROWS[1]);
   final ColumnInterpreter<Long, Long, EmptyMsg, LongMsg, LongMsg> ci =
       new LongColumnInterpreter();
   Double std = null;
   try {
     std = aClient.std(TEST_TABLE, ci, scan);
   } catch (Throwable e) {
   }
   assertEquals(null, std); // control should go to the catch block
 }
    public static void updateExistingEventsToEndAtLastLocation(
        Table VTEvent_Table, long imo, VesselLocation lastlocation) throws IOException {
      // update existing events that started BEFORE the first new location and end after the first
      // to end as the last location

      // Find existing events that started BEFORE the first new location and end after the first
      Scan getEventStartedBeforeAndEndAfter = new Scan();
      ;
      getEventStartedBeforeAndEndAfter
          .setStartRow(
              Bytes.toBytes(
                  LpadNum(imo, 7)
                      + LpadNum(Long.MAX_VALUE - lastlocation.recordtime, 19)
                      + "0000000000"))
          .setStopRow(Bytes.toBytes(LpadNum(imo, 7) + LpadNum(Long.MAX_VALUE, 19) + "9999999999"))
          .addColumn(details, exittime);
      getEventStartedBeforeAndEndAfter.setCaching(100);

      Filter ExistTimeValuefilter =
          new ValueFilter(
              CompareFilter.CompareOp.GREATER,
              new BinaryComparator(
                  Bytes.toBytes(new DateTime(lastlocation.recordtime).toString(rawformatter))));
      getEventStartedBeforeAndEndAfter.setFilter(ExistTimeValuefilter);

      ResultScanner Result_eventcross = VTEvent_Table.getScanner(getEventStartedBeforeAndEndAfter);

      List<Put> puts = new ArrayList<Put>();
      for (Result res : Result_eventcross) {

        // vessel event table
        // rowkey: imo(7)+timestamp(19 desc)+polygonid(8)
        // qualifier:entrytime,entrycoordinates,exittime,exitcoordinates,destination
        byte[] rowkey = res.getRow();
        Put updateevent = new Put(rowkey);
        updateevent.addColumn(
            details,
            exittime,
            Bytes.toBytes(new DateTime(lastlocation.recordtime).toString(rawformatter)));
        updateevent.addColumn(details, coordinates, Bytes.toBytes(lastlocation.coordinates));
        updateevent.addColumn(details, destination, Bytes.toBytes(lastlocation.destination));
        puts.add(updateevent);
      }

      Result_eventcross.close();
      VTEvent_Table.put(puts);
    }
 @Test
 public void testMaxWithInvalidRange2() throws Throwable {
   long max = Long.MIN_VALUE;
   Scan scan = new Scan();
   scan.addColumn(TEST_FAMILY, TEST_QUALIFIER);
   scan.setStartRow(ROWS[4]);
   scan.setStopRow(ROWS[4]);
   try {
     AggregationClient aClient = new AggregationClient(conf);
     final ColumnInterpreter<Long, Long, EmptyMsg, LongMsg, LongMsg> ci =
         new LongColumnInterpreter();
     max = aClient.max(TEST_TABLE, ci, scan);
   } catch (Exception e) {
     max = 0;
   }
   assertEquals(0, max); // control should go to the catch block
 }
  @Override
  public int run(String[] strings) throws Exception {
    Configuration conf = new Configuration();
    // String inputFileName = "/cluster/gmm.seq";
    String outputFileName = "/cluster/matrix_intermediate_" + level + ".seq";

    int result;
    System.out.println("level:" + level);
    conf.set("level", level + "");
    String table = "ClusterDatabase";
    // String seqFileName = "/cluster/gmm.seq";

    Scan scan = new Scan();
    scan.setStartRow((level + "|").getBytes());
    scan.setStopRow(
        Bytes.add((level + "|").getBytes(), Bytes.toBytes("ffffffffffffffffffffffffffffffff")));
    scan.addColumn("Cluster".getBytes(), "GMM".getBytes());

    // try (FileSystem fileSystem = FileSystem.get(conf)) {
    FileSystem fileSystem = FileSystem.get(conf);
    Path outputpath = new Path(outputFileName);
    if (fileSystem.exists(outputpath)) {
      fileSystem.delete(outputpath, true);
    }

    Job job = new Job(conf, "Matrix Creation I From HBase");
    job.setJarByClass(MatrixCreationI.class);
    TableMapReduceUtil.initTableMapperJob(
        table, scan, MatrixMapper.class, IntWritable.class, Text.class, job);
    job.setReducerClass(MatrixReducer.class);
    job.setMapOutputKeyClass(IntWritable.class);
    job.setMapOutputValueClass(Text.class);
    job.setOutputKeyClass(Text.class);
    job.setOutputValueClass(Text.class);

    //            job.setInputFormatClass(TableInputFormat.class);
    // job.setOutputFormatClass(TextOutputFormat.class);
    job.setOutputFormatClass(SequenceFileOutputFormat.class);

    job.setNumReduceTasks(1);
    // FileInputFormat.addInputPath(job, new Path(inputFileName + "/part*"));
    FileOutputFormat.setOutputPath(job, outputpath);
    result = job.waitForCompletion(true) ? 0 : 1;
    // }
    return result;
  }
  @Override
  public Result getCurrentRowState(Mutation m, Collection<? extends ColumnReference> columns)
      throws IOException {
    byte[] row = m.getRow();
    // need to use a scan here so we can get raw state, which Get doesn't provide.
    Scan s = IndexManagementUtil.newLocalStateScan(Collections.singletonList(columns));
    s.setStartRow(row);
    s.setStopRow(row);
    HRegion region = this.env.getRegion();
    RegionScanner scanner = region.getScanner(s);
    List<KeyValue> kvs = new ArrayList<KeyValue>(1);
    boolean more = scanner.next(kvs);
    assert !more : "Got more than one result when scanning" + " a single row in the primary table!";

    Result r = new Result(kvs);
    scanner.close();
    return r;
  }
Example #29
0
 /**
  * Fully scan a given region, on a given server starting with given row.
  *
  * @param hRegionInterface region server
  * @param visitor visitor
  * @param regionName name of region
  * @param startrow start row
  * @throws IOException
  * @deprecated Does not retry; use fullScan xxx instead. x
  */
 public static void fullScan(
     HRegionInterface hRegionInterface, Visitor visitor, final byte[] regionName, byte[] startrow)
     throws IOException {
   if (hRegionInterface == null) return;
   Scan scan = new Scan();
   if (startrow != null) scan.setStartRow(startrow);
   scan.addFamily(HConstants.CATALOG_FAMILY);
   long scannerid = hRegionInterface.openScanner(regionName, scan);
   try {
     Result data;
     while ((data = hRegionInterface.next(scannerid)) != null) {
       if (!data.isEmpty()) visitor.visit(data);
     }
   } finally {
     hRegionInterface.close(scannerid);
   }
   return;
 }
  /**
   * This will test the row count with startrow > endrow. The result should be -1.
   *
   * @throws Throwable
   */
  @Test
  public void testRowCountWithInvalidRange1() {
    AggregationClient aClient = new AggregationClient(conf);
    Scan scan = new Scan();
    scan.addColumn(TEST_FAMILY, TEST_QUALIFIER);
    scan.setStartRow(ROWS[5]);
    scan.setStopRow(ROWS[2]);

    final ColumnInterpreter<Long, Long, EmptyMsg, LongMsg, LongMsg> ci =
        new LongColumnInterpreter();
    long rowCount = -1;
    try {
      rowCount = aClient.rowCount(TEST_TABLE, ci, scan);
    } catch (Throwable e) {
      myLog.error("Exception thrown in the invalidRange method" + e.getStackTrace());
    }
    assertEquals(-1, rowCount);
  }