예제 #1
0
 public void testScanTimeRange() throws IOException {
   String r1 = "R1";
   // returns only 1 of these 2 even though same timestamp
   KeyValue[] kvs =
       new KeyValue[] {
         KeyValueTestUtil.create(r1, CF_STR, "a", 1, KeyValue.Type.Put, "dont-care"),
         KeyValueTestUtil.create(r1, CF_STR, "a", 2, KeyValue.Type.Put, "dont-care"),
         KeyValueTestUtil.create(r1, CF_STR, "a", 3, KeyValue.Type.Put, "dont-care"),
         KeyValueTestUtil.create(r1, CF_STR, "a", 4, KeyValue.Type.Put, "dont-care"),
         KeyValueTestUtil.create(r1, CF_STR, "a", 5, KeyValue.Type.Put, "dont-care"),
       };
   List<KeyValueScanner> scanners =
       Arrays.<KeyValueScanner>asList(
           new KeyValueScanner[] {new KeyValueScanFixture(KeyValue.COMPARATOR, kvs)});
   Scan scanSpec = new Scan(Bytes.toBytes(r1));
   scanSpec.setTimeRange(0, 6);
   scanSpec.setMaxVersions();
   StoreScanner scan = new StoreScanner(scanSpec, scanInfo, scanType, getCols("a"), scanners);
   List<KeyValue> results = new ArrayList<KeyValue>();
   assertEquals(true, scan.next(results));
   assertEquals(5, results.size());
   assertEquals(kvs[kvs.length - 1], results.get(0));
   // Scan limited TimeRange
   scanSpec = new Scan(Bytes.toBytes(r1));
   scanSpec.setTimeRange(1, 3);
   scanSpec.setMaxVersions();
   scan = new StoreScanner(scanSpec, scanInfo, scanType, getCols("a"), scanners);
   results = new ArrayList<KeyValue>();
   assertEquals(true, scan.next(results));
   assertEquals(2, results.size());
   // Another range.
   scanSpec = new Scan(Bytes.toBytes(r1));
   scanSpec.setTimeRange(5, 10);
   scanSpec.setMaxVersions();
   scan = new StoreScanner(scanSpec, scanInfo, scanType, getCols("a"), scanners);
   results = new ArrayList<KeyValue>();
   assertEquals(true, scan.next(results));
   assertEquals(1, results.size());
   // See how TimeRange and Versions interact.
   // Another range.
   scanSpec = new Scan(Bytes.toBytes(r1));
   scanSpec.setTimeRange(0, 10);
   scanSpec.setMaxVersions(3);
   scan = new StoreScanner(scanSpec, scanInfo, scanType, getCols("a"), scanners);
   results = new ArrayList<KeyValue>();
   assertEquals(true, scan.next(results));
   assertEquals(3, results.size());
 }
예제 #2
0
 public static void setTimeRange(Scan scan, long ts) {
   try {
     scan.setTimeRange(MetaDataProtocol.MIN_TABLE_TIMESTAMP, ts);
   } catch (IOException e) {
     throw new RuntimeException(e);
   }
 }
예제 #3
0
 public static void setTimeRange(Scan scan, TimeRange range) {
   try {
     scan.setTimeRange(range.getMin(), range.getMax());
   } catch (IOException e) {
     throw new RuntimeException(e);
   }
 }
예제 #4
0
  @TimeDepend
  @Test
  public void testScan_ts_same() throws Exception {

    recreateTable();

    Date ts = parse("2000-01-01", "yyyy-MM-dd");

    Put put = new Put(rowKey_ForTest);

    put.add(ColumnFamilyNameBytes, QName1, ts.getTime(), Bytes.toBytes("a"));

    table.put(put);

    Set<String> resultRowKeys = new HashSet<String>();
    Scan scan = new Scan(rowKey_ForTest, rowKey_ForTest);
    scan.setTimeRange(ts.getTime(), ts.getTime());

    ResultScanner resultScanner = table.getScanner(scan);
    for (Result result = resultScanner.next(); result != null; result = resultScanner.next()) {
      resultRowKeys.add(Bytes.toString(result.getRow()));
    }

    close(resultScanner);

    Assert.assertTrue(resultRowKeys.size() == 0);

    recreateTable();
  }
예제 #5
0
  /**
   * Gets an hbase scanner.
   *
   * @return An hbase scanner.
   * @throws IOException Error accessing hbase.
   */
  @Override
  protected Scan getScanner() throws IOException {
    Scan scan = super.getScanner();

    if (this.query != null) {
      if (this.query.getStartKey() != null) {
        scan.setStartRow(this.query.getStartKey());
      }

      if (this.query.getEndKey() != null) {
        scan.setStopRow(padWithMaxUnicode(this.query.getEndKey()));
      }

      if (this.query.getStartDate() != null && this.query.getEndDate() != null) {
        scan.setTimeRange(this.query.getStartDate().getTime(), this.query.getEndDate().getTime());
      }

      if (this.query.getWord() != null || this.query.getOperator().isUnary()) {
        WritableByteArrayComparable comparator;

        switch (this.query.getOperator()) {
          case Contains:
            comparator = new SubstringComparator(this.query.getWord());
            break;
          case StartsWith:
            comparator = new RegexStringComparator(String.format("^%s.*", this.query.getWord()));
            break;
          case EndsWith:
            comparator = new RegexStringComparator(String.format(".*%s$", this.query.getWord()));
            break;
          case Less:
          case LessOrEqual:
          case Equal:
          case NotEqual:
          case GreaterOrEqual:
          case Greater:
            comparator = new BinaryComparator(this.query.getWordAsByteArray());
            break;
          case IsNull:
          case IsNotNull:
            comparator = new BinaryComparator(EMPTY_BYTES_ARRAY);
            break;
          default:
            throw new IllegalArgumentException(
                String.format(
                    "The specified operator type '%s' is not supported.",
                    this.query.getOperator()));
        }

        scan.setFilter(
            new SingleColumnValueFilter(
                Bytes.toBytesBinary(this.query.getFamily()),
                Bytes.toBytesBinary(this.query.getColumn()),
                this.query.getOperator().toFilter(),
                comparator));
      }
    }

    return scan;
  }
예제 #6
0
 /**
  * Transactional version of {@link HTable#getScanner(Scan)}
  *
  * @param transactionState Identifier of the transaction
  * @see HTable#getScanner(Scan)
  * @throws IOException
  */
 public ResultScanner getScanner(TransactionState transactionState, Scan scan) throws IOException {
   Scan tsscan = new Scan(scan);
   //      tsscan.setRetainDeletesInOutput(true);
   //      int maxVersions = scan.getMaxVersions();
   tsscan.setMaxVersions((int) (versionsAvg + CACHE_VERSIONS_OVERHEAD));
   tsscan.setTimeRange(0, transactionState.getStartTimestamp() + 1);
   ClientScanner scanner =
       new ClientScanner(transactionState, tsscan, (int) (versionsAvg + CACHE_VERSIONS_OVERHEAD));
   scanner.initialize();
   return scanner;
 }
예제 #7
0
  public List<RowLogMessage> next(String subscription, Long minimalTimestamp, boolean problematic)
      throws RowLogException {
    byte[] rowPrefix;
    byte[] subscriptionBytes = Bytes.toBytes(subscription);
    if (problematic) {
      rowPrefix = PROBLEMATIC_MARKER;
      rowPrefix = Bytes.add(rowPrefix, subscriptionBytes);
    } else {
      rowPrefix = subscriptionBytes;
    }
    byte[] startRow = rowPrefix;
    if (minimalTimestamp != null) startRow = Bytes.add(startRow, Bytes.toBytes(minimalTimestamp));
    try {
      List<RowLogMessage> rowLogMessages = new ArrayList<RowLogMessage>();
      Scan scan = new Scan(startRow);
      if (minimalTimestamp != null) scan.setTimeRange(minimalTimestamp, Long.MAX_VALUE);
      scan.addColumn(MESSAGES_CF, MESSAGE_COLUMN);
      ResultScanner scanner = table.getScanner(scan);
      boolean keepScanning = problematic;
      do {
        Result[] results = scanner.next(batchSize);
        if (results.length == 0) {
          keepScanning = false;
        }
        for (Result next : results) {
          byte[] rowKey = next.getRow();
          if (!Bytes.startsWith(rowKey, rowPrefix)) {
            keepScanning = false;
            break; // There were no messages for this subscription
          }
          if (problematic) {
            rowKey = Bytes.tail(rowKey, rowKey.length - PROBLEMATIC_MARKER.length);
          }
          byte[] value = next.getValue(MESSAGES_CF, MESSAGE_COLUMN);
          byte[] messageId = Bytes.tail(rowKey, rowKey.length - subscriptionBytes.length);
          rowLogMessages.add(decodeMessage(messageId, value));
        }
      } while (keepScanning);

      // The scanner is not closed in a finally block, since when we get an IOException from
      // HBase, it is likely that closing the scanner will give problems too. Not closing
      // the scanner is not fatal since HBase will expire it after a while.
      Closer.close(scanner);

      return rowLogMessages;
    } catch (IOException e) {
      throw new RowLogException("Failed to fetch next message from RowLogShard", e);
    }
  }
예제 #8
0
  /*
   * Must be sure that column is in format like 'family: column'
   */
  public void prepare(String[] columns) throws IOException {
    this.scan = new Scan();
    if (this.startKey != null) {
      logger.info(String.format("HBaseReader set startkey to %s .", Bytes.toString(this.startKey)));
      scan.setStartRow(startKey);
    }
    if (this.endKey != null) {
      logger.info(String.format("HBaseReader set endkey to %s .", Bytes.toString(this.endKey)));
      scan.setStopRow(endKey);
    }
    if (this.minStamp > 0 && this.maxStamp > 0) {
      scan.setTimeRange(minStamp, maxStamp);
      if (StringUtils.isNotEmpty(primaryKey)) {
        String[] cs = StringUtils.split(primaryKey, ":");
        scan.addColumn(cs[0].getBytes(), cs[1].getBytes());
      }
    } else if (ETLStringUtils.isNotEmpty(startDate)
        && ETLStringUtils.isNotEmpty(familyName)
        && ETLStringUtils.isNotEmpty(fieldName)) {
      SingleColumnValueFilter scvf =
          new SingleColumnValueFilter(
              Bytes.toBytes(familyName),
              Bytes.toBytes(fieldName),
              CompareOp.GREATER_OR_EQUAL,
              new BinaryPrefixComparator(startDate.getBytes()));
      scvf.setFilterIfMissing(true);
      scvf.setLatestVersionOnly(true);
      if (ETLStringUtils.isNotEmpty(endDate)) {
        SingleColumnValueFilter scvf1 =
            new SingleColumnValueFilter(
                Bytes.toBytes(familyName),
                Bytes.toBytes(fieldName),
                CompareOp.LESS_OR_EQUAL,
                new BinaryPrefixComparator(endDate.getBytes()));
        scvf1.setFilterIfMissing(true);
        scvf1.setLatestVersionOnly(true);
        List<Filter> filters = new ArrayList<Filter>();
        filters.add(scvf);
        filters.add(scvf1);
        FilterList fl = new FilterList(FilterList.Operator.MUST_PASS_ALL, filters);
        scan.setFilter(fl);
      } else {
        scan.setFilter(scvf);
      }
    }
    this.families = new String[columns.length];
    this.columns = new String[columns.length];

    int idx = 0;
    for (String column : columns) {
      this.families[idx] = column.split(":")[0].trim();
      this.columns[idx] = column.split(":")[1].trim();
      if (!(this.minStamp > 0 && this.maxStamp > 0)) {
        scan.addColumn(this.families[idx].getBytes(), this.columns[idx].getBytes());
      }
      idx++;
    }
    if ("true".equalsIgnoreCase(this.isSaveOneObj)) {
      scan.addColumn(this.oneObjFamilyName.getBytes(), this.oneObjColName.getBytes());
    }
    if (this.maxversion > 0) {
      scan.setMaxVersions(this.maxversion);
    } else {
      scan.setMaxVersions();
    }

    htable.setScannerCaching(SCAN_CACHE);
    this.rs = htable.getScanner(this.scan);
  }