public void testScanTimeRange() throws IOException {
String r1 = "R1";
// returns only 1 of these 2 even though same timestamp
KeyValue[] kvs =
new KeyValue[] {
KeyValueTestUtil.create(r1, CF_STR, "a", 1, KeyValue.Type.Put, "dont-care"),
KeyValueTestUtil.create(r1, CF_STR, "a", 2, KeyValue.Type.Put, "dont-care"),
KeyValueTestUtil.create(r1, CF_STR, "a", 3, KeyValue.Type.Put, "dont-care"),
KeyValueTestUtil.create(r1, CF_STR, "a", 4, KeyValue.Type.Put, "dont-care"),
KeyValueTestUtil.create(r1, CF_STR, "a", 5, KeyValue.Type.Put, "dont-care"),
};
List<KeyValueScanner> scanners =
Arrays.<KeyValueScanner>asList(
new KeyValueScanner[] {new KeyValueScanFixture(KeyValue.COMPARATOR, kvs)});
Scan scanSpec = new Scan(Bytes.toBytes(r1));
scanSpec.setTimeRange(0, 6);
scanSpec.setMaxVersions();
StoreScanner scan = new StoreScanner(scanSpec, scanInfo, scanType, getCols("a"), scanners);
List<KeyValue> results = new ArrayList<KeyValue>();
assertEquals(true, scan.next(results));
assertEquals(5, results.size());
assertEquals(kvs[kvs.length - 1], results.get(0));
// Scan limited TimeRange
scanSpec = new Scan(Bytes.toBytes(r1));
scanSpec.setTimeRange(1, 3);
scanSpec.setMaxVersions();
scan = new StoreScanner(scanSpec, scanInfo, scanType, getCols("a"), scanners);
results = new ArrayList<KeyValue>();
assertEquals(true, scan.next(results));
assertEquals(2, results.size());
// Another range.
scanSpec = new Scan(Bytes.toBytes(r1));
scanSpec.setTimeRange(5, 10);
scanSpec.setMaxVersions();
scan = new StoreScanner(scanSpec, scanInfo, scanType, getCols("a"), scanners);
results = new ArrayList<KeyValue>();
assertEquals(true, scan.next(results));
assertEquals(1, results.size());
// See how TimeRange and Versions interact.
// Another range.
scanSpec = new Scan(Bytes.toBytes(r1));
scanSpec.setTimeRange(0, 10);
scanSpec.setMaxVersions(3);
scan = new StoreScanner(scanSpec, scanInfo, scanType, getCols("a"), scanners);
results = new ArrayList<KeyValue>();
assertEquals(true, scan.next(results));
assertEquals(3, results.size());
}
public static void setTimeRange(Scan scan, long ts) {
try {
scan.setTimeRange(MetaDataProtocol.MIN_TABLE_TIMESTAMP, ts);
} catch (IOException e) {
throw new RuntimeException(e);
}
}
public static void setTimeRange(Scan scan, TimeRange range) {
try {
scan.setTimeRange(range.getMin(), range.getMax());
} catch (IOException e) {
throw new RuntimeException(e);
}
}
@TimeDepend
@Test
public void testScan_ts_same() throws Exception {
recreateTable();
Date ts = parse("2000-01-01", "yyyy-MM-dd");
Put put = new Put(rowKey_ForTest);
put.add(ColumnFamilyNameBytes, QName1, ts.getTime(), Bytes.toBytes("a"));
table.put(put);
Set<String> resultRowKeys = new HashSet<String>();
Scan scan = new Scan(rowKey_ForTest, rowKey_ForTest);
scan.setTimeRange(ts.getTime(), ts.getTime());
ResultScanner resultScanner = table.getScanner(scan);
for (Result result = resultScanner.next(); result != null; result = resultScanner.next()) {
resultRowKeys.add(Bytes.toString(result.getRow()));
}
close(resultScanner);
Assert.assertTrue(resultRowKeys.size() == 0);
recreateTable();
}
/**
* Gets an hbase scanner.
*
* @return An hbase scanner.
* @throws IOException Error accessing hbase.
*/
@Override
protected Scan getScanner() throws IOException {
Scan scan = super.getScanner();
if (this.query != null) {
if (this.query.getStartKey() != null) {
scan.setStartRow(this.query.getStartKey());
}
if (this.query.getEndKey() != null) {
scan.setStopRow(padWithMaxUnicode(this.query.getEndKey()));
}
if (this.query.getStartDate() != null && this.query.getEndDate() != null) {
scan.setTimeRange(this.query.getStartDate().getTime(), this.query.getEndDate().getTime());
}
if (this.query.getWord() != null || this.query.getOperator().isUnary()) {
WritableByteArrayComparable comparator;
switch (this.query.getOperator()) {
case Contains:
comparator = new SubstringComparator(this.query.getWord());
break;
case StartsWith:
comparator = new RegexStringComparator(String.format("^%s.*", this.query.getWord()));
break;
case EndsWith:
comparator = new RegexStringComparator(String.format(".*%s$", this.query.getWord()));
break;
case Less:
case LessOrEqual:
case Equal:
case NotEqual:
case GreaterOrEqual:
case Greater:
comparator = new BinaryComparator(this.query.getWordAsByteArray());
break;
case IsNull:
case IsNotNull:
comparator = new BinaryComparator(EMPTY_BYTES_ARRAY);
break;
default:
throw new IllegalArgumentException(
String.format(
"The specified operator type '%s' is not supported.",
this.query.getOperator()));
}
scan.setFilter(
new SingleColumnValueFilter(
Bytes.toBytesBinary(this.query.getFamily()),
Bytes.toBytesBinary(this.query.getColumn()),
this.query.getOperator().toFilter(),
comparator));
}
}
return scan;
}
/**
* Transactional version of {@link HTable#getScanner(Scan)}
*
* @param transactionState Identifier of the transaction
* @see HTable#getScanner(Scan)
* @throws IOException
*/
public ResultScanner getScanner(TransactionState transactionState, Scan scan) throws IOException {
Scan tsscan = new Scan(scan);
// tsscan.setRetainDeletesInOutput(true);
// int maxVersions = scan.getMaxVersions();
tsscan.setMaxVersions((int) (versionsAvg + CACHE_VERSIONS_OVERHEAD));
tsscan.setTimeRange(0, transactionState.getStartTimestamp() + 1);
ClientScanner scanner =
new ClientScanner(transactionState, tsscan, (int) (versionsAvg + CACHE_VERSIONS_OVERHEAD));
scanner.initialize();
return scanner;
}
public List<RowLogMessage> next(String subscription, Long minimalTimestamp, boolean problematic)
throws RowLogException {
byte[] rowPrefix;
byte[] subscriptionBytes = Bytes.toBytes(subscription);
if (problematic) {
rowPrefix = PROBLEMATIC_MARKER;
rowPrefix = Bytes.add(rowPrefix, subscriptionBytes);
} else {
rowPrefix = subscriptionBytes;
}
byte[] startRow = rowPrefix;
if (minimalTimestamp != null) startRow = Bytes.add(startRow, Bytes.toBytes(minimalTimestamp));
try {
List<RowLogMessage> rowLogMessages = new ArrayList<RowLogMessage>();
Scan scan = new Scan(startRow);
if (minimalTimestamp != null) scan.setTimeRange(minimalTimestamp, Long.MAX_VALUE);
scan.addColumn(MESSAGES_CF, MESSAGE_COLUMN);
ResultScanner scanner = table.getScanner(scan);
boolean keepScanning = problematic;
do {
Result[] results = scanner.next(batchSize);
if (results.length == 0) {
keepScanning = false;
}
for (Result next : results) {
byte[] rowKey = next.getRow();
if (!Bytes.startsWith(rowKey, rowPrefix)) {
keepScanning = false;
break; // There were no messages for this subscription
}
if (problematic) {
rowKey = Bytes.tail(rowKey, rowKey.length - PROBLEMATIC_MARKER.length);
}
byte[] value = next.getValue(MESSAGES_CF, MESSAGE_COLUMN);
byte[] messageId = Bytes.tail(rowKey, rowKey.length - subscriptionBytes.length);
rowLogMessages.add(decodeMessage(messageId, value));
}
} while (keepScanning);
// The scanner is not closed in a finally block, since when we get an IOException from
// HBase, it is likely that closing the scanner will give problems too. Not closing
// the scanner is not fatal since HBase will expire it after a while.
Closer.close(scanner);
return rowLogMessages;
} catch (IOException e) {
throw new RowLogException("Failed to fetch next message from RowLogShard", e);
}
}
/*
* Must be sure that column is in format like 'family: column'
*/
public void prepare(String[] columns) throws IOException {
this.scan = new Scan();
if (this.startKey != null) {
logger.info(String.format("HBaseReader set startkey to %s .", Bytes.toString(this.startKey)));
scan.setStartRow(startKey);
}
if (this.endKey != null) {
logger.info(String.format("HBaseReader set endkey to %s .", Bytes.toString(this.endKey)));
scan.setStopRow(endKey);
}
if (this.minStamp > 0 && this.maxStamp > 0) {
scan.setTimeRange(minStamp, maxStamp);
if (StringUtils.isNotEmpty(primaryKey)) {
String[] cs = StringUtils.split(primaryKey, ":");
scan.addColumn(cs[0].getBytes(), cs[1].getBytes());
}
} else if (ETLStringUtils.isNotEmpty(startDate)
&& ETLStringUtils.isNotEmpty(familyName)
&& ETLStringUtils.isNotEmpty(fieldName)) {
SingleColumnValueFilter scvf =
new SingleColumnValueFilter(
Bytes.toBytes(familyName),
Bytes.toBytes(fieldName),
CompareOp.GREATER_OR_EQUAL,
new BinaryPrefixComparator(startDate.getBytes()));
scvf.setFilterIfMissing(true);
scvf.setLatestVersionOnly(true);
if (ETLStringUtils.isNotEmpty(endDate)) {
SingleColumnValueFilter scvf1 =
new SingleColumnValueFilter(
Bytes.toBytes(familyName),
Bytes.toBytes(fieldName),
CompareOp.LESS_OR_EQUAL,
new BinaryPrefixComparator(endDate.getBytes()));
scvf1.setFilterIfMissing(true);
scvf1.setLatestVersionOnly(true);
List<Filter> filters = new ArrayList<Filter>();
filters.add(scvf);
filters.add(scvf1);
FilterList fl = new FilterList(FilterList.Operator.MUST_PASS_ALL, filters);
scan.setFilter(fl);
} else {
scan.setFilter(scvf);
}
}
this.families = new String[columns.length];
this.columns = new String[columns.length];
int idx = 0;
for (String column : columns) {
this.families[idx] = column.split(":")[0].trim();
this.columns[idx] = column.split(":")[1].trim();
if (!(this.minStamp > 0 && this.maxStamp > 0)) {
scan.addColumn(this.families[idx].getBytes(), this.columns[idx].getBytes());
}
idx++;
}
if ("true".equalsIgnoreCase(this.isSaveOneObj)) {
scan.addColumn(this.oneObjFamilyName.getBytes(), this.oneObjColName.getBytes());
}
if (this.maxversion > 0) {
scan.setMaxVersions(this.maxversion);
} else {
scan.setMaxVersions();
}
htable.setScannerCaching(SCAN_CACHE);
this.rs = htable.getScanner(this.scan);
}