private void wait(byte[] row, HTable target, boolean isDeleted) throws Exception {
Get get = new Get(row);
for (int i = 0; i < NB_RETRIES; i++) {
if (i == NB_RETRIES - 1) {
fail(
"Waited too much time for replication. Row:"
+ Bytes.toString(row)
+ ". IsDeleteReplication:"
+ isDeleted);
}
Result res = target.get(get);
boolean sleep = isDeleted ? res.size() > 0 : res.size() == 0;
if (sleep) {
LOG.info(
"Waiting for more time for replication. Row:"
+ Bytes.toString(row)
+ ". IsDeleteReplication:"
+ isDeleted);
Thread.sleep(SLEEP_TIME);
} else {
if (!isDeleted) {
assertArrayEquals(res.value(), row);
}
LOG.info("Obtained row:" + Bytes.toString(row) + ". IsDeleteReplication:" + isDeleted);
break;
}
}
}
/**
* Looks at every value of the mapreduce output and verifies that indeed the values have been
* reversed.
*
* @param table Table to scan.
* @throws IOException
* @throws NullPointerException if we failed to find a cell value
*/
private void verifyAttempt(final Table table) throws IOException, NullPointerException {
Scan scan = new Scan();
scan.addFamily(INPUT_FAMILY);
scan.addFamily(OUTPUT_FAMILY);
ResultScanner scanner = table.getScanner(scan);
try {
Iterator<Result> itr = scanner.iterator();
assertTrue(itr.hasNext());
while (itr.hasNext()) {
Result r = itr.next();
if (LOG.isDebugEnabled()) {
if (r.size() > 2) {
throw new IOException("Too many results, expected 2 got " + r.size());
}
}
byte[] firstValue = null;
byte[] secondValue = null;
int count = 0;
for (Cell kv : r.listCells()) {
if (count == 0) {
firstValue = CellUtil.cloneValue(kv);
} else if (count == 1) {
secondValue = CellUtil.cloneValue(kv);
} else if (count == 2) {
break;
}
count++;
}
String first = "";
if (firstValue == null) {
throw new NullPointerException(Bytes.toString(r.getRow()) + ": first value is null");
}
first = Bytes.toString(firstValue);
String second = "";
if (secondValue == null) {
throw new NullPointerException(Bytes.toString(r.getRow()) + ": second value is null");
}
byte[] secondReversed = new byte[secondValue.length];
for (int i = 0, j = secondValue.length - 1; j >= 0; j--, i++) {
secondReversed[i] = secondValue[j];
}
second = Bytes.toString(secondReversed);
if (first.compareTo(second) != 0) {
if (LOG.isDebugEnabled()) {
LOG.debug(
"second key is not the reverse of first. row="
+ Bytes.toStringBinary(r.getRow())
+ ", first value="
+ first
+ ", second value="
+ second);
}
fail();
}
}
} finally {
scanner.close();
}
}
@Test
public void testStartStopRow() throws Exception {
final TableName TABLENAME1 = TableName.valueOf("testStartStopRow1");
final TableName TABLENAME2 = TableName.valueOf("testStartStopRow2");
final byte[] FAMILY = Bytes.toBytes("family");
final byte[] COLUMN1 = Bytes.toBytes("c1");
final byte[] ROW0 = Bytes.toBytesBinary("\\x01row0");
final byte[] ROW1 = Bytes.toBytesBinary("\\x01row1");
final byte[] ROW2 = Bytes.toBytesBinary("\\x01row2");
Table t1 = TEST_UTIL.createTable(TABLENAME1, FAMILY);
Table t2 = TEST_UTIL.createTable(TABLENAME2, FAMILY);
// put rows into the first table
Put p = new Put(ROW0);
p.addColumn(FAMILY, COLUMN1, COLUMN1);
t1.put(p);
p = new Put(ROW1);
p.addColumn(FAMILY, COLUMN1, COLUMN1);
t1.put(p);
p = new Put(ROW2);
p.addColumn(FAMILY, COLUMN1, COLUMN1);
t1.put(p);
CopyTable copy = new CopyTable();
assertEquals(
0,
ToolRunner.run(
new Configuration(TEST_UTIL.getConfiguration()),
copy,
new String[] {
"--new.name=" + TABLENAME2,
"--startrow=\\x01row1",
"--stoprow=\\x01row2",
TABLENAME1.getNameAsString()
}));
// verify the data was copied into table 2
// row1 exist, row0, row2 do not exist
Get g = new Get(ROW1);
Result r = t2.get(g);
assertEquals(1, r.size());
assertTrue(CellUtil.matchingQualifier(r.rawCells()[0], COLUMN1));
g = new Get(ROW0);
r = t2.get(g);
assertEquals(0, r.size());
g = new Get(ROW2);
r = t2.get(g);
assertEquals(0, r.size());
t1.close();
t2.close();
TEST_UTIL.deleteTable(TABLENAME1);
TEST_UTIL.deleteTable(TABLENAME2);
}
@Test
public void testTTL() throws Exception {
TableName tableName = TableName.valueOf("testTTL");
if (TEST_UTIL.getHBaseAdmin().tableExists(tableName)) {
TEST_UTIL.deleteTable(tableName);
}
HTableDescriptor desc = new HTableDescriptor(tableName);
HColumnDescriptor hcd = new HColumnDescriptor(F).setMaxVersions(10).setTimeToLive(1);
desc.addFamily(hcd);
TEST_UTIL.getHBaseAdmin().createTable(desc);
Table t = new HTable(new Configuration(TEST_UTIL.getConfiguration()), tableName);
long now = EnvironmentEdgeManager.currentTime();
ManualEnvironmentEdge me = new ManualEnvironmentEdge();
me.setValue(now);
EnvironmentEdgeManagerTestHelper.injectEdge(me);
// 2s in the past
long ts = now - 2000;
// Set the TTL override to 3s
Put p = new Put(R);
p.setAttribute("ttl", new byte[] {});
p.add(F, tableName.getName(), Bytes.toBytes(3000L));
t.put(p);
p = new Put(R);
p.add(F, Q, ts, Q);
t.put(p);
p = new Put(R);
p.add(F, Q, ts + 1, Q);
t.put(p);
// these two should be expired but for the override
// (their ts was 2s in the past)
Get g = new Get(R);
g.setMaxVersions(10);
Result r = t.get(g);
// still there?
assertEquals(2, r.size());
TEST_UTIL.flush(tableName);
TEST_UTIL.compact(tableName, true);
g = new Get(R);
g.setMaxVersions(10);
r = t.get(g);
// still there?
assertEquals(2, r.size());
// roll time forward 2s.
me.setValue(now + 2000);
// now verify that data eventually does expire
g = new Get(R);
g.setMaxVersions(10);
r = t.get(g);
// should be gone now
assertEquals(0, r.size());
t.close();
}
/**
* Confirm ImportTsv via data in online table.
*
* @param dataAvailable
*/
private static void validateTable(
Configuration conf,
TableName tableName,
String family,
int valueMultiplier,
boolean dataAvailable)
throws IOException {
LOG.debug("Validating table.");
Connection connection = ConnectionFactory.createConnection(conf);
Table table = connection.getTable(tableName);
boolean verified = false;
long pause = conf.getLong("hbase.client.pause", 5 * 1000);
int numRetries = conf.getInt(HConstants.HBASE_CLIENT_RETRIES_NUMBER, 5);
for (int i = 0; i < numRetries; i++) {
try {
Scan scan = new Scan();
// Scan entire family.
scan.addFamily(Bytes.toBytes(family));
if (dataAvailable) {
ResultScanner resScanner = table.getScanner(scan);
for (Result res : resScanner) {
LOG.debug("Getting results " + res.size());
assertTrue(res.size() == 2);
List<Cell> kvs = res.listCells();
assertTrue(CellUtil.matchingRow(kvs.get(0), Bytes.toBytes("KEY")));
assertTrue(CellUtil.matchingRow(kvs.get(1), Bytes.toBytes("KEY")));
assertTrue(
CellUtil.matchingValue(kvs.get(0), Bytes.toBytes("VALUE" + valueMultiplier)));
assertTrue(
CellUtil.matchingValue(kvs.get(1), Bytes.toBytes("VALUE" + 2 * valueMultiplier)));
// Only one result set is expected, so let it loop.
verified = true;
}
} else {
ResultScanner resScanner = table.getScanner(scan);
Result[] next = resScanner.next(2);
assertEquals(0, next.length);
verified = true;
}
break;
} catch (NullPointerException e) {
// If here, a cell was empty. Presume its because updates came in
// after the scanner had been opened. Wait a while and retry.
}
try {
Thread.sleep(pause);
} catch (InterruptedException e) {
// continue
}
}
table.close();
connection.close();
assertTrue(verified);
}
@Test
public void testIncrement() throws Exception {
byte[] row1 = Bytes.toBytes("row1");
byte[] col1 = Bytes.toBytes("col1");
byte[] col2 = Bytes.toBytes("col2");
byte[] col3 = Bytes.toBytes("col3");
// Setting up region
final WALFactory wals = new WALFactory(CONF, null, "TestIncrement");
byte[] tableName = Bytes.toBytes("TestIncrement");
final WAL wal = wals.getWAL(tableName);
HRegion region = createHRegion(tableName, "increment", wal, Durability.USE_DEFAULT);
// col1: amount = 1, 1 write back to WAL
Increment inc1 = new Increment(row1);
inc1.addColumn(FAMILY, col1, 1);
Result res = region.increment(inc1);
assertEquals(1, res.size());
assertEquals(1, Bytes.toLong(res.getValue(FAMILY, col1)));
verifyWALCount(wals, wal, 1);
// col1: amount = 0, 0 write back to WAL
inc1 = new Increment(row1);
inc1.addColumn(FAMILY, col1, 0);
res = region.increment(inc1);
assertEquals(1, res.size());
assertEquals(1, Bytes.toLong(res.getValue(FAMILY, col1)));
verifyWALCount(wals, wal, 1);
// col1: amount = 0, col2: amount = 0, col3: amount = 0
// 0 write back to WAL
inc1 = new Increment(row1);
inc1.addColumn(FAMILY, col1, 0);
inc1.addColumn(FAMILY, col2, 0);
inc1.addColumn(FAMILY, col3, 0);
res = region.increment(inc1);
assertEquals(3, res.size());
assertEquals(1, Bytes.toLong(res.getValue(FAMILY, col1)));
assertEquals(0, Bytes.toLong(res.getValue(FAMILY, col2)));
assertEquals(0, Bytes.toLong(res.getValue(FAMILY, col3)));
verifyWALCount(wals, wal, 1);
// col1: amount = 5, col2: amount = 4, col3: amount = 3
// 1 write back to WAL
inc1 = new Increment(row1);
inc1.addColumn(FAMILY, col1, 5);
inc1.addColumn(FAMILY, col2, 4);
inc1.addColumn(FAMILY, col3, 3);
res = region.increment(inc1);
assertEquals(3, res.size());
assertEquals(6, Bytes.toLong(res.getValue(FAMILY, col1)));
assertEquals(4, Bytes.toLong(res.getValue(FAMILY, col2)));
assertEquals(3, Bytes.toLong(res.getValue(FAMILY, col3)));
verifyWALCount(wals, wal, 2);
}
@Test
public void testBaseCases() throws Exception {
TableName tableName = TableName.valueOf("baseCases");
if (TEST_UTIL.getHBaseAdmin().tableExists(tableName)) {
TEST_UTIL.deleteTable(tableName);
}
Table t = TEST_UTIL.createTable(tableName, F, 1);
// set the version override to 2
Put p = new Put(R);
p.setAttribute("versions", new byte[] {});
p.add(F, tableName.getName(), Bytes.toBytes(2));
t.put(p);
long now = EnvironmentEdgeManager.currentTime();
// insert 2 versions
p = new Put(R);
p.add(F, Q, now, Q);
t.put(p);
p = new Put(R);
p.add(F, Q, now + 1, Q);
t.put(p);
Get g = new Get(R);
g.setMaxVersions(10);
Result r = t.get(g);
assertEquals(2, r.size());
TEST_UTIL.flush(tableName);
TEST_UTIL.compact(tableName, true);
// both version are still visible even after a flush/compaction
g = new Get(R);
g.setMaxVersions(10);
r = t.get(g);
assertEquals(2, r.size());
// insert a 3rd version
p = new Put(R);
p.add(F, Q, now + 2, Q);
t.put(p);
g = new Get(R);
g.setMaxVersions(10);
r = t.get(g);
// still only two version visible
assertEquals(2, r.size());
t.close();
}
private void putAndWait(byte[] row, byte[] fam, HTable source, HTable... targets)
throws Exception {
Put put = new Put(row);
put.add(fam, row, row);
source.put(put);
Get get = new Get(row);
for (int i = 0; i < NB_RETRIES; i++) {
if (i == NB_RETRIES - 1) {
fail("Waited too much time for put replication");
}
boolean replicatedToAll = true;
for (HTable target : targets) {
Result res = target.get(get);
if (res.size() == 0) {
LOG.info("Row not available");
replicatedToAll = false;
break;
} else {
assertArrayEquals(res.value(), row);
}
}
if (replicatedToAll) {
break;
} else {
Thread.sleep(SLEEP_TIME);
}
}
}
private void checkRow(byte[] row, int count, HTable... tables) throws IOException {
Get get = new Get(row);
for (HTable table : tables) {
Result res = table.get(get);
assertEquals(count, res.size());
}
}
private void check(byte[] row, byte[] fam, HTable t) throws IOException {
Get get = new Get(row);
Result res = t.get(get);
if (res.size() == 0) {
fail("Row is missing");
}
}
@Test(timeout = 300000)
public void testReplicationWithCellTags() throws Exception {
LOG.info("testSimplePutDelete");
Put put = new Put(ROW);
put.setAttribute("visibility", Bytes.toBytes("myTag3"));
put.add(FAMILY, ROW, ROW);
htable1 = utility1.getConnection().getTable(TABLE_NAME);
htable1.put(put);
Get get = new Get(ROW);
try {
for (int i = 0; i < NB_RETRIES; i++) {
if (i == NB_RETRIES - 1) {
fail("Waited too much time for put replication");
}
Result res = htable2.get(get);
if (res.size() == 0) {
LOG.info("Row not available");
Thread.sleep(SLEEP_TIME);
} else {
assertArrayEquals(res.value(), ROW);
assertEquals(1, TestCoprocessorForTagsAtSink.tags.size());
Tag tag = TestCoprocessorForTagsAtSink.tags.get(0);
assertEquals(TAG_TYPE, tag.getType());
break;
}
}
} finally {
TestCoprocessorForTagsAtSink.tags = null;
}
}
public static String get(final MockHTable table, final byte[] key) throws IOException {
Get get = new Get(key);
Result result = null;
result = table.get(get);
Assert.assertEquals(1, result.size());
return Bytes.toString(result.getValue(SYNC_COLUMN_FAMILY, SYNC_COLUMN_QUALIFIER));
}
private void deleteAndWait(byte[] row, HTable source, HTable... targets) throws Exception {
Delete del = new Delete(row);
source.delete(del);
Get get = new Get(row);
for (int i = 0; i < NB_RETRIES; i++) {
if (i == NB_RETRIES - 1) {
fail("Waited too much time for del replication");
}
boolean removedFromAll = true;
for (HTable target : targets) {
Result res = target.get(get);
if (res.size() >= 1) {
LOG.info("Row not deleted");
removedFromAll = false;
break;
}
}
if (removedFromAll) {
break;
} else {
Thread.sleep(SLEEP_TIME);
}
}
}
public static void testGetAfterPut3(int numtx) throws IOException, CommitUnsuccessfulException {
int newValue = 0;
for (newValue = 0; newValue < numtx; newValue++) {
// Begin
TransactionState transactionState = transactionManager.beginTransaction(100);
// Put
Put lv_put = new Put(Bytes.toBytes("Alto"));
lv_put.add(FAMILY, QUAL_A, Bytes.toBytes("Palo"));
lv_put.add(FAMILY, QUAL_B, Bytes.toBytes("Alto"));
table.put(transactionState, lv_put);
// Get
Result row1_A =
table.get(transactionState, new Get(Bytes.toBytes("Alto")).addColumn(FAMILY, QUAL_A));
System.out.println(
"Transactional Get result before put is committed:" + row1_A.size() + ":" + row1_A);
// Get
Result row1_B = table.get(new Get(Bytes.toBytes("Alto")).addColumn(FAMILY, QUAL_A));
System.out.println(
"Normal Get result before put is committed:" + row1_B.size() + ":" + row1_B);
// End
transactionManager.tryCommit(transactionState);
// Begin
transactionState = transactionManager.beginTransaction(101);
// Get
Result row1_C = table.get(new Get(Bytes.toBytes("Alto")).addColumn(FAMILY, QUAL_A));
System.out.println("Get after put was commited. Result: " + row1_C);
// Get
row1_A =
table.get(transactionState, new Get(Bytes.toBytes("Alto")).addColumn(FAMILY, QUAL_A));
System.out.println("Transactional Get after put was commited. Result : " + row1_A);
// End
transactionManager.tryCommit(transactionState);
try {
transactionState.completeRequest();
} catch (Exception e) {
System.out.println("\nCAUGHT\n");
}
}
}
private static void assertGet(byte[] row, byte[] familiy, byte[] qualifier, byte[] value)
throws IOException {
// run a get and see if the value matches
Get get = new Get(row);
get.addColumn(familiy, qualifier);
Result result = region.get(get, null);
assertEquals(1, result.size());
KeyValue kv = result.raw()[0];
byte[] r = kv.getValue();
assertTrue(Bytes.compareTo(r, value) == 0);
}
private void doCopyTableTest(boolean bulkload) throws Exception {
final TableName TABLENAME1 = TableName.valueOf("testCopyTable1");
final TableName TABLENAME2 = TableName.valueOf("testCopyTable2");
final byte[] FAMILY = Bytes.toBytes("family");
final byte[] COLUMN1 = Bytes.toBytes("c1");
try (Table t1 = TEST_UTIL.createTable(TABLENAME1, FAMILY);
Table t2 = TEST_UTIL.createTable(TABLENAME2, FAMILY); ) {
// put rows into the first table
for (int i = 0; i < 10; i++) {
Put p = new Put(Bytes.toBytes("row" + i));
p.addColumn(FAMILY, COLUMN1, COLUMN1);
t1.put(p);
}
CopyTable copy = new CopyTable();
int code;
if (bulkload) {
code =
ToolRunner.run(
new Configuration(TEST_UTIL.getConfiguration()),
copy,
new String[] {
"--new.name=" + TABLENAME2.getNameAsString(),
"--bulkload",
TABLENAME1.getNameAsString()
});
} else {
code =
ToolRunner.run(
new Configuration(TEST_UTIL.getConfiguration()),
copy,
new String[] {
"--new.name=" + TABLENAME2.getNameAsString(), TABLENAME1.getNameAsString()
});
}
assertEquals("copy job failed", 0, code);
// verify the data was copied into table 2
for (int i = 0; i < 10; i++) {
Get g = new Get(Bytes.toBytes("row" + i));
Result r = t2.get(g);
assertEquals(1, r.size());
assertTrue(CellUtil.matchingQualifier(r.rawCells()[0], COLUMN1));
}
} finally {
TEST_UTIL.deleteTable(TABLENAME1);
TEST_UTIL.deleteTable(TABLENAME2);
}
}
/**
* Puts are buffered, but this tests when a delete (not-buffered) is applied before the actual Put
* that creates it.
*
* @throws Exception
*/
@Test
public void testApplyDeleteBeforePut() throws Exception {
List<WALEntry> entries = new ArrayList<WALEntry>(5);
List<Cell> cells = new ArrayList<Cell>();
for (int i = 0; i < 2; i++) {
entries.add(createEntry(TABLE_NAME1, i, KeyValue.Type.Put, cells));
}
entries.add(createEntry(TABLE_NAME1, 1, KeyValue.Type.DeleteFamily, cells));
for (int i = 3; i < 5; i++) {
entries.add(createEntry(TABLE_NAME1, i, KeyValue.Type.Put, cells));
}
SINK.replicateEntries(entries, CellUtil.createCellScanner(cells.iterator()));
Get get = new Get(Bytes.toBytes(1));
Result res = table1.get(get);
assertEquals(0, res.size());
}
private void verifyData(Region newReg, int startRow, int numRows, byte[] qf, byte[]... families)
throws IOException {
for (int i = startRow; i < startRow + numRows; i++) {
byte[] row = Bytes.toBytes("" + i);
Get get = new Get(row);
for (byte[] family : families) {
get.addColumn(family, qf);
}
Result result = newReg.get(get);
Cell[] raw = result.rawCells();
assertEquals(families.length, result.size());
for (int j = 0; j < families.length; j++) {
assertTrue(CellUtil.matchingRow(raw[j], row));
assertTrue(CellUtil.matchingFamily(raw[j], families[j]));
assertTrue(CellUtil.matchingQualifier(raw[j], qf));
}
}
}
private void doGets(HRegion region) throws IOException {
for (int i = 0; i < NUM_ROWS; ++i) {
final byte[] rowKey = LoadTestKVGenerator.md5PrefixedKey(i).getBytes();
for (int j = 0; j < NUM_COLS_PER_ROW; ++j) {
final String qualStr = String.valueOf(j);
if (VERBOSE) {
System.err.println(
"Reading row " + i + ", column " + j + " " + Bytes.toString(rowKey) + "/" + qualStr);
}
final byte[] qualBytes = Bytes.toBytes(qualStr);
Get get = new Get(rowKey);
get.addColumn(CF_BYTES, qualBytes);
Result result = region.get(get);
assertEquals(1, result.size());
byte[] value = result.getValue(CF_BYTES, qualBytes);
assertTrue(LoadTestKVGenerator.verify(value, rowKey, qualBytes));
}
}
}
/**
* Pass the key, and reversed value to reduce
*
* @param key
* @param value
* @param context
* @throws IOException
*/
public void map(ImmutableBytesWritable key, Result value, Context context)
throws IOException, InterruptedException {
if (value.size() != 1) {
throw new IOException("There should only be one input column");
}
Map<byte[], NavigableMap<byte[], NavigableMap<Long, byte[]>>> cf = value.getMap();
if (!cf.containsKey(INPUT_FAMILY)) {
throw new IOException(
"Wrong input columns. Missing: '" + Bytes.toString(INPUT_FAMILY) + "'.");
}
// Get the original value and reverse it
String originalValue = Bytes.toString(value.getValue(INPUT_FAMILY, null));
StringBuilder newValue = new StringBuilder(originalValue);
newValue.reverse();
// Now set the value to be collected
Put outval = new Put(key.get());
outval.add(OUTPUT_FAMILY, null, Bytes.toBytes(newValue.toString()));
context.write(key, outval);
}
@Override
protected void map(ImmutableBytesWritable rowKey, Result result, Context context)
throws IOException, InterruptedException {
long recordSize = GeneralHelpers.getHBaseResultKVSize(result);
long colCount = result.size();
String rowKeyStr = null;
if (rowKeyType == RowKeyType.STRING) {
rowKeyStr = Bytes.toString(rowKey.get());
} else if (rowKeyType == RowKeyType.USER_ID) {
rowKeyStr = TruthyHelpers.getUserIDStrFromBytes(rowKey.get());
} else {
if (this.useBigInt) {
rowKeyStr = TruthyHelpers.getTweetIDStrFromBigIntBytes(rowKey.get());
} else {
rowKeyStr = TruthyHelpers.getTweetIDStrFromBytes(rowKey.get());
}
}
if (lastRowKeyStr == null) {
// start counting for the first row key
lastRowKeyStr = rowKeyStr;
totalColCount = colCount;
totalRecordSize = recordSize;
} else {
if (lastRowKeyStr.equals(rowKeyStr)) {
// keep counting for the same row key
totalColCount += colCount;
totalRecordSize += recordSize;
} else {
// write information about last row key
StringBuffer sb = new StringBuffer();
sb.append(totalColCount).append('\t');
sb.append(totalRecordSize);
context.write(new Text(lastRowKeyStr), new Text(sb.toString()));
// start counting for current row key
lastRowKeyStr = rowKeyStr;
totalColCount = colCount;
totalRecordSize = recordSize;
}
}
}
private Result filter(
TransactionState state, Result result, long startTimestamp, int localVersions)
throws IOException {
if (result == null) {
return null;
}
List<KeyValue> kvs = result.list();
if (kvs == null) {
return result;
}
Map<ByteArray, Map<ByteArray, Integer>> occurrences =
new HashMap<TransactionalTable.ByteArray, Map<ByteArray, Integer>>();
Map<ByteArray, Map<ByteArray, Long>> minTimestamp =
new HashMap<TransactionalTable.ByteArray, Map<ByteArray, Long>>();
List<KeyValue> nonDeletes = new ArrayList<KeyValue>();
List<KeyValue> filtered = new ArrayList<KeyValue>();
Map<ByteArray, Set<ByteArray>> read = new HashMap<ByteArray, Set<ByteArray>>();
DeleteTracker tracker = new DeleteTracker();
for (KeyValue kv : kvs) {
ByteArray family = new ByteArray(kv.getFamily());
ByteArray qualifier = new ByteArray(kv.getQualifier());
Set<ByteArray> readQualifiers = read.get(family);
if (readQualifiers == null) {
readQualifiers = new HashSet<TransactionalTable.ByteArray>();
read.put(family, readQualifiers);
} else if (readQualifiers.contains(qualifier)) continue;
// RowKey rk = new RowKey(kv.getRow(), getTableName());
if (state.tsoclient.validRead(kv.getTimestamp(), startTimestamp)) {
if (!tracker.addDeleted(kv)) nonDeletes.add(kv);
{
// Read valid value
readQualifiers.add(qualifier);
// statistics
// elementsGotten++;
Map<ByteArray, Integer> occurrencesCols = occurrences.get(family);
Integer times = null;
if (occurrencesCols != null) {
times = occurrencesCols.get(qualifier);
}
if (times != null) {
// elementsRead += times;
versionsAvg = times > versionsAvg ? times : alpha * versionsAvg + (1 - alpha) * times;
// extraVersionsAvg = times > extraVersionsAvg ? times : alpha *
// extraVersionsAvg + (1 - alpha) * times;
} else {
// elementsRead++;
versionsAvg = alpha * versionsAvg + (1 - alpha);
// extraVersionsAvg = alpha * extraVersionsAvg + (1 - alpha);
}
}
} else {
Map<ByteArray, Integer> occurrencesCols = occurrences.get(family);
Map<ByteArray, Long> minTimestampCols = minTimestamp.get(family);
if (occurrencesCols == null) {
occurrencesCols = new HashMap<TransactionalTable.ByteArray, Integer>();
minTimestampCols = new HashMap<TransactionalTable.ByteArray, Long>();
occurrences.put(family, occurrencesCols);
minTimestamp.put(family, minTimestampCols);
}
Integer times = occurrencesCols.get(qualifier);
Long timestamp = minTimestampCols.get(qualifier);
if (times == null) {
times = 0;
timestamp = kv.getTimestamp();
}
times++;
timestamp = Math.min(timestamp, kv.getTimestamp());
if (times == localVersions) {
// We need to fetch more versions
Get get = new Get(kv.getRow());
get.addColumn(kv.getFamily(), kv.getQualifier());
get.setMaxVersions(localVersions);
Result r;
GOTRESULT:
do {
extraGetsPerformed++;
get.setTimeRange(0, timestamp);
r = this.get(get);
List<KeyValue> list = r.list();
if (list == null) break;
for (KeyValue t : list) {
times++;
timestamp = Math.min(timestamp, t.getTimestamp());
// rk = new RowKey(kv.getRow(), getTableName());
if (state.tsoclient.validRead(t.getTimestamp(), startTimestamp)) {
if (!tracker.addDeleted(t)) nonDeletes.add(t);
readQualifiers.add(qualifier);
elementsGotten++;
elementsRead += times;
versionsAvg =
times > versionsAvg ? times : alpha * versionsAvg + (1 - alpha) * times;
extraVersionsAvg =
times > extraVersionsAvg
? times
: alpha * extraVersionsAvg + (1 - alpha) * times;
break GOTRESULT;
}
}
} while (r.size() == localVersions);
} else {
occurrencesCols.put(qualifier, times);
minTimestampCols.put(qualifier, timestamp);
}
}
}
for (KeyValue kv : nonDeletes) {
if (!tracker.isDeleted(kv)) {
filtered.add(kv);
}
}
// cacheVersions = (int) versionsAvg;
if (filtered.isEmpty()) {
return null;
}
return new Result(filtered);
}
/**
* Test transactional delete operations.
*
* @throws Exception
*/
@Test
public void testValidTransactionalDelete() throws Exception {
try (HTable hTable =
createTable(
Bytes.toBytes("TestValidTransactionalDelete"),
new byte[][] {TestBytes.family, TestBytes.family2})) {
TransactionAwareHTable txTable = new TransactionAwareHTable(hTable);
TransactionContext txContext =
new TransactionContext(new InMemoryTxSystemClient(txManager), txTable);
txContext.start();
Put put = new Put(TestBytes.row);
put.add(TestBytes.family, TestBytes.qualifier, TestBytes.value);
put.add(TestBytes.family2, TestBytes.qualifier, TestBytes.value2);
txTable.put(put);
txContext.finish();
txContext.start();
Result result = txTable.get(new Get(TestBytes.row));
txContext.finish();
byte[] value = result.getValue(TestBytes.family, TestBytes.qualifier);
assertArrayEquals(TestBytes.value, value);
value = result.getValue(TestBytes.family2, TestBytes.qualifier);
assertArrayEquals(TestBytes.value2, value);
// test full row delete
txContext.start();
Delete delete = new Delete(TestBytes.row);
txTable.delete(delete);
txContext.finish();
txContext.start();
result = txTable.get(new Get(TestBytes.row));
txContext.finish();
assertTrue(result.isEmpty());
// test column delete
// load 10 rows
txContext.start();
int rowCount = 10;
for (int i = 0; i < rowCount; i++) {
Put p = new Put(Bytes.toBytes("row" + i));
for (int j = 0; j < 10; j++) {
p.add(TestBytes.family, Bytes.toBytes(j), TestBytes.value);
}
txTable.put(p);
}
txContext.finish();
// verify loaded rows
txContext.start();
for (int i = 0; i < rowCount; i++) {
Get g = new Get(Bytes.toBytes("row" + i));
Result r = txTable.get(g);
assertFalse(r.isEmpty());
for (int j = 0; j < 10; j++) {
assertArrayEquals(TestBytes.value, r.getValue(TestBytes.family, Bytes.toBytes(j)));
}
}
txContext.finish();
// delete odds columns from odd rows and even columns from even rows
txContext.start();
for (int i = 0; i < rowCount; i++) {
Delete d = new Delete(Bytes.toBytes("row" + i));
for (int j = 0; j < 10; j++) {
if (i % 2 == j % 2) {
LOG.info("Deleting row={}, column={}", i, j);
d.deleteColumns(TestBytes.family, Bytes.toBytes(j));
}
}
txTable.delete(d);
}
txContext.finish();
// verify deleted columns
txContext.start();
for (int i = 0; i < rowCount; i++) {
Get g = new Get(Bytes.toBytes("row" + i));
Result r = txTable.get(g);
assertEquals(5, r.size());
for (Map.Entry<byte[], byte[]> entry : r.getFamilyMap(TestBytes.family).entrySet()) {
int col = Bytes.toInt(entry.getKey());
LOG.info("Got row={}, col={}", i, col);
// each row should only have the opposite mod (odd=even, even=odd)
assertNotEquals(i % 2, col % 2);
assertArrayEquals(TestBytes.value, entry.getValue());
}
}
txContext.finish();
// test family delete
// load 10 rows
txContext.start();
for (int i = 0; i < rowCount; i++) {
Put p = new Put(Bytes.toBytes("famrow" + i));
p.add(TestBytes.family, TestBytes.qualifier, TestBytes.value);
p.add(TestBytes.family2, TestBytes.qualifier2, TestBytes.value2);
txTable.put(p);
}
txContext.finish();
// verify all loaded rows
txContext.start();
for (int i = 0; i < rowCount; i++) {
Get g = new Get(Bytes.toBytes("famrow" + i));
Result r = txTable.get(g);
assertEquals(2, r.size());
assertArrayEquals(TestBytes.value, r.getValue(TestBytes.family, TestBytes.qualifier));
assertArrayEquals(TestBytes.value2, r.getValue(TestBytes.family2, TestBytes.qualifier2));
}
txContext.finish();
// delete family1 for even rows, family2 for odd rows
txContext.start();
for (int i = 0; i < rowCount; i++) {
Delete d = new Delete(Bytes.toBytes("famrow" + i));
d.deleteFamily((i % 2 == 0) ? TestBytes.family : TestBytes.family2);
txTable.delete(d);
}
txContext.finish();
// verify deleted families
txContext.start();
for (int i = 0; i < rowCount; i++) {
Get g = new Get(Bytes.toBytes("famrow" + i));
Result r = txTable.get(g);
assertEquals(1, r.size());
if (i % 2 == 0) {
assertNull(r.getValue(TestBytes.family, TestBytes.qualifier));
assertArrayEquals(TestBytes.value2, r.getValue(TestBytes.family2, TestBytes.qualifier2));
} else {
assertArrayEquals(TestBytes.value, r.getValue(TestBytes.family, TestBytes.qualifier));
assertNull(r.getValue(TestBytes.family2, TestBytes.qualifier2));
}
}
txContext.finish();
}
}
@Test
public void testRowDelete() throws Exception {
HTable hTable =
createTable(
Bytes.toBytes("TestRowDelete"), new byte[][] {TestBytes.family, TestBytes.family2});
try (TransactionAwareHTable txTable =
new TransactionAwareHTable(hTable, TxConstants.ConflictDetection.ROW)) {
TransactionContext txContext =
new TransactionContext(new InMemoryTxSystemClient(txManager), txTable);
// Test 1: full row delete
txContext.start();
txTable.put(
new Put(TestBytes.row)
.add(TestBytes.family, TestBytes.qualifier, TestBytes.value)
.add(TestBytes.family, TestBytes.qualifier2, TestBytes.value2)
.add(TestBytes.family2, TestBytes.qualifier, TestBytes.value)
.add(TestBytes.family2, TestBytes.qualifier2, TestBytes.value2));
txContext.finish();
txContext.start();
Get get = new Get(TestBytes.row);
Result result = txTable.get(get);
assertFalse(result.isEmpty());
assertArrayEquals(TestBytes.value, result.getValue(TestBytes.family, TestBytes.qualifier));
assertArrayEquals(TestBytes.value2, result.getValue(TestBytes.family, TestBytes.qualifier2));
assertArrayEquals(TestBytes.value, result.getValue(TestBytes.family2, TestBytes.qualifier));
assertArrayEquals(TestBytes.value2, result.getValue(TestBytes.family2, TestBytes.qualifier2));
txContext.finish();
// delete entire row
txContext.start();
txTable.delete(new Delete(TestBytes.row));
txContext.finish();
// verify row is now empty
txContext.start();
result = txTable.get(new Get(TestBytes.row));
assertTrue(result.isEmpty());
// verify row is empty for explicit column retrieval
result =
txTable.get(
new Get(TestBytes.row)
.addColumn(TestBytes.family, TestBytes.qualifier)
.addFamily(TestBytes.family2));
assertTrue(result.isEmpty());
// verify row is empty for scan
ResultScanner scanner = txTable.getScanner(new Scan(TestBytes.row));
assertNull(scanner.next());
scanner.close();
// verify row is empty for scan with explicit column
scanner =
txTable.getScanner(
new Scan(TestBytes.row).addColumn(TestBytes.family2, TestBytes.qualifier2));
assertNull(scanner.next());
scanner.close();
txContext.finish();
// write swapped values to one column per family
txContext.start();
txTable.put(
new Put(TestBytes.row)
.add(TestBytes.family, TestBytes.qualifier, TestBytes.value2)
.add(TestBytes.family2, TestBytes.qualifier2, TestBytes.value));
txContext.finish();
// verify new values appear
txContext.start();
result = txTable.get(new Get(TestBytes.row));
assertFalse(result.isEmpty());
assertEquals(2, result.size());
assertArrayEquals(TestBytes.value2, result.getValue(TestBytes.family, TestBytes.qualifier));
assertArrayEquals(TestBytes.value, result.getValue(TestBytes.family2, TestBytes.qualifier2));
scanner = txTable.getScanner(new Scan(TestBytes.row));
Result result1 = scanner.next();
assertNotNull(result1);
assertFalse(result1.isEmpty());
assertEquals(2, result1.size());
assertArrayEquals(TestBytes.value2, result.getValue(TestBytes.family, TestBytes.qualifier));
assertArrayEquals(TestBytes.value, result.getValue(TestBytes.family2, TestBytes.qualifier2));
scanner.close();
txContext.finish();
// Test 2: delete of first column family
txContext.start();
txTable.put(
new Put(TestBytes.row2)
.add(TestBytes.family, TestBytes.qualifier, TestBytes.value)
.add(TestBytes.family, TestBytes.qualifier2, TestBytes.value2)
.add(TestBytes.family2, TestBytes.qualifier, TestBytes.value)
.add(TestBytes.family2, TestBytes.qualifier2, TestBytes.value2));
txContext.finish();
txContext.start();
txTable.delete(new Delete(TestBytes.row2).deleteFamily(TestBytes.family));
txContext.finish();
txContext.start();
Result fam1Result = txTable.get(new Get(TestBytes.row2));
assertFalse(fam1Result.isEmpty());
assertEquals(2, fam1Result.size());
assertArrayEquals(
TestBytes.value, fam1Result.getValue(TestBytes.family2, TestBytes.qualifier));
assertArrayEquals(
TestBytes.value2, fam1Result.getValue(TestBytes.family2, TestBytes.qualifier2));
txContext.finish();
// Test 3: delete of second column family
txContext.start();
txTable.put(
new Put(TestBytes.row3)
.add(TestBytes.family, TestBytes.qualifier, TestBytes.value)
.add(TestBytes.family, TestBytes.qualifier2, TestBytes.value2)
.add(TestBytes.family2, TestBytes.qualifier, TestBytes.value)
.add(TestBytes.family2, TestBytes.qualifier2, TestBytes.value2));
txContext.finish();
txContext.start();
txTable.delete(new Delete(TestBytes.row3).deleteFamily(TestBytes.family2));
txContext.finish();
txContext.start();
Result fam2Result = txTable.get(new Get(TestBytes.row3));
assertFalse(fam2Result.isEmpty());
assertEquals(2, fam2Result.size());
assertArrayEquals(
TestBytes.value, fam2Result.getValue(TestBytes.family, TestBytes.qualifier));
assertArrayEquals(
TestBytes.value2, fam2Result.getValue(TestBytes.family, TestBytes.qualifier2));
txContext.finish();
// Test 4: delete specific rows in a range
txContext.start();
for (int i = 0; i < 10; i++) {
txTable.put(
new Put(Bytes.toBytes("z" + i))
.add(TestBytes.family, TestBytes.qualifier, Bytes.toBytes(i))
.add(TestBytes.family2, TestBytes.qualifier2, Bytes.toBytes(i)));
}
txContext.finish();
txContext.start();
// delete odd rows
for (int i = 1; i < 10; i += 2) {
txTable.delete(new Delete(Bytes.toBytes("z" + i)));
}
txContext.finish();
txContext.start();
int cnt = 0;
ResultScanner zScanner = txTable.getScanner(new Scan(Bytes.toBytes("z0")));
Result res;
while ((res = zScanner.next()) != null) {
assertFalse(res.isEmpty());
assertArrayEquals(Bytes.toBytes("z" + cnt), res.getRow());
assertArrayEquals(Bytes.toBytes(cnt), res.getValue(TestBytes.family, TestBytes.qualifier));
assertArrayEquals(
Bytes.toBytes(cnt), res.getValue(TestBytes.family2, TestBytes.qualifier2));
cnt += 2;
}
// Test 5: delete prior writes in the same transaction
txContext.start();
txTable.put(
new Put(TestBytes.row4)
.add(TestBytes.family, TestBytes.qualifier, TestBytes.value)
.add(TestBytes.family2, TestBytes.qualifier2, TestBytes.value2));
txTable.delete(new Delete(TestBytes.row4));
txContext.finish();
txContext.start();
Result row4Result = txTable.get(new Get(TestBytes.row4));
assertTrue(row4Result.isEmpty());
txContext.finish();
}
}
@Test
public void testCheckpoint() throws Exception {
// start a transaction, using checkpoints between writes
transactionContext.start();
transactionAwareHTable.put(
new Put(TestBytes.row).add(TestBytes.family, TestBytes.qualifier, TestBytes.value));
Transaction origTx = transactionContext.getCurrentTransaction();
transactionContext.checkpoint();
Transaction postCheckpointTx = transactionContext.getCurrentTransaction();
assertEquals(origTx.getTransactionId(), postCheckpointTx.getTransactionId());
assertNotEquals(origTx.getWritePointer(), postCheckpointTx.getWritePointer());
long[] checkpointPtrs = postCheckpointTx.getCheckpointWritePointers();
assertEquals(1, checkpointPtrs.length);
assertEquals(postCheckpointTx.getWritePointer(), checkpointPtrs[0]);
transactionAwareHTable.put(
new Put(TestBytes.row2).add(TestBytes.family, TestBytes.qualifier, TestBytes.value2));
transactionContext.checkpoint();
Transaction postCheckpointTx2 = transactionContext.getCurrentTransaction();
assertEquals(origTx.getTransactionId(), postCheckpointTx2.getTransactionId());
assertNotEquals(postCheckpointTx.getWritePointer(), postCheckpointTx2.getWritePointer());
long[] checkpointPtrs2 = postCheckpointTx2.getCheckpointWritePointers();
assertEquals(2, checkpointPtrs2.length);
assertEquals(postCheckpointTx.getWritePointer(), checkpointPtrs2[0]);
assertEquals(postCheckpointTx2.getWritePointer(), checkpointPtrs2[1]);
transactionAwareHTable.put(
new Put(TestBytes.row3).add(TestBytes.family, TestBytes.qualifier, TestBytes.value));
// by default, all rows should be visible with Read-Your-Writes
verifyRow(transactionAwareHTable, TestBytes.row, TestBytes.value);
verifyRow(transactionAwareHTable, TestBytes.row2, TestBytes.value2);
verifyRow(transactionAwareHTable, TestBytes.row3, TestBytes.value);
// when disabling current write pointer, only the previous checkpoints should be visible
transactionContext
.getCurrentTransaction()
.setVisibility(Transaction.VisibilityLevel.SNAPSHOT_EXCLUDE_CURRENT);
Get get = new Get(TestBytes.row);
verifyRow(transactionAwareHTable, get, TestBytes.value);
get = new Get(TestBytes.row2);
verifyRow(transactionAwareHTable, get, TestBytes.value2);
get = new Get(TestBytes.row3);
verifyRow(transactionAwareHTable, get, null);
// test scan results excluding current write pointer
Scan scan = new Scan();
ResultScanner scanner = transactionAwareHTable.getScanner(scan);
Result row = scanner.next();
assertNotNull(row);
assertArrayEquals(TestBytes.row, row.getRow());
assertEquals(1, row.size());
assertArrayEquals(TestBytes.value, row.getValue(TestBytes.family, TestBytes.qualifier));
row = scanner.next();
assertNotNull(row);
assertArrayEquals(TestBytes.row2, row.getRow());
assertEquals(1, row.size());
assertArrayEquals(TestBytes.value2, row.getValue(TestBytes.family, TestBytes.qualifier));
row = scanner.next();
assertNull(row);
scanner.close();
transactionContext.getCurrentTransaction().setVisibility(Transaction.VisibilityLevel.SNAPSHOT);
// check that writes are still not visible to other clients
TransactionAwareHTable txTable2 = new TransactionAwareHTable(new HTable(conf, TestBytes.table));
TransactionContext txContext2 =
new TransactionContext(new InMemoryTxSystemClient(txManager), txTable2);
txContext2.start();
verifyRow(txTable2, TestBytes.row, null);
verifyRow(txTable2, TestBytes.row2, null);
verifyRow(txTable2, TestBytes.row3, null);
txContext2.finish();
// commit transaction, verify writes are visible
transactionContext.finish();
txContext2.start();
verifyRow(txTable2, TestBytes.row, TestBytes.value);
verifyRow(txTable2, TestBytes.row2, TestBytes.value2);
verifyRow(txTable2, TestBytes.row3, TestBytes.value);
txContext2.finish();
txTable2.close();
}