@Test
public void testV1CodecV2Compat() throws Exception {
long now = System.currentTimeMillis();
// NOTE: set visibilityUpperBound to 0 as this is expected default for decoding older version
// that doesn't store it
TreeMap<Long, TransactionManager.InProgressTx> inProgress =
Maps.newTreeMap(
ImmutableSortedMap.of(
16L, new TransactionManager.InProgressTx(0L, now + 1000),
17L, new TransactionManager.InProgressTx(0L, now + 1000)));
TransactionSnapshot snapshot =
new TransactionSnapshot(
now,
15,
18,
Lists.newArrayList(5L, 7L),
inProgress,
ImmutableMap.<Long, Set<ChangeId>>of(
17L,
Sets.newHashSet(
new ChangeId(Bytes.toBytes("ch1")), new ChangeId(Bytes.toBytes("ch2")))),
ImmutableMap.<Long, Set<ChangeId>>of(
16L,
Sets.newHashSet(
new ChangeId(Bytes.toBytes("ch2")), new ChangeId(Bytes.toBytes("ch3")))));
Configuration configV1 = HBaseConfiguration.create();
configV1.setStrings(
TxConstants.Persist.CFG_TX_SNAPHOT_CODEC_CLASSES, SnapshotCodecV1.class.getName());
SnapshotCodecProvider codecV1 = new SnapshotCodecProvider(configV1);
// encoding with codec of v1
ByteArrayOutputStream out = new ByteArrayOutputStream();
try {
codecV1.encode(out, snapshot);
} finally {
out.close();
}
// decoding
Configuration configV1V2 = HBaseConfiguration.create();
configV1V2.setStrings(
TxConstants.Persist.CFG_TX_SNAPHOT_CODEC_CLASSES,
SnapshotCodecV1.class.getName(),
SnapshotCodecV2.class.getName());
SnapshotCodecProvider codecV1V2 = new SnapshotCodecProvider(configV1V2);
TransactionSnapshot decoded = codecV1V2.decode(new ByteArrayInputStream(out.toByteArray()));
assertEquals(snapshot, decoded);
}
/**
* Returns the top words associated with the specified word and the number of times the words have
* appeared together.
*
* @param word the word of interest
* @param limit the number of associations to return, at most
* @return a map of the top associated words to their co-occurrence count
*/
public Map<String, Long> readWordAssocs(String word, int limit) {
// Retrieve all columns of the word’s row
Row result = this.table.get(new Get(word));
TopKCollector collector = new TopKCollector(limit);
if (!result.isEmpty()) {
// Iterate over all columns
for (Map.Entry<byte[], byte[]> entry : result.getColumns().entrySet()) {
collector.add(Bytes.toLong(entry.getValue()), Bytes.toString(entry.getKey()));
}
}
return collector.getTopK();
}
@Test
public void testCLFLogWithEscapedDoubleQuotes()
throws UnsupportedTypeException, UnexpectedFormatException {
CombinedLogRecordFormat format = new CombinedLogRecordFormat();
FormatSpecification spec =
new FormatSpecification(
CombinedLogRecordFormat.class.getCanonicalName(),
null,
ImmutableMap.<String, String>of());
format.initialize(spec);
String data =
"10.10.10.10 - - [01/Feb/2015:06:38:58 +0000] \"GET /plugins/servlet/buildStatusImage/CDAP-DUT "
+ "HTTP/1.1\" 301 257 \"http://cdap.io/\" \"\\\"Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, "
+ "like Gecko) Chrome/31.0.1650.57 Safari/537.36 OPR/18.0.1284.49\\\"\"";
StructuredRecord output = format.read(new StreamEvent(ByteBuffer.wrap(Bytes.toBytes(data))));
Assert.assertEquals("10.10.10.10", output.get("remote_host"));
Assert.assertNull(output.get("remote_login"));
Assert.assertNull(output.get("auth_user"));
Assert.assertEquals("01/Feb/2015:06:38:58 +0000", output.get("date"));
Assert.assertEquals(
"GET /plugins/servlet/buildStatusImage/CDAP-DUT HTTP/1.1", output.get("request"));
Assert.assertEquals(301, output.get("status"));
Assert.assertEquals(257, output.get("content_length"));
Assert.assertEquals("http://cdap.io/", output.get("referrer"));
Assert.assertEquals(
"\\\"Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.36 (KHTML, like Gecko) "
+ "Chrome/31.0.1650.57 Safari/537.36 OPR/18.0.1284.49\\\"",
output.get("user_agent"));
}
@Test
public void testCLFLog() throws UnsupportedTypeException, UnexpectedFormatException {
CombinedLogRecordFormat format = new CombinedLogRecordFormat();
FormatSpecification spec =
new FormatSpecification(
CombinedLogRecordFormat.class.getCanonicalName(),
null,
ImmutableMap.<String, String>of());
format.initialize(spec);
String data =
"10.10.10.10 - - [01/Feb/2015:06:47:10 +0000] \"GET /browse/COOP-DBT-JOB1-238/artifact HTTP/1.1\""
+ " 301 256 \"-\" \"Mozilla/5.0 (compatible; AhrefsBot/5.0; +http://ahrefs.com/robot/)\"";
StructuredRecord output = format.read(new StreamEvent(ByteBuffer.wrap(Bytes.toBytes(data))));
Assert.assertEquals("10.10.10.10", output.get("remote_host"));
Assert.assertNull(output.get("remote_login"));
Assert.assertNull(output.get("auth_user"));
Assert.assertEquals("01/Feb/2015:06:47:10 +0000", output.get("date"));
Assert.assertEquals("GET /browse/COOP-DBT-JOB1-238/artifact HTTP/1.1", output.get("request"));
Assert.assertEquals(301, output.get("status"));
Assert.assertEquals(256, output.get("content_length"));
Assert.assertNull(output.get("referrer"));
Assert.assertEquals(
"Mozilla/5.0 (compatible; AhrefsBot/5.0; +http://ahrefs.com/robot/)",
output.get("user_agent"));
}
/**
* Stores associations between the specified set of words. That is, for every word in the set, an
* association will be stored for each of the other words in the set.
*
* @param words words to store associations between
*/
public void writeWordAssocs(Set<String> words) {
// For sets of less than 2 words, there are no associations
int n = words.size();
if (n < 2) {
return;
}
// Every word will get (n-1) increments (one for each of the other words)
long[] values = new long[n - 1];
Arrays.fill(values, 1);
// Convert all words to bytes
byte[][] wordBytes = new byte[n][];
int i = 0;
for (String word : words) {
wordBytes[i++] = Bytes.toBytes(word);
}
// Generate an increment for each word
for (int j = 0; j < n; j++) {
byte[] row = wordBytes[j];
byte[][] columns = new byte[n - 1][];
System.arraycopy(wordBytes, 0, columns, 0, j);
System.arraycopy(wordBytes, j + 1, columns, j, n - j - 1);
this.table.increment(row, columns, values);
}
}
@Test
public void useTransactionTest() throws Exception {
// Performing admin operations to create dataset instance
// keyValueTable is a system dataset module
Id.DatasetInstance myTableInstance = Id.DatasetInstance.from(namespace, "myTable");
dsFramework.addInstance("keyValueTable", myTableInstance, DatasetProperties.EMPTY);
Assert.assertTrue(feedManager.createFeed(FEED1));
try {
Cancellable cancellable =
notificationService.subscribe(
FEED1,
new NotificationHandler<String>() {
private int received = 0;
@Override
public Type getNotificationType() {
return String.class;
}
@Override
public void received(
final String notification, NotificationContext notificationContext) {
notificationContext.execute(
new TxRunnable() {
@Override
public void run(DatasetContext context) throws Exception {
KeyValueTable table = context.getDataset("myTable");
table.write("foo", String.format("%s-%d", notification, received++));
}
},
TxRetryPolicy.maxRetries(5));
}
});
TimeUnit.SECONDS.sleep(2);
try {
notificationService.publish(FEED1, "foobar");
// Waiting for the subscriber to receive that notification
TimeUnit.SECONDS.sleep(2);
KeyValueTable table =
dsFramework.getDataset(myTableInstance, DatasetDefinition.NO_ARGUMENTS, null);
Assert.assertNotNull(table);
Transaction tx1 = txManager.startShort(100);
table.startTx(tx1);
Assert.assertEquals("foobar-0", Bytes.toString(table.read("foo")));
Assert.assertTrue(table.commitTx());
txManager.canCommit(tx1, table.getTxChanges());
txManager.commit(tx1);
table.postTxCommit();
} finally {
cancellable.cancel();
}
} finally {
dsFramework.deleteInstance(myTableInstance);
feedManager.deleteFeed(FEED1);
}
}
@Test(expected = UnexpectedFormatException.class)
public void testInvalid() throws UnsupportedTypeException, UnexpectedFormatException {
CombinedLogRecordFormat format = new CombinedLogRecordFormat();
FormatSpecification spec =
new FormatSpecification(
CombinedLogRecordFormat.class.getCanonicalName(),
null,
ImmutableMap.<String, String>of());
format.initialize(spec);
String data = "10.10.10.10[01/Feb/2015:06:47:10 +0000";
StructuredRecord output = format.read(new StreamEvent(ByteBuffer.wrap(Bytes.toBytes(data))));
}
@Test
public void testColumnFamily() throws Exception {
DatasetProperties props =
DatasetProperties.builder().add(Table.PROPERTY_COLUMN_FAMILY, "t").build();
HBaseTableDefinition tableDefinition = new HBaseTableDefinition("foo");
String tableName = "testcf";
DatasetSpecification spec = tableDefinition.configure(tableName, props);
DatasetAdmin admin =
new HBaseTableAdmin(
CONTEXT1,
spec,
testHBase.getConfiguration(),
hBaseTableUtil,
CConfiguration.create(),
new LocalLocationFactory(tmpFolder.newFolder()));
admin.create();
final HBaseTable table =
new HBaseTable(CONTEXT1, spec, cConf, testHBase.getConfiguration(), hBaseTableUtil);
TransactionSystemClient txClient = new DetachedTxSystemClient();
TransactionExecutor executor = new DefaultTransactionExecutor(txClient, table);
executor.execute(
new TransactionExecutor.Subroutine() {
@Override
public void apply() throws Exception {
table.put(new Put("row", "column", "testValue"));
}
});
final HBaseTable table2 =
new HBaseTable(CONTEXT1, spec, cConf, testHBase.getConfiguration(), hBaseTableUtil);
executor = new DefaultTransactionExecutor(txClient, table2);
executor.execute(
new TransactionExecutor.Subroutine() {
@Override
public void apply() throws Exception {
Assert.assertEquals(
"testValue", table2.get(new Get("row", "column")).getString("column"));
}
});
// Verify the column family name
HTableDescriptor htd =
hBaseTableUtil.getHTableDescriptor(
testHBase.getHBaseAdmin(), TableId.from(CONTEXT1.getNamespaceId(), tableName));
HColumnDescriptor hcd = htd.getFamily(Bytes.toBytes("t"));
Assert.assertNotNull(hcd);
Assert.assertEquals("t", hcd.getNameAsString());
}
@Test
public void testCLFLogWithNull() throws UnsupportedTypeException, UnexpectedFormatException {
CombinedLogRecordFormat format = new CombinedLogRecordFormat();
FormatSpecification spec =
new FormatSpecification(
CombinedLogRecordFormat.class.getCanonicalName(),
null,
ImmutableMap.<String, String>of());
format.initialize(spec);
String data = "10.10.10.10 - - [01/Feb/2015:09:58:24 +0000] \"-\" 408 - \"-\" \"-\"";
StructuredRecord output = format.read(new StreamEvent(ByteBuffer.wrap(Bytes.toBytes(data))));
Assert.assertEquals("10.10.10.10", output.get("remote_host"));
Assert.assertNull(output.get("remote_login"));
Assert.assertNull(output.get("auth_user"));
Assert.assertEquals("01/Feb/2015:09:58:24 +0000", output.get("date"));
Assert.assertNull(output.get("request"));
Assert.assertEquals(408, output.get("status"));
Assert.assertNull(output.get("content_length"));
Assert.assertNull(output.get("referrer"));
Assert.assertNull(output.get("user_agent"));
}
@Test
public void testPreSplit() throws Exception {
byte[][] splits = new byte[][] {Bytes.toBytes("a"), Bytes.toBytes("b"), Bytes.toBytes("c")};
DatasetProperties props =
DatasetProperties.builder().add("hbase.splits", new Gson().toJson(splits)).build();
String presplittedTable = "presplitted";
getTableAdmin(CONTEXT1, presplittedTable, props).create();
HBaseAdmin hBaseAdmin = testHBase.getHBaseAdmin();
try {
List<HRegionInfo> regions =
hBaseTableUtil.getTableRegions(
hBaseAdmin, TableId.from(NAMESPACE1.getId(), presplittedTable));
// note: first region starts at very first row key, so we have one extra to the splits count
Assert.assertEquals(4, regions.size());
Assert.assertArrayEquals(Bytes.toBytes("a"), regions.get(1).getStartKey());
Assert.assertArrayEquals(Bytes.toBytes("b"), regions.get(2).getStartKey());
Assert.assertArrayEquals(Bytes.toBytes("c"), regions.get(3).getStartKey());
} finally {
hBaseAdmin.close();
}
}
public void put(int key, String value) throws Exception {
table.put(Bytes.toBytes(key), COL, Bytes.toBytes(value));
}
@Test
public void testTTL() throws Exception {
// for the purpose of this test it is fine not to configure ttl when creating table: we want to
// see if it
// applies on reading
int ttl = 1000;
String ttlTable = "ttl";
String noTtlTable = "nottl";
DatasetProperties props =
DatasetProperties.builder().add(Table.PROPERTY_TTL, String.valueOf(ttl)).build();
getTableAdmin(CONTEXT1, ttlTable, props).create();
DatasetSpecification ttlTableSpec =
DatasetSpecification.builder(ttlTable, HBaseTable.class.getName())
.properties(props.getProperties())
.build();
HBaseTable table =
new HBaseTable(CONTEXT1, ttlTableSpec, cConf, testHBase.getConfiguration(), hBaseTableUtil);
DetachedTxSystemClient txSystemClient = new DetachedTxSystemClient();
Transaction tx = txSystemClient.startShort();
table.startTx(tx);
table.put(b("row1"), b("col1"), b("val1"));
table.commitTx();
TimeUnit.SECONDS.sleep(2);
tx = txSystemClient.startShort();
table.startTx(tx);
table.put(b("row2"), b("col2"), b("val2"));
table.commitTx();
// now, we should not see first as it should have expired, but see the last one
tx = txSystemClient.startShort();
table.startTx(tx);
byte[] val = table.get(b("row1"), b("col1"));
if (val != null) {
LOG.info("Unexpected value " + Bytes.toStringBinary(val));
}
Assert.assertNull(val);
Assert.assertArrayEquals(b("val2"), table.get(b("row2"), b("col2")));
// test a table with no TTL
DatasetProperties props2 =
DatasetProperties.builder().add(Table.PROPERTY_TTL, String.valueOf(Tables.NO_TTL)).build();
getTableAdmin(CONTEXT1, noTtlTable, props2).create();
DatasetSpecification noTtlTableSpec =
DatasetSpecification.builder(noTtlTable, HBaseTable.class.getName())
.properties(props2.getProperties())
.build();
HBaseTable table2 =
new HBaseTable(
CONTEXT1, noTtlTableSpec, cConf, testHBase.getConfiguration(), hBaseTableUtil);
tx = txSystemClient.startShort();
table2.startTx(tx);
table2.put(b("row1"), b("col1"), b("val1"));
table2.commitTx();
TimeUnit.SECONDS.sleep(2);
tx = txSystemClient.startShort();
table2.startTx(tx);
table2.put(b("row2"), b("col2"), b("val2"));
table2.commitTx();
// if ttl is -1 (unlimited), it should see both
tx = txSystemClient.startShort();
table2.startTx(tx);
Assert.assertArrayEquals(b("val1"), table2.get(b("row1"), b("col1")));
Assert.assertArrayEquals(b("val2"), table2.get(b("row2"), b("col2")));
}
private static byte[] b(String s) {
return Bytes.toBytes(s);
}
@Test
public void testEnableIncrements() throws Exception {
// setup a table with increments disabled and with it enabled
String disableTableName = "incr-disable";
String enabledTableName = "incr-enable";
TableId disabledTableId = TableId.from(NAMESPACE1.getId(), disableTableName);
TableId enabledTableId = TableId.from(NAMESPACE1.getId(), enabledTableName);
HBaseTableAdmin disabledAdmin =
getTableAdmin(CONTEXT1, disableTableName, DatasetProperties.EMPTY);
disabledAdmin.create();
HBaseAdmin admin = testHBase.getHBaseAdmin();
DatasetProperties props =
DatasetProperties.builder().add(Table.PROPERTY_READLESS_INCREMENT, "true").build();
HBaseTableAdmin enabledAdmin = getTableAdmin(CONTEXT1, enabledTableName, props);
enabledAdmin.create();
try {
try {
HTableDescriptor htd = hBaseTableUtil.getHTableDescriptor(admin, disabledTableId);
List<String> cps = htd.getCoprocessors();
assertFalse(cps.contains(IncrementHandler.class.getName()));
htd = hBaseTableUtil.getHTableDescriptor(admin, enabledTableId);
cps = htd.getCoprocessors();
assertTrue(cps.contains(IncrementHandler.class.getName()));
} finally {
admin.close();
}
BufferingTable table = getTable(CONTEXT1, enabledTableName, ConflictDetection.COLUMN);
byte[] row = Bytes.toBytes("row1");
byte[] col = Bytes.toBytes("col1");
DetachedTxSystemClient txSystemClient = new DetachedTxSystemClient();
Transaction tx = txSystemClient.startShort();
table.startTx(tx);
table.increment(row, col, 10);
table.commitTx();
// verify that value was written as a delta value
final byte[] expectedValue =
Bytes.add(IncrementHandlerState.DELTA_MAGIC_PREFIX, Bytes.toBytes(10L));
final AtomicBoolean foundValue = new AtomicBoolean();
byte[] enabledTableNameBytes =
hBaseTableUtil.getHTableDescriptor(admin, enabledTableId).getName();
testHBase.forEachRegion(
enabledTableNameBytes,
new Function<HRegion, Object>() {
@Nullable
@Override
public Object apply(@Nullable HRegion hRegion) {
Scan scan = new Scan();
try {
RegionScanner scanner = hRegion.getScanner(scan);
List<Cell> results = Lists.newArrayList();
boolean hasMore;
do {
hasMore = scanner.next(results);
for (Cell cell : results) {
if (CellUtil.matchingValue(cell, expectedValue)) {
foundValue.set(true);
}
}
} while (hasMore);
} catch (IOException ioe) {
fail("IOException scanning region: " + ioe.getMessage());
}
return null;
}
});
assertTrue(
"Should have seen the expected encoded delta value in the "
+ enabledTableName
+ " table region",
foundValue.get());
} finally {
disabledAdmin.drop();
enabledAdmin.drop();
}
}
public void put(String key, KeyValue.Value value) throws Exception {
table.put(Bytes.toBytes(key), COL, Bytes.toBytes(GSON.toJson(value)));
}
public KeyValue.Value get(String key) throws Exception {
return GSON.fromJson(
Bytes.toString(table.get(Bytes.toBytes(key), COL)), KeyValue.Value.class);
}
public String get(int key) throws Exception {
return Bytes.toString(table.get(Bytes.toBytes(key), COL));
}
public void reduce(Text key, Iterable<Text> values, Context context)
throws IOException, InterruptedException {
for (Text value : values) {
context.write(Bytes.toBytes(key.toString()), Bytes.toBytes(value.toString()));
}
}
@Override
public KeyValue makeRecord(byte[] key, Row row) {
return new KeyValue(
Bytes.toInt(key),
GSON.fromJson(Bytes.toString(row.get(KeyValueTable.COL)), String.class));
}