@Test(timeout = 30000)
public void testCreateDeleteTable() throws IOException {
// Create table then get the single region for our new table.
HTableDescriptor hdt = HTU.createTableDescriptor("testCreateDeleteTable");
hdt.setRegionReplication(NB_SERVERS);
hdt.addCoprocessor(SlowMeCopro.class.getName());
Table table = HTU.createTable(hdt, new byte[][] {f}, HTU.getConfiguration());
Put p = new Put(row);
p.add(f, row, row);
table.put(p);
Get g = new Get(row);
Result r = table.get(g);
Assert.assertFalse(r.isStale());
try {
// But if we ask for stale we will get it
SlowMeCopro.cdl.set(new CountDownLatch(1));
g = new Get(row);
g.setConsistency(Consistency.TIMELINE);
r = table.get(g);
Assert.assertTrue(r.isStale());
SlowMeCopro.cdl.get().countDown();
} finally {
SlowMeCopro.cdl.get().countDown();
SlowMeCopro.sleepTime.set(0);
}
HTU.getHBaseAdmin().disableTable(hdt.getTableName());
HTU.deleteTable(hdt.getTableName());
}
public Edge getAssociationWeight(Vertex v1, Vertex v2) {
Edge edge = graph.get(v1, v2);
if (edge == null) {
edge = graph.get(v2, v1);
}
return edge;
}
public void testPurgeForce() {
Table<Integer> table = new Table<Integer>(3, 10, 0);
for (int i = 1; i <= 30; i++) table.add(i, i);
System.out.println("table = " + table);
table.purge(15, true);
System.out.println("table = " + table);
assertIndices(table, 15, 15, 30);
for (int i = 1; i <= 15; i++) assert table._get(i) == null;
for (int i = 16; i <= 30; i++) assert table._get(i) != null;
assert table.get(5) == null && table.get(25) != null;
table.purge(30, true);
System.out.println("table = " + table);
assertIndices(table, 30, 30, 30);
assert table.isEmpty();
for (int i = 1; i <= 30; i++) assert table._get(i) == null;
for (int i = 31; i <= 40; i++) table.add(i, i);
System.out.println("table = " + table);
assert table.size() == 10;
assertIndices(table, 30, 30, 40);
table.purge(50, true);
System.out.println("table = " + table);
assert table.isEmpty();
assertIndices(table, 40, 40, 40);
}
public void testResizeWithPurge2() {
Table<Integer> table = new Table<Integer>(3, 10, 0);
for (int i = 1; i <= 50; i++) addAndGet(table, i);
System.out.println("table = " + table);
assert table.size() == 50;
assertCapacity(table.capacity(), table.getNumRows(), 10);
assertIndices(table, 0, 0, 50);
table.removeMany(false, 43);
System.out.println("table = " + table);
assertIndices(table, 0, 43, 50);
table.purge(43);
System.out.println("table = " + table);
assertIndices(table, 43, 43, 50);
addAndGet(table, 52);
assert table.get(43) == null;
for (long i = 44; i <= 50; i++) {
Integer num = table.get(i);
assert num != null && num == i;
}
assert table.get(50) != null;
assert table.get(51) == null;
Integer num = table.get(52);
assert num != null && num == 52;
assert table.get(53) == null;
}
@Test(timeout = 120000)
public void testChangeTable() throws Exception {
HTableDescriptor hdt = HTU.createTableDescriptor("testChangeTable");
hdt.setRegionReplication(NB_SERVERS);
hdt.addCoprocessor(SlowMeCopro.class.getName());
Table table = HTU.createTable(hdt, new byte[][] {f}, HTU.getConfiguration());
// basic test: it should work.
Put p = new Put(row);
p.add(f, row, row);
table.put(p);
Get g = new Get(row);
Result r = table.get(g);
Assert.assertFalse(r.isStale());
// Add a CF, it should work.
HTableDescriptor bHdt = HTU.getHBaseAdmin().getTableDescriptor(hdt.getTableName());
HColumnDescriptor hcd = new HColumnDescriptor(row);
hdt.addFamily(hcd);
HTU.getHBaseAdmin().disableTable(hdt.getTableName());
HTU.getHBaseAdmin().modifyTable(hdt.getTableName(), hdt);
HTU.getHBaseAdmin().enableTable(hdt.getTableName());
HTableDescriptor nHdt = HTU.getHBaseAdmin().getTableDescriptor(hdt.getTableName());
Assert.assertEquals(
"fams=" + Arrays.toString(nHdt.getColumnFamilies()),
bHdt.getColumnFamilies().length + 1,
nHdt.getColumnFamilies().length);
p = new Put(row);
p.add(row, row, row);
table.put(p);
g = new Get(row);
r = table.get(g);
Assert.assertFalse(r.isStale());
try {
SlowMeCopro.cdl.set(new CountDownLatch(1));
g = new Get(row);
g.setConsistency(Consistency.TIMELINE);
r = table.get(g);
Assert.assertTrue(r.isStale());
} finally {
SlowMeCopro.cdl.get().countDown();
SlowMeCopro.sleepTime.set(0);
}
Admin admin = HTU.getHBaseAdmin();
nHdt = admin.getTableDescriptor(hdt.getTableName());
Assert.assertEquals(
"fams=" + Arrays.toString(nHdt.getColumnFamilies()),
bHdt.getColumnFamilies().length + 1,
nHdt.getColumnFamilies().length);
admin.disableTable(hdt.getTableName());
admin.deleteTable(hdt.getTableName());
admin.close();
}
public void testGet() {
final Table<Integer> buf = new Table<Integer>(3, 10, 0);
for (int i : Arrays.asList(1, 2, 3, 4, 5)) buf.add(i, i);
assert buf.get(0) == null;
assert buf.get(1) == 1;
assert buf.get(10) == null;
assert buf.get(5) == 5;
assert buf.get(6) == null;
}
public void testCreateCopyArrayTable() {
Table<String, Integer, Character> original =
create("foo", 1, 'a', "bar", 1, 'b', "foo", 3, 'c');
Table<String, Integer, Character> copy = ArrayTable.create(original);
assertEquals(original, copy);
original.put("foo", 1, 'd');
assertEquals((Character) 'd', original.get("foo", 1));
assertEquals((Character) 'a', copy.get("foo", 1));
assertEquals(copy.rowKeySet(), original.rowKeySet());
assertEquals(copy.columnKeySet(), original.columnKeySet());
}
public void testPurge3() {
Table<Integer> table = new Table<Integer>(3, 10, 0);
for (int i = 1; i <= 100; i++) table.add(i, i);
System.out.println("table = " + table);
table.removeMany(true, 53);
for (int i = 54; i <= 100; i++) assert table.get(i) == i;
}
public Optional<Response> get(Optional<Request> request) {
if (!valid) {
Logger.error("CANNOT GET! NO VALID CONNECTION");
return Optional.empty();
}
Response response = new Response();
if (request.isPresent()) {
Request r = request.get();
response.key = r.key;
response.table = r.table;
try {
final Table htable = connection.getTable(TableName.valueOf(r.table));
Result result = htable.get(new Get(r.key));
if (result == null || result.isEmpty()) {
return Optional.empty();
}
r.columns.forEach(
c ->
response.columns.add(
new Request.Column(
c.family,
c.qualifier,
result.getValue(c.family.getBytes(), c.qualifier.getBytes()))));
} catch (IOException e) {
e.printStackTrace();
}
}
return Optional.of(response);
}
public void testGetList() {
final Table<Integer> buf = new Table<Integer>(3, 10, 0);
for (int i : Arrays.asList(1, 2, 3, 4, 5)) buf.add(i, i);
List<Integer> elements = buf.get(3, 5);
System.out.println("elements = " + elements);
assert elements != null && elements.size() == 3;
assert elements.contains(3) && elements.contains(4) && elements.contains(5);
elements = buf.get(4, 10);
System.out.println("elements = " + elements);
assert elements != null && elements.size() == 2;
assert elements.contains(4) && elements.contains(5);
elements = buf.get(10, 20);
assert elements == null;
}
public Optional<byte[]> get(
Optional<String> table,
Optional<String> family,
Optional<String> qualifier,
Optional<String> key) {
if (!valid) {
Logger.error("CANNOT GET! NO VALID CONNECTION");
return Optional.empty();
}
if (table.isPresent()
&& family.isPresent()
&& qualifier.isPresent()
&& key.isPresent()
&& !key.get().isEmpty()) {
try {
final Table htable = connection.getTable(TableName.valueOf(table.get()));
Result result = htable.get(new Get(key.get().getBytes("UTF8")));
return Optional.ofNullable(
result.getValue(family.get().getBytes("UTF8"), qualifier.get().getBytes("UTF8")));
} catch (IOException e) {
e.printStackTrace();
}
}
return Optional.empty();
}
/**
* We need to resend our first message with our conn_id
*
* @param sender
* @param seqno Resend the non null messages in the range [lowest .. seqno]
*/
protected void handleResendingOfFirstMessage(Address sender, long seqno) {
if (log.isTraceEnabled())
log.trace(local_addr + " <-- SEND_FIRST_SEQNO(" + sender + "," + seqno + ")");
SenderEntry entry = send_table.get(sender);
Table<Message> win = entry != null ? entry.sent_msgs : null;
if (win == null) {
if (log.isErrorEnabled())
log.error(local_addr + ": sender window for " + sender + " not found");
return;
}
boolean first_sent = false;
for (long i = win.getLow() + 1; i <= seqno; i++) {
Message rsp = win.get(i);
if (rsp == null) continue;
if (first_sent) {
down_prot.down(new Event(Event.MSG, rsp));
} else {
first_sent = true;
// We need to copy the UnicastHeader and put it back into the message because Message.copy()
// doesn't copy
// the headers and therefore we'd modify the original message in the sender retransmission
// window
// (https://jira.jboss.org/jira/browse/JGRP-965)
Message copy = rsp.copy();
Unicast2Header hdr = (Unicast2Header) copy.getHeader(this.id);
Unicast2Header newhdr = hdr.copy();
newhdr.first = true;
copy.putHeader(this.id, newhdr);
down_prot.down(new Event(Event.MSG, copy));
}
}
}
int test(Node node, Table test_table) {
int successes = 0;
// Grab an example from the test table
// run it through the node root
// Compare the output of the tree with the actual value from the table (the first column)
// If it's correct increase the number of successes.
Node pointer; // Ponter to the node in the tree currently being evaluated
pointer = node;
int result = -1;
// We have the line here and we run it though the tree
for (int row = 0; row < test_table.rows(); row++) {
// If node is leaf return eiither 1 or 0
// look at node.atrib and compare it with the column from the table
// if it's 1 go left, if it's 0 go right
test_table.getRow(row);
pointer = node;
while (true) {
// System.out.print("node testing is" );
// pointer.print();
if (pointer.id_child_left == -1 && pointer.id_child_right == -1) {
result = pointer.atribute;
break;
}
// System.out.println("Test is "+test_table.get(row, pointer.atribute) );
if (test_table.get(row, pointer.atribute) == 1) {
pointer = pointer.child_left;
continue;
}
if (test_table.get(row, pointer.atribute) == 0) {
pointer = pointer.child_right;
continue;
}
}
if (result == test_table.get(row, 0)) {
successes++;
}
}
return successes;
// throw new UnsupportedOperationException("Not supported yet."); //To change body of generated
// methods, choose Tools | Templates.
}
public void execute(String[] args, Shell shell) {
if (table.exists(args[0])) {
System.out.println("found");
System.out.println(table.get(args[0]));
} else {
System.out.println("not found");
}
}
public void testCreateCopyHashBasedTable() {
Table<String, Integer, Character> original = HashBasedTable.create();
original.put("foo", 1, 'a');
original.put("bar", 1, 'b');
original.put("foo", 3, 'c');
Table<String, Integer, Character> copy = ArrayTable.create(original);
assertEquals(4, copy.size());
assertEquals((Character) 'a', copy.get("foo", 1));
assertEquals((Character) 'b', copy.get("bar", 1));
assertEquals((Character) 'c', copy.get("foo", 3));
assertNull(copy.get("bar", 3));
original.put("foo", 1, 'd');
assertEquals((Character) 'd', original.get("foo", 1));
assertEquals((Character) 'a', copy.get("foo", 1));
assertEquals(copy.rowKeySet(), ImmutableSet.of("foo", "bar"));
assertEquals(copy.columnKeySet(), ImmutableSet.of(1, 3));
}
public static void testDuplicateAddition() {
Table<Integer> table = new Table<Integer>(3, 10, 0);
addAndGet(table, 1, 5, 9, 10);
assert !table.add(5, 5);
assert table.get(5) == 5;
assert table.size() == 4;
assertIndices(table, 0, 0, 10);
}
public static void testCreation() {
Table<Integer> table = new Table<Integer>(3, 10, 0);
System.out.println("table = " + table);
int size = table.size();
assert size == 0;
assert table.get(15) == null;
assertIndices(table, 0, 0, 0);
}
protected static void addAndGet(Table<Integer> table, int... seqnos) {
for (int seqno : seqnos) {
boolean added = table.add((long) seqno, seqno);
assert added;
Integer val = table.get(seqno);
assert val != null && val == seqno;
}
}
public Result get(String table, String family, byte[] key) {
final Table htable;
try {
htable = connection.getTable(TableName.valueOf(table));
return htable.get(new Get(key));
} catch (IOException e) {
e.printStackTrace();
}
return null;
}
@Test
public void whenCreateTable_thenCreated() {
final Table<String, String, Integer> distance = HashBasedTable.create();
distance.put("London", "Paris", 340);
distance.put("New York", "Los Angeles", 3940);
distance.put("London", "New York", 5576);
assertEquals(3940, distance.get("New York", "Los Angeles").intValue());
assertThat(distance.columnKeySet(), containsInAnyOrder("Paris", "New York", "Los Angeles"));
assertThat(distance.rowKeySet(), containsInAnyOrder("London", "New York"));
}
public static void testAdditionListWithOffset() {
Table<Integer> table = new Table<Integer>(3, 10, 100);
long seqnos[] = {101, 105, 109, 110, 111, 119, 120, 129};
List<Tuple<Long, Integer>> msgs = createList(seqnos);
System.out.println("table: " + table.dump());
assert table.add(msgs);
assert table.size() == 8;
for (long seqno : seqnos) assert table.get(seqno) == seqno;
assertCapacity(table.capacity(), 3, 10);
assertIndices(table, 100, 100, 129);
}
public void testPurge5() {
Table<Integer> table = new Table<Integer>(3, 10, 0);
for (int i = 1; i <= 100; i++) table.add(i, i);
System.out.println("table = " + table);
table.removeMany(false, 0);
table.purge(10);
for (int i = 1; i <= 10; i++) assert table._get(i) == null;
for (int i = 11; i <= 100; i++) assert table.get(i) == i;
table.purge(10);
for (int i = 11; i <= 100; i++) assert table.get(i) == i;
table.purge(50);
for (int i = 1; i <= 50; i++) assert table._get(i) == null;
for (int i = 51; i <= 100; i++) assert table.get(i) == i;
table.purge(100);
for (int i = 51; i <= 100; i++) assert table._get(i) == null;
}
public void run() {
for (SenderEntry val : send_table.values()) {
Table<Message> buf = val != null ? val.sent_msgs : null;
if (buf != null && !buf.isEmpty()) {
long from = buf.getHighestDelivered() + 1, to = buf.getHighestReceived();
List<Message> list = buf.get(from, to);
if (list != null) {
for (Message msg : list) retransmit(msg);
}
}
}
}
public static void main(String[] args) {
Table<Integer> t = new Table<Integer>();
t.put(Symbol.symbol("abc"), new Integer(1));
t.put(Symbol.symbol("efg"), new Integer(2));
System.out.println(t.get(Symbol.symbol("efg")));
System.out.println(t.get(Symbol.symbol("")));
t.put(Symbol.symbol("efg"), new Integer(3));
System.out.println(t.get(Symbol.symbol("efg")));
t.beginScope();
System.out.println(t.get(Symbol.symbol("efg")));
t.put(Symbol.symbol("efg"), new Integer(4));
System.out.println(t.get(Symbol.symbol("efg")));
t.put(Symbol.symbol("efg"), new Integer(5));
System.out.println(t.get(Symbol.symbol("efg")));
t.put(Symbol.symbol("new"), new Integer(6));
System.out.println(t.get(Symbol.symbol("d")));
System.out.println(t.get(Symbol.symbol("new")));
t.endScope();
System.out.println(t.get(Symbol.symbol("efg")));
System.out.println(t.get(Symbol.symbol("new")));
}
public static void testAdditionList() {
Table<Integer> table = new Table<Integer>(3, 10, 0);
List<Tuple<Long, Integer>> msgs = createList(0);
assert !table.add(msgs);
long[] seqnos = {1, 5, 9, 10, 11, 19, 20, 29};
msgs = createList(seqnos);
assert table.add(msgs);
System.out.println("table: " + table.dump());
for (long seqno : seqnos) assert table.get(seqno) == seqno;
assert table.size() == 8;
int size = table.computeSize();
assert size == 8;
assert table.size() == table.computeSize();
assertCapacity(table.capacity(), 3, 10);
assertIndices(table, 0, 0, 29);
}
@SuppressWarnings("unchecked")
public static <T> T getEntity(
final DbUtils db, final Cursor cursor, Class<T> entityType, long findCacheSequence) {
if (db == null || cursor == null) return null;
EntityTempCache.setSeq(findCacheSequence);
try {
Table table = Table.get(db, entityType);
Id id = table.id;
String idColumnName = id.getColumnName();
int idIndex = id.getIndex();
if (idIndex < 0) {
idIndex = cursor.getColumnIndex(idColumnName);
}
Object idValue = id.getColumnConverter().getFieldValue(cursor, idIndex);
T entity = EntityTempCache.get(entityType, idValue);
if (entity == null) {
entity = entityType.newInstance();
id.setValue2Entity(entity, cursor, idIndex);
EntityTempCache.put(entityType, idValue, entity);
} else {
return entity;
}
int columnCount = cursor.getColumnCount();
for (int i = 0; i < columnCount; i++) {
String columnName = cursor.getColumnName(i);
Column column = table.columnMap.get(columnName);
if (column != null) {
column.setValue2Entity(entity, cursor, i);
}
}
// init finder
for (Finder finder : table.finderMap.values()) {
finder.setValue2Entity(entity, null, 0);
}
return entity;
} catch (Throwable e) {
LogUtils.e(e.getMessage(), e);
}
return null;
}
@Test(timeout = 300000)
public void testDisableAndEnableTables() throws IOException {
final byte[] row = Bytes.toBytes("row");
final byte[] qualifier = Bytes.toBytes("qualifier");
final byte[] value = Bytes.toBytes("value");
final TableName table1 = TableName.valueOf("testDisableAndEnableTable1");
final TableName table2 = TableName.valueOf("testDisableAndEnableTable2");
Table ht1 = TEST_UTIL.createTable(table1, HConstants.CATALOG_FAMILY);
Table ht2 = TEST_UTIL.createTable(table2, HConstants.CATALOG_FAMILY);
Put put = new Put(row);
put.add(HConstants.CATALOG_FAMILY, qualifier, value);
ht1.put(put);
ht2.put(put);
Get get = new Get(row);
get.addColumn(HConstants.CATALOG_FAMILY, qualifier);
ht1.get(get);
ht2.get(get);
this.admin.disableTables("testDisableAndEnableTable.*");
// Test that tables are disabled
get = new Get(row);
get.addColumn(HConstants.CATALOG_FAMILY, qualifier);
boolean ok = false;
try {
ht1.get(get);
ht2.get(get);
} catch (org.apache.hadoop.hbase.DoNotRetryIOException e) {
ok = true;
}
assertTrue(ok);
this.admin.enableTables("testDisableAndEnableTable.*");
// Test that tables are enabled
try {
ht1.get(get);
} catch (IOException e) {
ok = false;
}
try {
ht2.get(get);
} catch (IOException e) {
ok = false;
}
assertTrue(ok);
ht1.close();
ht2.close();
}
/**
* Purge all messages in window for local_addr, which are <= low. Check if the window's highest
* received message is > high: if true, retransmit all messages from high - win.high to sender
*
* @param sender
* @param hd Highest delivered seqno
* @param hr Highest received seqno
*/
protected void stable(Address sender, short conn_id, long hd, long hr) {
SenderEntry entry = send_table.get(sender);
Table<Message> win = entry != null ? entry.sent_msgs : null;
if (win == null) return;
if (log.isTraceEnabled())
log.trace(
new StringBuilder()
.append(local_addr)
.append(" <-- STABLE(")
.append(sender)
.append(": ")
.append(hd)
.append("-")
.append(hr)
.append(", conn_id=" + conn_id)
+ ")");
if (entry.send_conn_id != conn_id) {
log.warn(
local_addr
+ ": my conn_id ("
+ entry.send_conn_id
+ ") != received conn_id ("
+ conn_id
+ "); discarding STABLE message !");
return;
}
win.purge(hd, true);
long win_hr = win.getHighestReceived();
if (win_hr > hr) {
for (long seqno = hr; seqno <= win_hr; seqno++) {
Message msg =
win.get(
seqno); // destination is still the same (the member which sent the STABLE message)
if (msg != null) down_prot.down(new Event(Event.MSG, msg));
}
}
}
protected void handleXmitRequest(Address sender, SeqnoList missing) {
if (log.isTraceEnabled())
log.trace(
new StringBuilder()
.append(local_addr)
.append(" <-- XMIT(")
.append(sender)
.append(": #")
.append(missing)
.append(')'));
SenderEntry entry = send_table.get(sender);
xmit_reqs_received.addAndGet(missing.size());
Table<Message> win = entry != null ? entry.sent_msgs : null;
if (win != null) {
for (long seqno : missing) {
Message msg = win.get(seqno);
if (msg == null) {
if (log.isWarnEnabled() && !local_addr.equals(sender)) {
StringBuilder sb = new StringBuilder();
sb.append("(requester=").append(sender).append(", local_addr=").append(this.local_addr);
sb.append(") message ").append(sender).append("::").append(seqno);
sb.append(" not found in retransmission table of ")
.append(sender)
.append(":\n")
.append(win);
log.warn(sb.toString());
}
continue;
}
down_prot.down(new Event(Event.MSG, msg));
xmit_rsps_sent.incrementAndGet();
}
}
}
@Test
public void testInterrupt50Percent() throws IOException, InterruptedException {
final AtomicInteger noEx = new AtomicInteger(0);
final AtomicInteger badEx = new AtomicInteger(0);
final AtomicInteger noInt = new AtomicInteger(0);
final AtomicInteger done = new AtomicInteger(0);
List<Thread> threads = new ArrayList<Thread>();
final int nbThread = 100;
for (int i = 0; i < nbThread; i++) {
Thread t =
new Thread() {
@Override
public void run() {
try {
Table ht = util.getConnection().getTable(tableName);
Result r = ht.get(new Get(row1));
noEx.incrementAndGet();
} catch (IOException e) {
LOG.info("exception", e);
if (!(e instanceof InterruptedIOException)
|| (e instanceof SocketTimeoutException)) {
badEx.incrementAndGet();
} else {
if (Thread.currentThread().isInterrupted()) {
noInt.incrementAndGet();
LOG.info("The thread should NOT be with the 'interrupt' status.");
}
}
} finally {
done.incrementAndGet();
}
}
};
t.setName("TestClientOperationInterrupt #" + i);
threads.add(t);
t.start();
}
for (int i = 0; i < nbThread / 2; i++) {
threads.get(i).interrupt();
}
boolean stillAlive = true;
while (stillAlive) {
stillAlive = false;
for (Thread t : threads) {
if (t.isAlive()) {
stillAlive = true;
}
}
Threads.sleep(10);
}
Assert.assertFalse(Thread.currentThread().isInterrupted());
Assert.assertTrue(
" noEx: " + noEx.get() + ", badEx=" + badEx.get() + ", noInt=" + noInt.get(),
noEx.get() == nbThread / 2 && badEx.get() == 0);
// The problem here is that we need the server to free its handlers to handle all operations
while (done.get() != nbThread) {
Thread.sleep(1);
}
Table ht = util.getConnection().getTable(tableName);
Result r = ht.get(new Get(row1));
Assert.assertFalse(r.isEmpty());
}