/** Write replication ingest entries for each provided file with the given {@link Status}. */
public static void updateFiles(
Credentials creds, KeyExtent extent, Collection<String> files, Status stat) {
if (log.isDebugEnabled()) {
log.debug(
"Updating replication status for "
+ extent
+ " with "
+ files
+ " using "
+ ProtobufUtil.toString(stat));
}
// TODO could use batch writer, would need to handle failure and retry like update does -
// ACCUMULO-1294
if (files.isEmpty()) {
return;
}
Value v = ProtobufUtil.toValue(stat);
for (String file : files) {
// TODO Can preclude this addition if the extent is for a table we don't need to replicate
update(creds, createUpdateMutation(new Path(file), v, extent), extent);
}
}
@Test
public void waitsUntilEntriesAreReplicated() throws Exception {
Connector conn = inst.getConnector("root", new PasswordToken(""));
conn.tableOperations().create("foo");
Text tableId = new Text(conn.tableOperations().tableIdMap().get("foo"));
String file1 = "/accumulo/wals/tserver+port/" + UUID.randomUUID(),
file2 = "/accumulo/wals/tserver+port/" + UUID.randomUUID();
Status stat =
Status.newBuilder()
.setBegin(0)
.setEnd(10000)
.setInfiniteEnd(false)
.setClosed(false)
.build();
BatchWriter bw = ReplicationTable.getBatchWriter(conn);
Mutation m = new Mutation(file1);
StatusSection.add(m, tableId, ProtobufUtil.toValue(stat));
bw.addMutation(m);
m = new Mutation(file2);
StatusSection.add(m, tableId, ProtobufUtil.toValue(stat));
bw.addMutation(m);
bw.close();
bw = conn.createBatchWriter(MetadataTable.NAME, new BatchWriterConfig());
m = new Mutation(ReplicationSection.getRowPrefix() + file1);
m.put(ReplicationSection.COLF, tableId, ProtobufUtil.toValue(stat));
bw.addMutation(m);
m = new Mutation(ReplicationSection.getRowPrefix() + file2);
m.put(ReplicationSection.COLF, tableId, ProtobufUtil.toValue(stat));
bw.close();
final AtomicBoolean done = new AtomicBoolean(false);
final AtomicBoolean exception = new AtomicBoolean(false);
ClientContext context =
new ClientContext(
inst, new Credentials("root", new PasswordToken("")), new ClientConfiguration());
final ReplicationOperationsImpl roi = new ReplicationOperationsImpl(context);
Thread t =
new Thread(
new Runnable() {
@Override
public void run() {
try {
roi.drain("foo");
} catch (Exception e) {
log.error("Got error", e);
exception.set(true);
}
done.set(true);
}
});
t.start();
// With the records, we shouldn't be drained
Assert.assertFalse(done.get());
bw = conn.createBatchWriter(MetadataTable.NAME, new BatchWriterConfig());
m = new Mutation(ReplicationSection.getRowPrefix() + file1);
m.putDelete(ReplicationSection.COLF, tableId);
bw.addMutation(m);
bw.flush();
Assert.assertFalse(done.get());
m = new Mutation(ReplicationSection.getRowPrefix() + file2);
m.putDelete(ReplicationSection.COLF, tableId);
bw.addMutation(m);
bw.flush();
bw.close();
// Removing metadata entries doesn't change anything
Assert.assertFalse(done.get());
// Remove the replication entries too
bw = ReplicationTable.getBatchWriter(conn);
m = new Mutation(file1);
m.putDelete(StatusSection.NAME, tableId);
bw.addMutation(m);
bw.flush();
Assert.assertFalse(done.get());
m = new Mutation(file2);
m.putDelete(StatusSection.NAME, tableId);
bw.addMutation(m);
bw.flush();
try {
t.join(5000);
} catch (InterruptedException e) {
Assert.fail("ReplicationOperations.drain did not complete");
}
// After both metadata and replication
Assert.assertTrue(done.get());
Assert.assertFalse(exception.get());
}
@Test
public void laterCreatedLogsDontBlockExecution() throws Exception {
Connector conn = inst.getConnector("root", new PasswordToken(""));
conn.tableOperations().create("foo");
Text tableId1 = new Text(conn.tableOperations().tableIdMap().get("foo"));
String file1 = "/accumulo/wals/tserver+port/" + UUID.randomUUID();
Status stat =
Status.newBuilder()
.setBegin(0)
.setEnd(10000)
.setInfiniteEnd(false)
.setClosed(false)
.build();
BatchWriter bw = ReplicationTable.getBatchWriter(conn);
Mutation m = new Mutation(file1);
StatusSection.add(m, tableId1, ProtobufUtil.toValue(stat));
bw.addMutation(m);
bw.close();
bw = conn.createBatchWriter(MetadataTable.NAME, new BatchWriterConfig());
m = new Mutation(ReplicationSection.getRowPrefix() + file1);
m.put(ReplicationSection.COLF, tableId1, ProtobufUtil.toValue(stat));
bw.addMutation(m);
bw.close();
System.out.println("Reading metadata first time");
for (Entry<Key, Value> e : conn.createScanner(MetadataTable.NAME, Authorizations.EMPTY)) {
System.out.println(e.getKey());
}
final AtomicBoolean done = new AtomicBoolean(false);
final AtomicBoolean exception = new AtomicBoolean(false);
ClientContext context =
new ClientContext(
inst, new Credentials("root", new PasswordToken("")), new ClientConfiguration());
final ReplicationOperationsImpl roi = new ReplicationOperationsImpl(context);
Thread t =
new Thread(
new Runnable() {
@Override
public void run() {
try {
roi.drain("foo");
} catch (Exception e) {
log.error("Got error", e);
exception.set(true);
}
done.set(true);
}
});
t.start();
// We need to wait long enough for the table to read once
Thread.sleep(2000);
// Write another file, but also delete the old files
bw = conn.createBatchWriter(MetadataTable.NAME, new BatchWriterConfig());
m =
new Mutation(
ReplicationSection.getRowPrefix() + "/accumulo/wals/tserver+port/" + UUID.randomUUID());
m.put(ReplicationSection.COLF, tableId1, ProtobufUtil.toValue(stat));
bw.addMutation(m);
m = new Mutation(ReplicationSection.getRowPrefix() + file1);
m.putDelete(ReplicationSection.COLF, tableId1);
bw.addMutation(m);
bw.close();
System.out.println("Reading metadata second time");
for (Entry<Key, Value> e : conn.createScanner(MetadataTable.NAME, Authorizations.EMPTY)) {
System.out.println(e.getKey());
}
bw = ReplicationTable.getBatchWriter(conn);
m = new Mutation(file1);
m.putDelete(StatusSection.NAME, tableId1);
bw.addMutation(m);
bw.close();
try {
t.join(5000);
} catch (InterruptedException e) {
Assert.fail("ReplicationOperatiotns.drain did not complete");
}
// We should pass immediately because we aren't waiting on both files to be deleted (just the
// one that we did)
Assert.assertTrue(done.get());
}
@Test
public void inprogressReplicationRecordsBlockExecution() throws Exception {
Connector conn = inst.getConnector("root", new PasswordToken(""));
conn.tableOperations().create("foo");
Text tableId1 = new Text(conn.tableOperations().tableIdMap().get("foo"));
String file1 = "/accumulo/wals/tserver+port/" + UUID.randomUUID();
Status stat =
Status.newBuilder()
.setBegin(0)
.setEnd(10000)
.setInfiniteEnd(false)
.setClosed(false)
.build();
BatchWriter bw = ReplicationTable.getBatchWriter(conn);
Mutation m = new Mutation(file1);
StatusSection.add(m, tableId1, ProtobufUtil.toValue(stat));
bw.addMutation(m);
bw.close();
LogEntry logEntry = new LogEntry();
logEntry.extent = new KeyExtent(new Text(tableId1), null, null);
logEntry.server = "tserver";
logEntry.filename = file1;
logEntry.tabletId = 1;
logEntry.logSet = Arrays.asList(file1);
logEntry.timestamp = System.currentTimeMillis();
bw = conn.createBatchWriter(MetadataTable.NAME, new BatchWriterConfig());
m = new Mutation(ReplicationSection.getRowPrefix() + file1);
m.put(ReplicationSection.COLF, tableId1, ProtobufUtil.toValue(stat));
bw.addMutation(m);
m = new Mutation(logEntry.getRow());
m.put(logEntry.getColumnFamily(), logEntry.getColumnQualifier(), logEntry.getValue());
bw.addMutation(m);
bw.close();
final AtomicBoolean done = new AtomicBoolean(false);
final AtomicBoolean exception = new AtomicBoolean(false);
ClientContext context =
new ClientContext(
inst, new Credentials("root", new PasswordToken("")), new ClientConfiguration());
final ReplicationOperationsImpl roi = new ReplicationOperationsImpl(context);
Thread t =
new Thread(
new Runnable() {
@Override
public void run() {
try {
roi.drain("foo");
} catch (Exception e) {
log.error("Got error", e);
exception.set(true);
}
done.set(true);
}
});
t.start();
// With the records, we shouldn't be drained
Assert.assertFalse(done.get());
Status newStatus =
Status.newBuilder()
.setBegin(1000)
.setEnd(2000)
.setInfiniteEnd(false)
.setClosed(true)
.build();
bw = conn.createBatchWriter(MetadataTable.NAME, new BatchWriterConfig());
m = new Mutation(ReplicationSection.getRowPrefix() + file1);
m.put(ReplicationSection.COLF, tableId1, ProtobufUtil.toValue(newStatus));
bw.addMutation(m);
bw.flush();
// Removing metadata entries doesn't change anything
Assert.assertFalse(done.get());
// Remove the replication entries too
bw = ReplicationTable.getBatchWriter(conn);
m = new Mutation(file1);
m.put(StatusSection.NAME, tableId1, ProtobufUtil.toValue(newStatus));
bw.addMutation(m);
bw.flush();
try {
t.join(5000);
} catch (InterruptedException e) {
Assert.fail("ReplicationOperations.drain did not complete");
}
// New records, but not fully replicated ones don't cause it to complete
Assert.assertFalse(done.get());
Assert.assertFalse(exception.get());
}
@Test
public void dataWasReplicatedToThePeerWithoutDrain() throws Exception {
MiniAccumuloConfigImpl peerCfg =
new MiniAccumuloConfigImpl(
createTestDir(
this.getClass().getName() + "_" + this.testName.getMethodName() + "_peer"),
ROOT_PASSWORD);
peerCfg.setNumTservers(1);
peerCfg.setInstanceName("peer");
updatePeerConfigFromPrimary(getCluster().getConfig(), peerCfg);
peerCfg.setProperty(Property.REPLICATION_NAME, "peer");
MiniAccumuloClusterImpl peerCluster = new MiniAccumuloClusterImpl(peerCfg);
peerCluster.start();
Connector connMaster = getConnector();
Connector connPeer = peerCluster.getConnector("root", new PasswordToken(ROOT_PASSWORD));
String peerUserName = "******";
String peerPassword = "******";
// Create a user on the peer for replication to use
connPeer.securityOperations().createLocalUser(peerUserName, new PasswordToken(peerPassword));
String peerClusterName = "peer";
// ...peer = AccumuloReplicaSystem,instanceName,zookeepers
connMaster
.instanceOperations()
.setProperty(
Property.REPLICATION_PEERS.getKey() + peerClusterName,
ReplicaSystemFactory.getPeerConfigurationValue(
AccumuloReplicaSystem.class,
AccumuloReplicaSystem.buildConfiguration(
peerCluster.getInstanceName(), peerCluster.getZooKeepers())));
// Configure the credentials we should use to authenticate ourselves to the peer for replication
connMaster
.instanceOperations()
.setProperty(Property.REPLICATION_PEER_USER.getKey() + peerClusterName, peerUserName);
connMaster
.instanceOperations()
.setProperty(Property.REPLICATION_PEER_PASSWORD.getKey() + peerClusterName, peerPassword);
String masterTable = "master", peerTable = "peer";
connMaster.tableOperations().create(masterTable);
String masterTableId = connMaster.tableOperations().tableIdMap().get(masterTable);
Assert.assertNotNull(masterTableId);
connPeer.tableOperations().create(peerTable);
String peerTableId = connPeer.tableOperations().tableIdMap().get(peerTable);
Assert.assertNotNull(peerTableId);
// Give our replication user the ability to write to the table
connPeer
.securityOperations()
.grantTablePermission(peerUserName, peerTable, TablePermission.WRITE);
// Replicate this table to the peerClusterName in a table with the peerTableId table id
connMaster
.tableOperations()
.setProperty(masterTable, Property.TABLE_REPLICATION.getKey(), "true");
connMaster
.tableOperations()
.setProperty(
masterTable, Property.TABLE_REPLICATION_TARGET.getKey() + peerClusterName, peerTableId);
// Write some data to table1
BatchWriter bw = connMaster.createBatchWriter(masterTable, new BatchWriterConfig());
for (int rows = 0; rows < 5000; rows++) {
Mutation m = new Mutation(Integer.toString(rows));
for (int cols = 0; cols < 100; cols++) {
String value = Integer.toString(cols);
m.put(value, "", value);
}
bw.addMutation(m);
}
bw.close();
log.info("Wrote all data to master cluster");
Set<String> files = connMaster.replicationOperations().referencedFiles(masterTable);
for (String s : files) {
log.info("Found referenced file for " + masterTable + ": " + s);
}
for (ProcessReference proc : cluster.getProcesses().get(ServerType.TABLET_SERVER)) {
cluster.killProcess(ServerType.TABLET_SERVER, proc);
}
cluster.exec(TabletServer.class);
Iterators.size(connMaster.createScanner(masterTable, Authorizations.EMPTY).iterator());
for (Entry<Key, Value> kv :
connMaster.createScanner(ReplicationTable.NAME, Authorizations.EMPTY)) {
log.debug(
kv.getKey().toStringNoTruncate()
+ " "
+ ProtobufUtil.toString(Status.parseFrom(kv.getValue().get())));
}
connMaster.replicationOperations().drain(masterTable, files);
Scanner master = connMaster.createScanner(masterTable, Authorizations.EMPTY),
peer = connPeer.createScanner(peerTable, Authorizations.EMPTY);
Iterator<Entry<Key, Value>> masterIter = master.iterator(), peerIter = peer.iterator();
Assert.assertTrue("No data in master table", masterIter.hasNext());
Assert.assertTrue("No data in peer table", peerIter.hasNext());
while (masterIter.hasNext() && peerIter.hasNext()) {
Entry<Key, Value> masterEntry = masterIter.next(), peerEntry = peerIter.next();
Assert.assertEquals(
peerEntry.getKey() + " was not equal to " + peerEntry.getKey(),
0,
masterEntry.getKey().compareTo(peerEntry.getKey(), PartialKey.ROW_COLFAM_COLQUAL_COLVIS));
Assert.assertEquals(masterEntry.getValue(), peerEntry.getValue());
}
Assert.assertFalse("Had more data to read from the master", masterIter.hasNext());
Assert.assertFalse("Had more data to read from the peer", peerIter.hasNext());
peerCluster.stop();
}
@Test
public void dataWasReplicatedToThePeer() throws Exception {
MiniAccumuloConfigImpl peerCfg =
new MiniAccumuloConfigImpl(
createTestDir(
this.getClass().getName() + "_" + this.testName.getMethodName() + "_peer"),
ROOT_PASSWORD);
peerCfg.setNumTservers(1);
peerCfg.setInstanceName("peer");
updatePeerConfigFromPrimary(getCluster().getConfig(), peerCfg);
peerCfg.setProperty(Property.REPLICATION_NAME, "peer");
MiniAccumuloClusterImpl peerCluster = new MiniAccumuloClusterImpl(peerCfg);
peerCluster.start();
try {
final Connector connMaster = getConnector();
final Connector connPeer = peerCluster.getConnector("root", new PasswordToken(ROOT_PASSWORD));
ReplicationTable.setOnline(connMaster);
String peerUserName = "******", peerPassword = "******";
String peerClusterName = "peer";
connPeer.securityOperations().createLocalUser(peerUserName, new PasswordToken(peerPassword));
connMaster
.instanceOperations()
.setProperty(Property.REPLICATION_PEER_USER.getKey() + peerClusterName, peerUserName);
connMaster
.instanceOperations()
.setProperty(Property.REPLICATION_PEER_PASSWORD.getKey() + peerClusterName, peerPassword);
// ...peer = AccumuloReplicaSystem,instanceName,zookeepers
connMaster
.instanceOperations()
.setProperty(
Property.REPLICATION_PEERS.getKey() + peerClusterName,
ReplicaSystemFactory.getPeerConfigurationValue(
AccumuloReplicaSystem.class,
AccumuloReplicaSystem.buildConfiguration(
peerCluster.getInstanceName(), peerCluster.getZooKeepers())));
final String masterTable = "master", peerTable = "peer";
connMaster.tableOperations().create(masterTable);
String masterTableId = connMaster.tableOperations().tableIdMap().get(masterTable);
Assert.assertNotNull(masterTableId);
connPeer.tableOperations().create(peerTable);
String peerTableId = connPeer.tableOperations().tableIdMap().get(peerTable);
Assert.assertNotNull(peerTableId);
connPeer
.securityOperations()
.grantTablePermission(peerUserName, peerTable, TablePermission.WRITE);
// Replicate this table to the peerClusterName in a table with the peerTableId table id
connMaster
.tableOperations()
.setProperty(masterTable, Property.TABLE_REPLICATION.getKey(), "true");
connMaster
.tableOperations()
.setProperty(
masterTable,
Property.TABLE_REPLICATION_TARGET.getKey() + peerClusterName,
peerTableId);
// Wait for zookeeper updates (configuration) to propagate
sleepUninterruptibly(3, TimeUnit.SECONDS);
// Write some data to table1
BatchWriter bw = connMaster.createBatchWriter(masterTable, new BatchWriterConfig());
for (int rows = 0; rows < 5000; rows++) {
Mutation m = new Mutation(Integer.toString(rows));
for (int cols = 0; cols < 100; cols++) {
String value = Integer.toString(cols);
m.put(value, "", value);
}
bw.addMutation(m);
}
bw.close();
log.info("Wrote all data to master cluster");
final Set<String> filesNeedingReplication =
connMaster.replicationOperations().referencedFiles(masterTable);
for (ProcessReference proc : cluster.getProcesses().get(ServerType.TABLET_SERVER)) {
cluster.killProcess(ServerType.TABLET_SERVER, proc);
}
cluster.exec(TabletServer.class);
log.info("TabletServer restarted");
Iterators.size(ReplicationTable.getScanner(connMaster).iterator());
log.info("TabletServer is online");
log.info("");
log.info("Fetching metadata records:");
for (Entry<Key, Value> kv :
connMaster.createScanner(MetadataTable.NAME, Authorizations.EMPTY)) {
if (ReplicationSection.COLF.equals(kv.getKey().getColumnFamily())) {
log.info(
kv.getKey().toStringNoTruncate()
+ " "
+ ProtobufUtil.toString(Status.parseFrom(kv.getValue().get())));
} else {
log.info(kv.getKey().toStringNoTruncate() + " " + kv.getValue());
}
}
log.info("");
log.info("Fetching replication records:");
for (Entry<Key, Value> kv : ReplicationTable.getScanner(connMaster)) {
log.info(
kv.getKey().toStringNoTruncate()
+ " "
+ ProtobufUtil.toString(Status.parseFrom(kv.getValue().get())));
}
Future<Boolean> future =
executor.submit(
new Callable<Boolean>() {
@Override
public Boolean call() throws Exception {
connMaster.replicationOperations().drain(masterTable, filesNeedingReplication);
log.info("Drain completed");
return true;
}
});
long timeoutSeconds = timeoutFactor * 30;
try {
future.get(timeoutSeconds, TimeUnit.SECONDS);
} catch (TimeoutException e) {
future.cancel(true);
Assert.fail("Drain did not finish within " + timeoutSeconds + " seconds");
}
log.info("drain completed");
log.info("");
log.info("Fetching metadata records:");
for (Entry<Key, Value> kv :
connMaster.createScanner(MetadataTable.NAME, Authorizations.EMPTY)) {
if (ReplicationSection.COLF.equals(kv.getKey().getColumnFamily())) {
log.info(
kv.getKey().toStringNoTruncate()
+ " "
+ ProtobufUtil.toString(Status.parseFrom(kv.getValue().get())));
} else {
log.info(kv.getKey().toStringNoTruncate() + " " + kv.getValue());
}
}
log.info("");
log.info("Fetching replication records:");
for (Entry<Key, Value> kv : ReplicationTable.getScanner(connMaster)) {
log.info(
kv.getKey().toStringNoTruncate()
+ " "
+ ProtobufUtil.toString(Status.parseFrom(kv.getValue().get())));
}
Scanner master = connMaster.createScanner(masterTable, Authorizations.EMPTY),
peer = connPeer.createScanner(peerTable, Authorizations.EMPTY);
Iterator<Entry<Key, Value>> masterIter = master.iterator(), peerIter = peer.iterator();
Entry<Key, Value> masterEntry = null, peerEntry = null;
while (masterIter.hasNext() && peerIter.hasNext()) {
masterEntry = masterIter.next();
peerEntry = peerIter.next();
Assert.assertEquals(
masterEntry.getKey() + " was not equal to " + peerEntry.getKey(),
0,
masterEntry
.getKey()
.compareTo(peerEntry.getKey(), PartialKey.ROW_COLFAM_COLQUAL_COLVIS));
Assert.assertEquals(masterEntry.getValue(), peerEntry.getValue());
}
log.info("Last master entry: " + masterEntry);
log.info("Last peer entry: " + peerEntry);
Assert.assertFalse("Had more data to read from the master", masterIter.hasNext());
Assert.assertFalse("Had more data to read from the peer", peerIter.hasNext());
} finally {
peerCluster.stop();
}
}
