public int logManyTablets(Map<CommitSession, List<Mutation>> mutations) throws IOException {
final Map<CommitSession, List<Mutation>> loggables =
new HashMap<CommitSession, List<Mutation>>(mutations);
for (CommitSession t : mutations.keySet()) {
if (!enabled(t)) loggables.remove(t);
}
if (loggables.size() == 0) return -1;
int seq =
write(
loggables.keySet(),
false,
new Writer() {
@Override
public LoggerOperation write(DfsLogger logger, int ignored) throws Exception {
List<TabletMutations> copy = new ArrayList<TabletMutations>(loggables.size());
for (Entry<CommitSession, List<Mutation>> entry : loggables.entrySet()) {
CommitSession cs = entry.getKey();
copy.add(new TabletMutations(cs.getLogId(), cs.getWALogSeq(), entry.getValue()));
}
return logger.logManyTablets(copy);
}
});
for (List<Mutation> entry : loggables.values()) {
if (entry.size() < 1)
throw new IllegalArgumentException("logManyTablets: logging empty mutation list");
for (Mutation m : entry) {
logSizeEstimate.addAndGet(m.numBytes());
}
}
return seq;
}
public int log(final CommitSession commitSession, final int tabletSeq, final Mutation m)
throws IOException {
if (!enabled(commitSession)) return -1;
int seq =
write(
commitSession,
false,
new Writer() {
@Override
public LoggerOperation write(DfsLogger logger, int ignored) throws Exception {
return logger.log(tabletSeq, commitSession.getLogId(), m);
}
});
logSizeEstimate.addAndGet(m.numBytes());
return seq;
}
@Override
public void client() {
Connector conn = getConnector();
String tableName = getTestProperty("TABLE");
// get batch writer configuration
long maxMemory = Long.parseLong(getTestProperty("MAX_MEMORY"));
long maxLatency = Long.parseLong(getTestProperty("MAX_LATENCY"));
int maxWriteThreads = Integer.parseInt(getTestProperty("NUM_THREADS"));
// create batch writer
BatchWriter bw = null;
try {
bw =
conn.createBatchWriter(
tableName,
new BatchWriterConfig()
.setMaxMemory(maxMemory)
.setMaxLatency(maxLatency, TimeUnit.MILLISECONDS)
.setMaxWriteThreads(maxWriteThreads));
} catch (TableNotFoundException e) {
log.error("Table '" + tableName + "' not found.", e);
}
// configure writing
Random r = new Random();
String ingestInstanceId = UUID.randomUUID().toString();
long numIngestEntries = Long.parseLong(getTestProperty("NUM_ENTRIES"));
long minRow = 0L;
long maxRow = 9223372036854775807L;
int maxColF = 32767;
int maxColQ = 32767;
long count = 0;
long totalBytes = 0;
ColumnVisibility cv = new ColumnVisibility();
// start timer
startTimer();
// write specified number of entries
while (count < numIngestEntries) {
count++;
long rowId = ContinuousIngest.genLong(minRow, maxRow, r);
Mutation m =
ContinuousIngest.genMutation(
rowId,
r.nextInt(maxColF),
r.nextInt(maxColQ),
cv,
ingestInstanceId.getBytes(StandardCharsets.UTF_8),
count,
null,
r,
false);
totalBytes += m.numBytes();
try {
bw.addMutation(m);
} catch (MutationsRejectedException e) {
log.error("Mutations rejected.", e);
System.exit(-1);
}
}
// close writer
try {
bw.close();
} catch (MutationsRejectedException e) {
log.error("Could not close BatchWriter due to mutations being rejected.", e);
System.exit(-1);
}
// stop timer
stopTimer(count, totalBytes);
}
private MutationSet sendMutationsToTabletServer(
String location, Map<KeyExtent, List<Mutation>> tabMuts, TimeoutTracker timeoutTracker)
throws IOException, AccumuloSecurityException, AccumuloServerException {
if (tabMuts.size() == 0) {
return new MutationSet();
}
TInfo tinfo = Tracer.traceInfo();
// TODO remove this
TTransport transport = null;
timeoutTracker.startingWrite();
try {
TabletClientService.Iface client;
if (timeoutTracker.getTimeOut()
< instance.getConfiguration().getTimeInMillis(Property.GENERAL_RPC_TIMEOUT))
client = ThriftUtil.getTServerClient(location, timeoutTracker.getTimeOut());
else client = ThriftUtil.getTServerClient(location, instance.getConfiguration());
try {
MutationSet allFailures = new MutationSet();
if (tabMuts.size() == 1 && tabMuts.values().iterator().next().size() == 1) {
Entry<KeyExtent, List<Mutation>> entry = tabMuts.entrySet().iterator().next();
try {
client.update(
tinfo,
credentials.toThrift(instance),
entry.getKey().toThrift(),
entry.getValue().get(0).toThrift());
} catch (NotServingTabletException e) {
allFailures.addAll(entry.getKey().getTableId().toString(), entry.getValue());
TabletLocator.getLocator(instance, new Text(entry.getKey().getTableId()))
.invalidateCache(entry.getKey());
} catch (ConstraintViolationException e) {
updatedConstraintViolations(
Translator.translate(e.violationSummaries, Translator.TCVST));
}
timeoutTracker.madeProgress();
} else {
long usid = client.startUpdate(tinfo, credentials.toThrift(instance));
List<TMutation> updates = new ArrayList<TMutation>();
for (Entry<KeyExtent, List<Mutation>> entry : tabMuts.entrySet()) {
long size = 0;
Iterator<Mutation> iter = entry.getValue().iterator();
while (iter.hasNext()) {
while (size < MUTATION_BATCH_SIZE && iter.hasNext()) {
Mutation mutation = iter.next();
updates.add(mutation.toThrift());
size += mutation.numBytes();
}
client.applyUpdates(tinfo, usid, entry.getKey().toThrift(), updates);
updates.clear();
size = 0;
}
}
UpdateErrors updateErrors = client.closeUpdate(tinfo, usid);
Map<KeyExtent, Long> failures =
Translator.translate(updateErrors.failedExtents, Translator.TKET);
updatedConstraintViolations(
Translator.translate(updateErrors.violationSummaries, Translator.TCVST));
updateAuthorizationFailures(
Translator.translate(updateErrors.authorizationFailures, Translator.TKET));
long totalCommitted = 0;
for (Entry<KeyExtent, Long> entry : failures.entrySet()) {
KeyExtent failedExtent = entry.getKey();
int numCommitted = (int) (long) entry.getValue();
totalCommitted += numCommitted;
String table = failedExtent.getTableId().toString();
TabletLocator.getLocator(instance, new Text(table)).invalidateCache(failedExtent);
ArrayList<Mutation> mutations = (ArrayList<Mutation>) tabMuts.get(failedExtent);
allFailures.addAll(table, mutations.subList(numCommitted, mutations.size()));
}
if (failures.keySet().containsAll(tabMuts.keySet()) && totalCommitted == 0) {
// nothing was successfully written
timeoutTracker.wroteNothing();
} else {
// successfully wrote something to tablet server
timeoutTracker.madeProgress();
}
}
return allFailures;
} finally {
ThriftUtil.returnClient((TServiceClient) client);
}
} catch (TTransportException e) {
timeoutTracker.errorOccured(e);
throw new IOException(e);
} catch (TApplicationException tae) {
updateServerErrors(location, tae);
throw new AccumuloServerException(location, tae);
} catch (ThriftSecurityException e) {
updateAuthorizationFailures(tabMuts.keySet(), e.code);
throw new AccumuloSecurityException(e.user, e.code, e);
} catch (NoSuchScanIDException e) {
throw new IOException(e);
} catch (TException e) {
throw new IOException(e);
} finally {
ThriftTransportPool.getInstance().returnTransport(transport);
}
}