public static Pair<String, ClientService.Client> getConnection(
ClientContext context, boolean preferCachedConnections, long rpcTimeout)
throws TTransportException {
checkArgument(context != null, "context is null");
// create list of servers
ArrayList<ThriftTransportKey> servers = new ArrayList<ThriftTransportKey>();
// add tservers
Instance instance = context.getInstance();
ZooCache zc =
new ZooCacheFactory()
.getZooCache(instance.getZooKeepers(), instance.getZooKeepersSessionTimeOut());
for (String tserver : zc.getChildren(ZooUtil.getRoot(instance) + Constants.ZTSERVERS)) {
String path = ZooUtil.getRoot(instance) + Constants.ZTSERVERS + "/" + tserver;
byte[] data = ZooUtil.getLockData(zc, path);
if (data != null) {
String strData = new String(data, UTF_8);
if (!strData.equals("master"))
servers.add(
new ThriftTransportKey(
new ServerServices(strData).getAddress(Service.TSERV_CLIENT),
rpcTimeout,
context));
}
}
boolean opened = false;
try {
Pair<String, TTransport> pair =
ThriftTransportPool.getInstance().getAnyTransport(servers, preferCachedConnections);
ClientService.Client client =
ThriftUtil.createClient(new ClientService.Client.Factory(), pair.getSecond());
opened = true;
warnedAboutTServersBeingDown = false;
return new Pair<String, ClientService.Client>(pair.getFirst(), client);
} finally {
if (!opened) {
if (!warnedAboutTServersBeingDown) {
if (servers.isEmpty()) {
log.warn("There are no tablet servers: check that zookeeper and accumulo are running.");
} else {
log.warn("Failed to find an available server in the list of servers: " + servers);
}
warnedAboutTServersBeingDown = true;
}
}
}
}
public static boolean getBatchFromServer(
ClientContext context,
Range range,
KeyExtent extent,
String server,
SortedMap<Key, Value> results,
SortedSet<Column> fetchedColumns,
List<IterInfo> serverSideIteratorList,
Map<String, Map<String, String>> serverSideIteratorOptions,
int size,
Authorizations authorizations,
boolean retry,
long batchTimeOut,
String classLoaderContext)
throws AccumuloException, AccumuloSecurityException, NotServingTabletException {
if (server == null) throw new AccumuloException(new IOException());
final HostAndPort parsedServer = HostAndPort.fromString(server);
try {
TInfo tinfo = Tracer.traceInfo();
TabletClientService.Client client = ThriftUtil.getTServerClient(parsedServer, context);
try {
// not reading whole rows (or stopping on row boundries) so there is no need to enable
// isolation below
ScanState scanState =
new ScanState(
context,
extent.getTableId(),
authorizations,
range,
fetchedColumns,
size,
serverSideIteratorList,
serverSideIteratorOptions,
false,
Constants.SCANNER_DEFAULT_READAHEAD_THRESHOLD,
null,
batchTimeOut,
classLoaderContext);
TabletType ttype = TabletType.type(extent);
boolean waitForWrites = !serversWaitedForWrites.get(ttype).contains(server);
InitialScan isr =
client.startScan(
tinfo,
scanState.context.rpcCreds(),
extent.toThrift(),
scanState.range.toThrift(),
Translator.translate(scanState.columns, Translators.CT),
scanState.size,
scanState.serverSideIteratorList,
scanState.serverSideIteratorOptions,
scanState.authorizations.getAuthorizationsBB(),
waitForWrites,
scanState.isolated,
scanState.readaheadThreshold,
null,
scanState.batchTimeOut,
classLoaderContext);
if (waitForWrites) serversWaitedForWrites.get(ttype).add(server);
Key.decompress(isr.result.results);
for (TKeyValue kv : isr.result.results) results.put(new Key(kv.key), new Value(kv.value));
client.closeScan(tinfo, isr.scanID);
return isr.result.more;
} finally {
ThriftUtil.returnClient(client);
}
} catch (TApplicationException tae) {
throw new AccumuloServerException(server, tae);
} catch (TooManyFilesException e) {
log.debug("Tablet ({}) has too many files {} : {}", extent, server, e.getMessage());
} catch (ThriftSecurityException e) {
log.warn("Security Violation in scan request to {}: {}", server, e.getMessage());
throw new AccumuloSecurityException(e.user, e.code, e);
} catch (TException e) {
log.debug("Error getting transport to {}: {}", server, e.getMessage());
}
throw new AccumuloException("getBatchFromServer: failed");
}
private static List<KeyValue> scan(TabletLocation loc, ScanState scanState, ClientContext context)
throws AccumuloSecurityException, NotServingTabletException, TException,
NoSuchScanIDException, TooManyFilesException, TSampleNotPresentException {
if (scanState.finished) return null;
OpTimer timer = null;
final TInfo tinfo = Tracer.traceInfo();
final HostAndPort parsedLocation = HostAndPort.fromString(loc.tablet_location);
TabletClientService.Client client = ThriftUtil.getTServerClient(parsedLocation, context);
String old = Thread.currentThread().getName();
try {
ScanResult sr;
if (scanState.prevLoc != null && !scanState.prevLoc.equals(loc)) scanState.scanID = null;
scanState.prevLoc = loc;
if (scanState.scanID == null) {
String msg =
"Starting scan tserver="
+ loc.tablet_location
+ " tablet="
+ loc.tablet_extent
+ " range="
+ scanState.range
+ " ssil="
+ scanState.serverSideIteratorList
+ " ssio="
+ scanState.serverSideIteratorOptions;
Thread.currentThread().setName(msg);
if (log.isTraceEnabled()) {
log.trace("tid={} {}", Thread.currentThread().getId(), msg);
timer = new OpTimer().start();
}
TabletType ttype = TabletType.type(loc.tablet_extent);
boolean waitForWrites = !serversWaitedForWrites.get(ttype).contains(loc.tablet_location);
InitialScan is =
client.startScan(
tinfo,
scanState.context.rpcCreds(),
loc.tablet_extent.toThrift(),
scanState.range.toThrift(),
Translator.translate(scanState.columns, Translators.CT),
scanState.size,
scanState.serverSideIteratorList,
scanState.serverSideIteratorOptions,
scanState.authorizations.getAuthorizationsBB(),
waitForWrites,
scanState.isolated,
scanState.readaheadThreshold,
SamplerConfigurationImpl.toThrift(scanState.samplerConfig),
scanState.batchTimeOut,
scanState.classLoaderContext);
if (waitForWrites) serversWaitedForWrites.get(ttype).add(loc.tablet_location);
sr = is.result;
if (sr.more) scanState.scanID = is.scanID;
else client.closeScan(tinfo, is.scanID);
} else {
// log.debug("Calling continue scan : "+scanState.range+" loc = "+loc);
String msg =
"Continuing scan tserver=" + loc.tablet_location + " scanid=" + scanState.scanID;
Thread.currentThread().setName(msg);
if (log.isTraceEnabled()) {
log.trace("tid={} {}", Thread.currentThread().getId(), msg);
timer = new OpTimer().start();
}
sr = client.continueScan(tinfo, scanState.scanID);
if (!sr.more) {
client.closeScan(tinfo, scanState.scanID);
scanState.scanID = null;
}
}
if (!sr.more) {
// log.debug("No more : tab end row = "+loc.tablet_extent.getEndRow()+" range =
// "+scanState.range);
if (loc.tablet_extent.getEndRow() == null) {
scanState.finished = true;
if (timer != null) {
timer.stop();
log.trace(
"tid={} Completely finished scan in {} #results={}",
Thread.currentThread().getId(),
String.format("%.3f secs", timer.scale(TimeUnit.SECONDS)),
sr.results.size());
}
} else if (scanState.range.getEndKey() == null
|| !scanState.range.afterEndKey(
new Key(loc.tablet_extent.getEndRow()).followingKey(PartialKey.ROW))) {
scanState.startRow = loc.tablet_extent.getEndRow();
scanState.skipStartRow = true;
if (timer != null) {
timer.stop();
log.trace(
"tid={} Finished scanning tablet in {} #results={}",
Thread.currentThread().getId(),
String.format("%.3f secs", timer.scale(TimeUnit.SECONDS)),
sr.results.size());
}
} else {
scanState.finished = true;
if (timer != null) {
timer.stop();
log.trace(
"tid={} Completely finished in {} #results={}",
Thread.currentThread().getId(),
String.format("%.3f secs", timer.scale(TimeUnit.SECONDS)),
sr.results.size());
}
}
} else {
if (timer != null) {
timer.stop();
log.trace(
"tid={} Finished scan in {} #results={} scanid={}",
Thread.currentThread().getId(),
String.format("%.3f secs", timer.scale(TimeUnit.SECONDS)),
sr.results.size(),
scanState.scanID);
}
}
Key.decompress(sr.results);
if (sr.results.size() > 0 && !scanState.finished)
scanState.range =
new Range(
new Key(sr.results.get(sr.results.size() - 1).key),
false,
scanState.range.getEndKey(),
scanState.range.isEndKeyInclusive());
List<KeyValue> results = new ArrayList<KeyValue>(sr.results.size());
for (TKeyValue tkv : sr.results) results.add(new KeyValue(new Key(tkv.key), tkv.value));
return results;
} catch (ThriftSecurityException e) {
throw new AccumuloSecurityException(e.user, e.code, e);
} finally {
ThriftUtil.returnClient(client);
Thread.currentThread().setName(old);
}
}