public static Pair<String, ClientService.Client> getConnection( ClientContext context, boolean preferCachedConnections, long rpcTimeout) throws TTransportException { checkArgument(context != null, "context is null"); // create list of servers ArrayList<ThriftTransportKey> servers = new ArrayList<ThriftTransportKey>(); // add tservers Instance instance = context.getInstance(); ZooCache zc = new ZooCacheFactory() .getZooCache(instance.getZooKeepers(), instance.getZooKeepersSessionTimeOut()); for (String tserver : zc.getChildren(ZooUtil.getRoot(instance) + Constants.ZTSERVERS)) { String path = ZooUtil.getRoot(instance) + Constants.ZTSERVERS + "/" + tserver; byte[] data = ZooUtil.getLockData(zc, path); if (data != null) { String strData = new String(data, UTF_8); if (!strData.equals("master")) servers.add( new ThriftTransportKey( new ServerServices(strData).getAddress(Service.TSERV_CLIENT), rpcTimeout, context)); } } boolean opened = false; try { Pair<String, TTransport> pair = ThriftTransportPool.getInstance().getAnyTransport(servers, preferCachedConnections); ClientService.Client client = ThriftUtil.createClient(new ClientService.Client.Factory(), pair.getSecond()); opened = true; warnedAboutTServersBeingDown = false; return new Pair<String, ClientService.Client>(pair.getFirst(), client); } finally { if (!opened) { if (!warnedAboutTServersBeingDown) { if (servers.isEmpty()) { log.warn("There are no tablet servers: check that zookeeper and accumulo are running."); } else { log.warn("Failed to find an available server in the list of servers: " + servers); } warnedAboutTServersBeingDown = true; } } } }
public static boolean getBatchFromServer( ClientContext context, Range range, KeyExtent extent, String server, SortedMap<Key, Value> results, SortedSet<Column> fetchedColumns, List<IterInfo> serverSideIteratorList, Map<String, Map<String, String>> serverSideIteratorOptions, int size, Authorizations authorizations, boolean retry, long batchTimeOut, String classLoaderContext) throws AccumuloException, AccumuloSecurityException, NotServingTabletException { if (server == null) throw new AccumuloException(new IOException()); final HostAndPort parsedServer = HostAndPort.fromString(server); try { TInfo tinfo = Tracer.traceInfo(); TabletClientService.Client client = ThriftUtil.getTServerClient(parsedServer, context); try { // not reading whole rows (or stopping on row boundries) so there is no need to enable // isolation below ScanState scanState = new ScanState( context, extent.getTableId(), authorizations, range, fetchedColumns, size, serverSideIteratorList, serverSideIteratorOptions, false, Constants.SCANNER_DEFAULT_READAHEAD_THRESHOLD, null, batchTimeOut, classLoaderContext); TabletType ttype = TabletType.type(extent); boolean waitForWrites = !serversWaitedForWrites.get(ttype).contains(server); InitialScan isr = client.startScan( tinfo, scanState.context.rpcCreds(), extent.toThrift(), scanState.range.toThrift(), Translator.translate(scanState.columns, Translators.CT), scanState.size, scanState.serverSideIteratorList, scanState.serverSideIteratorOptions, scanState.authorizations.getAuthorizationsBB(), waitForWrites, scanState.isolated, scanState.readaheadThreshold, null, scanState.batchTimeOut, classLoaderContext); if (waitForWrites) serversWaitedForWrites.get(ttype).add(server); Key.decompress(isr.result.results); for (TKeyValue kv : isr.result.results) results.put(new Key(kv.key), new Value(kv.value)); client.closeScan(tinfo, isr.scanID); return isr.result.more; } finally { ThriftUtil.returnClient(client); } } catch (TApplicationException tae) { throw new AccumuloServerException(server, tae); } catch (TooManyFilesException e) { log.debug("Tablet ({}) has too many files {} : {}", extent, server, e.getMessage()); } catch (ThriftSecurityException e) { log.warn("Security Violation in scan request to {}: {}", server, e.getMessage()); throw new AccumuloSecurityException(e.user, e.code, e); } catch (TException e) { log.debug("Error getting transport to {}: {}", server, e.getMessage()); } throw new AccumuloException("getBatchFromServer: failed"); }
private static List<KeyValue> scan(TabletLocation loc, ScanState scanState, ClientContext context) throws AccumuloSecurityException, NotServingTabletException, TException, NoSuchScanIDException, TooManyFilesException, TSampleNotPresentException { if (scanState.finished) return null; OpTimer timer = null; final TInfo tinfo = Tracer.traceInfo(); final HostAndPort parsedLocation = HostAndPort.fromString(loc.tablet_location); TabletClientService.Client client = ThriftUtil.getTServerClient(parsedLocation, context); String old = Thread.currentThread().getName(); try { ScanResult sr; if (scanState.prevLoc != null && !scanState.prevLoc.equals(loc)) scanState.scanID = null; scanState.prevLoc = loc; if (scanState.scanID == null) { String msg = "Starting scan tserver=" + loc.tablet_location + " tablet=" + loc.tablet_extent + " range=" + scanState.range + " ssil=" + scanState.serverSideIteratorList + " ssio=" + scanState.serverSideIteratorOptions; Thread.currentThread().setName(msg); if (log.isTraceEnabled()) { log.trace("tid={} {}", Thread.currentThread().getId(), msg); timer = new OpTimer().start(); } TabletType ttype = TabletType.type(loc.tablet_extent); boolean waitForWrites = !serversWaitedForWrites.get(ttype).contains(loc.tablet_location); InitialScan is = client.startScan( tinfo, scanState.context.rpcCreds(), loc.tablet_extent.toThrift(), scanState.range.toThrift(), Translator.translate(scanState.columns, Translators.CT), scanState.size, scanState.serverSideIteratorList, scanState.serverSideIteratorOptions, scanState.authorizations.getAuthorizationsBB(), waitForWrites, scanState.isolated, scanState.readaheadThreshold, SamplerConfigurationImpl.toThrift(scanState.samplerConfig), scanState.batchTimeOut, scanState.classLoaderContext); if (waitForWrites) serversWaitedForWrites.get(ttype).add(loc.tablet_location); sr = is.result; if (sr.more) scanState.scanID = is.scanID; else client.closeScan(tinfo, is.scanID); } else { // log.debug("Calling continue scan : "+scanState.range+" loc = "+loc); String msg = "Continuing scan tserver=" + loc.tablet_location + " scanid=" + scanState.scanID; Thread.currentThread().setName(msg); if (log.isTraceEnabled()) { log.trace("tid={} {}", Thread.currentThread().getId(), msg); timer = new OpTimer().start(); } sr = client.continueScan(tinfo, scanState.scanID); if (!sr.more) { client.closeScan(tinfo, scanState.scanID); scanState.scanID = null; } } if (!sr.more) { // log.debug("No more : tab end row = "+loc.tablet_extent.getEndRow()+" range = // "+scanState.range); if (loc.tablet_extent.getEndRow() == null) { scanState.finished = true; if (timer != null) { timer.stop(); log.trace( "tid={} Completely finished scan in {} #results={}", Thread.currentThread().getId(), String.format("%.3f secs", timer.scale(TimeUnit.SECONDS)), sr.results.size()); } } else if (scanState.range.getEndKey() == null || !scanState.range.afterEndKey( new Key(loc.tablet_extent.getEndRow()).followingKey(PartialKey.ROW))) { scanState.startRow = loc.tablet_extent.getEndRow(); scanState.skipStartRow = true; if (timer != null) { timer.stop(); log.trace( "tid={} Finished scanning tablet in {} #results={}", Thread.currentThread().getId(), String.format("%.3f secs", timer.scale(TimeUnit.SECONDS)), sr.results.size()); } } else { scanState.finished = true; if (timer != null) { timer.stop(); log.trace( "tid={} Completely finished in {} #results={}", Thread.currentThread().getId(), String.format("%.3f secs", timer.scale(TimeUnit.SECONDS)), sr.results.size()); } } } else { if (timer != null) { timer.stop(); log.trace( "tid={} Finished scan in {} #results={} scanid={}", Thread.currentThread().getId(), String.format("%.3f secs", timer.scale(TimeUnit.SECONDS)), sr.results.size(), scanState.scanID); } } Key.decompress(sr.results); if (sr.results.size() > 0 && !scanState.finished) scanState.range = new Range( new Key(sr.results.get(sr.results.size() - 1).key), false, scanState.range.getEndKey(), scanState.range.isEndKeyInclusive()); List<KeyValue> results = new ArrayList<KeyValue>(sr.results.size()); for (TKeyValue tkv : sr.results) results.add(new KeyValue(new Key(tkv.key), tkv.value)); return results; } catch (ThriftSecurityException e) { throw new AccumuloSecurityException(e.user, e.code, e); } finally { ThriftUtil.returnClient(client); Thread.currentThread().setName(old); } }