Esempio n. 1
0
  public static Pair<String, ClientService.Client> getConnection(
      ClientContext context, boolean preferCachedConnections, long rpcTimeout)
      throws TTransportException {
    checkArgument(context != null, "context is null");
    // create list of servers
    ArrayList<ThriftTransportKey> servers = new ArrayList<ThriftTransportKey>();

    // add tservers
    Instance instance = context.getInstance();
    ZooCache zc =
        new ZooCacheFactory()
            .getZooCache(instance.getZooKeepers(), instance.getZooKeepersSessionTimeOut());
    for (String tserver : zc.getChildren(ZooUtil.getRoot(instance) + Constants.ZTSERVERS)) {
      String path = ZooUtil.getRoot(instance) + Constants.ZTSERVERS + "/" + tserver;
      byte[] data = ZooUtil.getLockData(zc, path);
      if (data != null) {
        String strData = new String(data, UTF_8);
        if (!strData.equals("master"))
          servers.add(
              new ThriftTransportKey(
                  new ServerServices(strData).getAddress(Service.TSERV_CLIENT),
                  rpcTimeout,
                  context));
      }
    }

    boolean opened = false;
    try {
      Pair<String, TTransport> pair =
          ThriftTransportPool.getInstance().getAnyTransport(servers, preferCachedConnections);
      ClientService.Client client =
          ThriftUtil.createClient(new ClientService.Client.Factory(), pair.getSecond());
      opened = true;
      warnedAboutTServersBeingDown = false;
      return new Pair<String, ClientService.Client>(pair.getFirst(), client);
    } finally {
      if (!opened) {
        if (!warnedAboutTServersBeingDown) {
          if (servers.isEmpty()) {
            log.warn("There are no tablet servers: check that zookeeper and accumulo are running.");
          } else {
            log.warn("Failed to find an available server in the list of servers: " + servers);
          }
          warnedAboutTServersBeingDown = true;
        }
      }
    }
  }
Esempio n. 2
0
  public static boolean getBatchFromServer(
      ClientContext context,
      Range range,
      KeyExtent extent,
      String server,
      SortedMap<Key, Value> results,
      SortedSet<Column> fetchedColumns,
      List<IterInfo> serverSideIteratorList,
      Map<String, Map<String, String>> serverSideIteratorOptions,
      int size,
      Authorizations authorizations,
      boolean retry,
      long batchTimeOut,
      String classLoaderContext)
      throws AccumuloException, AccumuloSecurityException, NotServingTabletException {
    if (server == null) throw new AccumuloException(new IOException());

    final HostAndPort parsedServer = HostAndPort.fromString(server);
    try {
      TInfo tinfo = Tracer.traceInfo();
      TabletClientService.Client client = ThriftUtil.getTServerClient(parsedServer, context);
      try {
        // not reading whole rows (or stopping on row boundries) so there is no need to enable
        // isolation below
        ScanState scanState =
            new ScanState(
                context,
                extent.getTableId(),
                authorizations,
                range,
                fetchedColumns,
                size,
                serverSideIteratorList,
                serverSideIteratorOptions,
                false,
                Constants.SCANNER_DEFAULT_READAHEAD_THRESHOLD,
                null,
                batchTimeOut,
                classLoaderContext);

        TabletType ttype = TabletType.type(extent);
        boolean waitForWrites = !serversWaitedForWrites.get(ttype).contains(server);
        InitialScan isr =
            client.startScan(
                tinfo,
                scanState.context.rpcCreds(),
                extent.toThrift(),
                scanState.range.toThrift(),
                Translator.translate(scanState.columns, Translators.CT),
                scanState.size,
                scanState.serverSideIteratorList,
                scanState.serverSideIteratorOptions,
                scanState.authorizations.getAuthorizationsBB(),
                waitForWrites,
                scanState.isolated,
                scanState.readaheadThreshold,
                null,
                scanState.batchTimeOut,
                classLoaderContext);
        if (waitForWrites) serversWaitedForWrites.get(ttype).add(server);

        Key.decompress(isr.result.results);

        for (TKeyValue kv : isr.result.results) results.put(new Key(kv.key), new Value(kv.value));

        client.closeScan(tinfo, isr.scanID);

        return isr.result.more;
      } finally {
        ThriftUtil.returnClient(client);
      }
    } catch (TApplicationException tae) {
      throw new AccumuloServerException(server, tae);
    } catch (TooManyFilesException e) {
      log.debug("Tablet ({}) has too many files {} : {}", extent, server, e.getMessage());
    } catch (ThriftSecurityException e) {
      log.warn("Security Violation in scan request to {}: {}", server, e.getMessage());
      throw new AccumuloSecurityException(e.user, e.code, e);
    } catch (TException e) {
      log.debug("Error getting transport to {}: {}", server, e.getMessage());
    }

    throw new AccumuloException("getBatchFromServer: failed");
  }
Esempio n. 3
0
  private static List<KeyValue> scan(TabletLocation loc, ScanState scanState, ClientContext context)
      throws AccumuloSecurityException, NotServingTabletException, TException,
          NoSuchScanIDException, TooManyFilesException, TSampleNotPresentException {
    if (scanState.finished) return null;

    OpTimer timer = null;

    final TInfo tinfo = Tracer.traceInfo();
    final HostAndPort parsedLocation = HostAndPort.fromString(loc.tablet_location);
    TabletClientService.Client client = ThriftUtil.getTServerClient(parsedLocation, context);

    String old = Thread.currentThread().getName();
    try {
      ScanResult sr;

      if (scanState.prevLoc != null && !scanState.prevLoc.equals(loc)) scanState.scanID = null;

      scanState.prevLoc = loc;

      if (scanState.scanID == null) {
        String msg =
            "Starting scan tserver="
                + loc.tablet_location
                + " tablet="
                + loc.tablet_extent
                + " range="
                + scanState.range
                + " ssil="
                + scanState.serverSideIteratorList
                + " ssio="
                + scanState.serverSideIteratorOptions;
        Thread.currentThread().setName(msg);

        if (log.isTraceEnabled()) {
          log.trace("tid={} {}", Thread.currentThread().getId(), msg);
          timer = new OpTimer().start();
        }

        TabletType ttype = TabletType.type(loc.tablet_extent);
        boolean waitForWrites = !serversWaitedForWrites.get(ttype).contains(loc.tablet_location);

        InitialScan is =
            client.startScan(
                tinfo,
                scanState.context.rpcCreds(),
                loc.tablet_extent.toThrift(),
                scanState.range.toThrift(),
                Translator.translate(scanState.columns, Translators.CT),
                scanState.size,
                scanState.serverSideIteratorList,
                scanState.serverSideIteratorOptions,
                scanState.authorizations.getAuthorizationsBB(),
                waitForWrites,
                scanState.isolated,
                scanState.readaheadThreshold,
                SamplerConfigurationImpl.toThrift(scanState.samplerConfig),
                scanState.batchTimeOut,
                scanState.classLoaderContext);
        if (waitForWrites) serversWaitedForWrites.get(ttype).add(loc.tablet_location);

        sr = is.result;

        if (sr.more) scanState.scanID = is.scanID;
        else client.closeScan(tinfo, is.scanID);

      } else {
        // log.debug("Calling continue scan : "+scanState.range+" loc = "+loc);
        String msg =
            "Continuing scan tserver=" + loc.tablet_location + " scanid=" + scanState.scanID;
        Thread.currentThread().setName(msg);

        if (log.isTraceEnabled()) {
          log.trace("tid={} {}", Thread.currentThread().getId(), msg);
          timer = new OpTimer().start();
        }

        sr = client.continueScan(tinfo, scanState.scanID);
        if (!sr.more) {
          client.closeScan(tinfo, scanState.scanID);
          scanState.scanID = null;
        }
      }

      if (!sr.more) {
        // log.debug("No more : tab end row = "+loc.tablet_extent.getEndRow()+" range =
        // "+scanState.range);
        if (loc.tablet_extent.getEndRow() == null) {
          scanState.finished = true;

          if (timer != null) {
            timer.stop();
            log.trace(
                "tid={} Completely finished scan in {} #results={}",
                Thread.currentThread().getId(),
                String.format("%.3f secs", timer.scale(TimeUnit.SECONDS)),
                sr.results.size());
          }

        } else if (scanState.range.getEndKey() == null
            || !scanState.range.afterEndKey(
                new Key(loc.tablet_extent.getEndRow()).followingKey(PartialKey.ROW))) {
          scanState.startRow = loc.tablet_extent.getEndRow();
          scanState.skipStartRow = true;

          if (timer != null) {
            timer.stop();
            log.trace(
                "tid={} Finished scanning tablet in {} #results={}",
                Thread.currentThread().getId(),
                String.format("%.3f secs", timer.scale(TimeUnit.SECONDS)),
                sr.results.size());
          }
        } else {
          scanState.finished = true;
          if (timer != null) {
            timer.stop();
            log.trace(
                "tid={} Completely finished in {} #results={}",
                Thread.currentThread().getId(),
                String.format("%.3f secs", timer.scale(TimeUnit.SECONDS)),
                sr.results.size());
          }
        }
      } else {
        if (timer != null) {
          timer.stop();
          log.trace(
              "tid={} Finished scan in {} #results={} scanid={}",
              Thread.currentThread().getId(),
              String.format("%.3f secs", timer.scale(TimeUnit.SECONDS)),
              sr.results.size(),
              scanState.scanID);
        }
      }

      Key.decompress(sr.results);

      if (sr.results.size() > 0 && !scanState.finished)
        scanState.range =
            new Range(
                new Key(sr.results.get(sr.results.size() - 1).key),
                false,
                scanState.range.getEndKey(),
                scanState.range.isEndKeyInclusive());

      List<KeyValue> results = new ArrayList<KeyValue>(sr.results.size());
      for (TKeyValue tkv : sr.results) results.add(new KeyValue(new Key(tkv.key), tkv.value));

      return results;

    } catch (ThriftSecurityException e) {
      throw new AccumuloSecurityException(e.user, e.code, e);
    } finally {
      ThriftUtil.returnClient(client);
      Thread.currentThread().setName(old);
    }
  }