コード例 #1
0
    /**
     * Configures the iterators on a scanner for the given table name.
     *
     * @param job the Hadoop job configuration
     * @param scanner the scanner for which to configure the iterators
     * @param tableName the table name for which the scanner is configured
     * @since 1.7.0
     */
    private void setupIterators(
        JobConf job,
        ScannerBase scanner,
        String tableName,
        org.apache.accumulo.core.client.mapreduce.RangeInputSplit split) {
      List<IteratorSetting> iterators = null;

      if (null == split) {
        iterators = jobIterators(job, tableName);
      } else {
        iterators = split.getIterators();
        if (null == iterators) {
          iterators = jobIterators(job, tableName);
        }
      }

      for (IteratorSetting iterator : iterators) scanner.addScanIterator(iterator);
    }
コード例 #2
0
 @Override
 public float getProgress() throws IOException {
   if (numKeysRead > 0 && currentKey == null) return 1.0f;
   return baseSplit.getProgress(currentKey);
 }
コード例 #3
0
    /** Initialize a scanner over the given input split using this task attempt configuration. */
    public void initialize(InputSplit inSplit, JobConf job) throws IOException {
      baseSplit = (org.apache.accumulo.core.client.mapreduce.RangeInputSplit) inSplit;
      log.debug("Initializing input split: " + baseSplit.toString());

      Instance instance = baseSplit.getInstance(getClientConfiguration(job));
      if (null == instance) {
        instance = getInstance(job);
      }

      String principal = baseSplit.getPrincipal();
      if (null == principal) {
        principal = getPrincipal(job);
      }

      AuthenticationToken token = baseSplit.getToken();
      if (null == token) {
        token = getAuthenticationToken(job);
      }

      Authorizations authorizations = baseSplit.getAuths();
      if (null == authorizations) {
        authorizations = getScanAuthorizations(job);
      }
      String classLoaderContext = getClassLoaderContext(job);
      String table = baseSplit.getTableName();

      // in case the table name changed, we can still use the previous name for terms of
      // configuration,
      // but the scanner will use the table id resolved at job setup time
      InputTableConfig tableConfig = getInputTableConfig(job, baseSplit.getTableName());

      log.debug("Creating connector with user: "******"Creating scanner for table: " + table);
      log.debug("Authorizations are: " + authorizations);

      if (baseSplit instanceof BatchInputSplit) {
        BatchScanner scanner;
        BatchInputSplit multiRangeSplit = (BatchInputSplit) baseSplit;

        try {
          // Note: BatchScanner will use at most one thread per tablet, currently BatchInputSplit
          // will not span tablets
          int scanThreads = 1;
          scanner =
              instance
                  .getConnector(principal, token)
                  .createBatchScanner(baseSplit.getTableName(), authorizations, scanThreads);
          setupIterators(job, scanner, baseSplit.getTableName(), baseSplit);
          if (null != classLoaderContext) {
            scanner.setClassLoaderContext(classLoaderContext);
          }
        } catch (Exception e) {
          throw new IOException(e);
        }

        scanner.setRanges(multiRangeSplit.getRanges());
        scannerBase = scanner;

      } else if (baseSplit instanceof RangeInputSplit) {
        split = (RangeInputSplit) baseSplit;
        Boolean isOffline = baseSplit.isOffline();
        if (null == isOffline) {
          isOffline = tableConfig.isOfflineScan();
        }

        Boolean isIsolated = baseSplit.isIsolatedScan();
        if (null == isIsolated) {
          isIsolated = tableConfig.shouldUseIsolatedScanners();
        }

        Boolean usesLocalIterators = baseSplit.usesLocalIterators();
        if (null == usesLocalIterators) {
          usesLocalIterators = tableConfig.shouldUseLocalIterators();
        }

        Scanner scanner;

        try {
          if (isOffline) {
            scanner =
                new OfflineScanner(
                    instance,
                    new Credentials(principal, token),
                    baseSplit.getTableId(),
                    authorizations);
          } else if (DeprecationUtil.isMockInstance(instance)) {
            scanner =
                instance
                    .getConnector(principal, token)
                    .createScanner(baseSplit.getTableName(), authorizations);
          } else {
            ClientConfiguration clientConf = getClientConfiguration(job);
            ClientContext context =
                new ClientContext(instance, new Credentials(principal, token), clientConf);
            scanner = new ScannerImpl(context, baseSplit.getTableId(), authorizations);
          }
          if (isIsolated) {
            log.info("Creating isolated scanner");
            scanner = new IsolatedScanner(scanner);
          }
          if (usesLocalIterators) {
            log.info("Using local iterators");
            scanner = new ClientSideIteratorScanner(scanner);
          }
          setupIterators(job, scanner, baseSplit.getTableName(), baseSplit);
        } catch (Exception e) {
          throw new IOException(e);
        }

        scanner.setRange(baseSplit.getRange());
        scannerBase = scanner;
      } else {
        throw new IllegalArgumentException(
            "Can not initialize from " + baseSplit.getClass().toString());
      }

      Collection<Pair<Text, Text>> columns = baseSplit.getFetchedColumns();
      if (null == columns) {
        columns = tableConfig.getFetchedColumns();
      }

      // setup a scanner within the bounds of this split
      for (Pair<Text, Text> c : columns) {
        if (c.getSecond() != null) {
          log.debug("Fetching column " + c.getFirst() + ":" + c.getSecond());
          scannerBase.fetchColumn(c.getFirst(), c.getSecond());
        } else {
          log.debug("Fetching column family " + c.getFirst());
          scannerBase.fetchColumnFamily(c.getFirst());
        }
      }

      SamplerConfiguration samplerConfig = baseSplit.getSamplerConfiguration();
      if (null == samplerConfig) {
        samplerConfig = tableConfig.getSamplerConfiguration();
      }

      if (samplerConfig != null) {
        scannerBase.setSamplerConfiguration(samplerConfig);
      }

      scannerIterator = scannerBase.iterator();
      numKeysRead = 0;
    }
コード例 #4
0
 @SuppressWarnings("deprecation")
 public static boolean isMockInstanceSet(RangeInputSplit split) {
   return split.isMockInstance();
 }
コード例 #5
0
 @SuppressWarnings("deprecation")
 public static void setMockInstance(RangeInputSplit split, boolean isMockInstance) {
   split.setMockInstance(isMockInstance);
 }