コード例 #1
0
  @Test
  public void testSimpleWritable() throws IOException {
    RangeInputSplit split =
        new RangeInputSplit(
            "table", "1", new Range(new Key("a"), new Key("b")), new String[] {"localhost"});

    ByteArrayOutputStream baos = new ByteArrayOutputStream();
    DataOutputStream dos = new DataOutputStream(baos);
    split.write(dos);

    RangeInputSplit newSplit = new RangeInputSplit();

    ByteArrayInputStream bais = new ByteArrayInputStream(baos.toByteArray());
    DataInputStream dis = new DataInputStream(bais);
    newSplit.readFields(dis);

    Assert.assertEquals(split.getTableName(), newSplit.getTableName());
    Assert.assertEquals(split.getTableId(), newSplit.getTableId());
    Assert.assertEquals(split.getRange(), newSplit.getRange());
    Assert.assertTrue(Arrays.equals(split.getLocations(), newSplit.getLocations()));
  }
コード例 #2
0
  @Test
  public void testAllFieldsWritable() throws IOException {
    RangeInputSplit split =
        new RangeInputSplit(
            "table", "1", new Range(new Key("a"), new Key("b")), new String[] {"localhost"});

    Set<Pair<Text, Text>> fetchedColumns = new HashSet<Pair<Text, Text>>();

    fetchedColumns.add(new Pair<Text, Text>(new Text("colf1"), new Text("colq1")));
    fetchedColumns.add(new Pair<Text, Text>(new Text("colf2"), new Text("colq2")));

    // Fake some iterators
    ArrayList<IteratorSetting> iterators = new ArrayList<IteratorSetting>();
    IteratorSetting setting = new IteratorSetting(50, SummingCombiner.class);
    setting.addOption("foo", "bar");
    iterators.add(setting);

    setting = new IteratorSetting(100, WholeRowIterator.class);
    setting.addOption("bar", "foo");
    iterators.add(setting);

    split.setTableName("table");
    split.setAuths(new Authorizations("foo"));
    split.setOffline(true);
    split.setIsolatedScan(true);
    split.setUsesLocalIterators(true);
    split.setFetchedColumns(fetchedColumns);
    split.setToken(new PasswordToken("password"));
    split.setPrincipal("root");
    split.setInstanceName("instance");
    DeprecationUtil.setMockInstance(split, true);
    split.setZooKeepers("localhost");
    split.setIterators(iterators);
    split.setLogLevel(Level.WARN);

    ByteArrayOutputStream baos = new ByteArrayOutputStream();
    DataOutputStream dos = new DataOutputStream(baos);
    split.write(dos);

    RangeInputSplit newSplit = new RangeInputSplit();

    ByteArrayInputStream bais = new ByteArrayInputStream(baos.toByteArray());
    DataInputStream dis = new DataInputStream(bais);
    newSplit.readFields(dis);

    Assert.assertEquals(split.getRange(), newSplit.getRange());
    Assert.assertArrayEquals(split.getLocations(), newSplit.getLocations());

    Assert.assertEquals(split.getTableName(), newSplit.getTableName());
    Assert.assertEquals(split.getAuths(), newSplit.getAuths());
    Assert.assertEquals(split.isOffline(), newSplit.isOffline());
    Assert.assertEquals(split.isIsolatedScan(), newSplit.isOffline());
    Assert.assertEquals(split.usesLocalIterators(), newSplit.usesLocalIterators());
    Assert.assertEquals(split.getFetchedColumns(), newSplit.getFetchedColumns());
    Assert.assertEquals(split.getToken(), newSplit.getToken());
    Assert.assertEquals(split.getPrincipal(), newSplit.getPrincipal());
    Assert.assertEquals(split.getInstanceName(), newSplit.getInstanceName());
    Assert.assertEquals(
        DeprecationUtil.isMockInstanceSet(split), DeprecationUtil.isMockInstanceSet(newSplit));
    Assert.assertEquals(split.getZooKeepers(), newSplit.getZooKeepers());
    Assert.assertEquals(split.getIterators(), newSplit.getIterators());
    Assert.assertEquals(split.getLogLevel(), newSplit.getLogLevel());
  }
コード例 #3
0
    @Override
    public void initialize(InputSplit inSplit, TaskAttemptContext attempt) throws IOException {

      split = (RangeInputSplit) inSplit;
      log.debug("Initializing input split: " + split.toString());

      Instance instance = split.getInstance(getClientConfiguration(attempt));
      if (null == instance) {
        instance = getInstance(attempt);
      }

      String principal = split.getPrincipal();
      if (null == principal) {
        principal = getPrincipal(attempt);
      }

      AuthenticationToken token = split.getToken();
      if (null == token) {
        token = getAuthenticationToken(attempt);
      }

      Authorizations authorizations = split.getAuths();
      if (null == authorizations) {
        authorizations = getScanAuthorizations(attempt);
      }
      String classLoaderContext = getClassLoaderContext(attempt);
      String table = split.getTableName();

      // in case the table name changed, we can still use the previous name for terms of
      // configuration,
      // but the scanner will use the table id resolved at job setup time
      InputTableConfig tableConfig = getInputTableConfig(attempt, split.getTableName());

      log.debug("Creating connector with user: "******"Creating scanner for table: " + table);
      log.debug("Authorizations are: " + authorizations);

      if (split instanceof BatchInputSplit) {
        BatchInputSplit batchSplit = (BatchInputSplit) split;

        BatchScanner scanner;
        try {
          // Note: BatchScanner will use at most one thread per tablet, currently BatchInputSplit
          // will not span tablets
          int scanThreads = 1;
          scanner =
              instance
                  .getConnector(principal, token)
                  .createBatchScanner(split.getTableName(), authorizations, scanThreads);
          setupIterators(attempt, scanner, split.getTableName(), split);
          if (null != classLoaderContext) {
            scanner.setClassLoaderContext(classLoaderContext);
          }
        } catch (Exception e) {
          e.printStackTrace();
          throw new IOException(e);
        }

        scanner.setRanges(batchSplit.getRanges());
        scannerBase = scanner;
      } else {
        Scanner scanner;

        Boolean isOffline = split.isOffline();
        if (null == isOffline) {
          isOffline = tableConfig.isOfflineScan();
        }

        Boolean isIsolated = split.isIsolatedScan();
        if (null == isIsolated) {
          isIsolated = tableConfig.shouldUseIsolatedScanners();
        }

        Boolean usesLocalIterators = split.usesLocalIterators();
        if (null == usesLocalIterators) {
          usesLocalIterators = tableConfig.shouldUseLocalIterators();
        }

        try {
          if (isOffline) {
            scanner =
                new OfflineScanner(
                    instance,
                    new Credentials(principal, token),
                    split.getTableId(),
                    authorizations);
          } else if (DeprecationUtil.isMockInstance(instance)) {
            scanner =
                instance
                    .getConnector(principal, token)
                    .createScanner(split.getTableName(), authorizations);
          } else {
            ClientConfiguration clientConf = getClientConfiguration(attempt);
            ClientContext context =
                new ClientContext(instance, new Credentials(principal, token), clientConf);
            scanner = new ScannerImpl(context, split.getTableId(), authorizations);
          }
          if (isIsolated) {
            log.info("Creating isolated scanner");
            scanner = new IsolatedScanner(scanner);
          }
          if (usesLocalIterators) {
            log.info("Using local iterators");
            scanner = new ClientSideIteratorScanner(scanner);
          }

          setupIterators(attempt, scanner, split.getTableName(), split);
        } catch (Exception e) {
          throw new IOException(e);
        }

        scanner.setRange(split.getRange());
        scannerBase = scanner;
      }

      Collection<Pair<Text, Text>> columns = split.getFetchedColumns();
      if (null == columns) {
        columns = tableConfig.getFetchedColumns();
      }

      // setup a scanner within the bounds of this split
      for (Pair<Text, Text> c : columns) {
        if (c.getSecond() != null) {
          log.debug("Fetching column " + c.getFirst() + ":" + c.getSecond());
          scannerBase.fetchColumn(c.getFirst(), c.getSecond());
        } else {
          log.debug("Fetching column family " + c.getFirst());
          scannerBase.fetchColumnFamily(c.getFirst());
        }
      }

      SamplerConfiguration samplerConfig = split.getSamplerConfiguration();
      if (null == samplerConfig) {
        samplerConfig = tableConfig.getSamplerConfiguration();
      }

      if (samplerConfig != null) {
        scannerBase.setSamplerConfiguration(samplerConfig);
      }

      scannerIterator = scannerBase.iterator();
      numKeysRead = 0;
    }