@Test
public void testSimpleWritable() throws IOException {
RangeInputSplit split =
new RangeInputSplit(
"table", "1", new Range(new Key("a"), new Key("b")), new String[] {"localhost"});
ByteArrayOutputStream baos = new ByteArrayOutputStream();
DataOutputStream dos = new DataOutputStream(baos);
split.write(dos);
RangeInputSplit newSplit = new RangeInputSplit();
ByteArrayInputStream bais = new ByteArrayInputStream(baos.toByteArray());
DataInputStream dis = new DataInputStream(bais);
newSplit.readFields(dis);
Assert.assertEquals(split.getTableName(), newSplit.getTableName());
Assert.assertEquals(split.getTableId(), newSplit.getTableId());
Assert.assertEquals(split.getRange(), newSplit.getRange());
Assert.assertTrue(Arrays.equals(split.getLocations(), newSplit.getLocations()));
}
@Test
public void testAllFieldsWritable() throws IOException {
RangeInputSplit split =
new RangeInputSplit(
"table", "1", new Range(new Key("a"), new Key("b")), new String[] {"localhost"});
Set<Pair<Text, Text>> fetchedColumns = new HashSet<Pair<Text, Text>>();
fetchedColumns.add(new Pair<Text, Text>(new Text("colf1"), new Text("colq1")));
fetchedColumns.add(new Pair<Text, Text>(new Text("colf2"), new Text("colq2")));
// Fake some iterators
ArrayList<IteratorSetting> iterators = new ArrayList<IteratorSetting>();
IteratorSetting setting = new IteratorSetting(50, SummingCombiner.class);
setting.addOption("foo", "bar");
iterators.add(setting);
setting = new IteratorSetting(100, WholeRowIterator.class);
setting.addOption("bar", "foo");
iterators.add(setting);
split.setTableName("table");
split.setAuths(new Authorizations("foo"));
split.setOffline(true);
split.setIsolatedScan(true);
split.setUsesLocalIterators(true);
split.setFetchedColumns(fetchedColumns);
split.setToken(new PasswordToken("password"));
split.setPrincipal("root");
split.setInstanceName("instance");
DeprecationUtil.setMockInstance(split, true);
split.setZooKeepers("localhost");
split.setIterators(iterators);
split.setLogLevel(Level.WARN);
ByteArrayOutputStream baos = new ByteArrayOutputStream();
DataOutputStream dos = new DataOutputStream(baos);
split.write(dos);
RangeInputSplit newSplit = new RangeInputSplit();
ByteArrayInputStream bais = new ByteArrayInputStream(baos.toByteArray());
DataInputStream dis = new DataInputStream(bais);
newSplit.readFields(dis);
Assert.assertEquals(split.getRange(), newSplit.getRange());
Assert.assertArrayEquals(split.getLocations(), newSplit.getLocations());
Assert.assertEquals(split.getTableName(), newSplit.getTableName());
Assert.assertEquals(split.getAuths(), newSplit.getAuths());
Assert.assertEquals(split.isOffline(), newSplit.isOffline());
Assert.assertEquals(split.isIsolatedScan(), newSplit.isOffline());
Assert.assertEquals(split.usesLocalIterators(), newSplit.usesLocalIterators());
Assert.assertEquals(split.getFetchedColumns(), newSplit.getFetchedColumns());
Assert.assertEquals(split.getToken(), newSplit.getToken());
Assert.assertEquals(split.getPrincipal(), newSplit.getPrincipal());
Assert.assertEquals(split.getInstanceName(), newSplit.getInstanceName());
Assert.assertEquals(
DeprecationUtil.isMockInstanceSet(split), DeprecationUtil.isMockInstanceSet(newSplit));
Assert.assertEquals(split.getZooKeepers(), newSplit.getZooKeepers());
Assert.assertEquals(split.getIterators(), newSplit.getIterators());
Assert.assertEquals(split.getLogLevel(), newSplit.getLogLevel());
}
@Override
public void initialize(InputSplit inSplit, TaskAttemptContext attempt) throws IOException {
split = (RangeInputSplit) inSplit;
log.debug("Initializing input split: " + split.toString());
Instance instance = split.getInstance(getClientConfiguration(attempt));
if (null == instance) {
instance = getInstance(attempt);
}
String principal = split.getPrincipal();
if (null == principal) {
principal = getPrincipal(attempt);
}
AuthenticationToken token = split.getToken();
if (null == token) {
token = getAuthenticationToken(attempt);
}
Authorizations authorizations = split.getAuths();
if (null == authorizations) {
authorizations = getScanAuthorizations(attempt);
}
String classLoaderContext = getClassLoaderContext(attempt);
String table = split.getTableName();
// in case the table name changed, we can still use the previous name for terms of
// configuration,
// but the scanner will use the table id resolved at job setup time
InputTableConfig tableConfig = getInputTableConfig(attempt, split.getTableName());
log.debug("Creating connector with user: "******"Creating scanner for table: " + table);
log.debug("Authorizations are: " + authorizations);
if (split instanceof BatchInputSplit) {
BatchInputSplit batchSplit = (BatchInputSplit) split;
BatchScanner scanner;
try {
// Note: BatchScanner will use at most one thread per tablet, currently BatchInputSplit
// will not span tablets
int scanThreads = 1;
scanner =
instance
.getConnector(principal, token)
.createBatchScanner(split.getTableName(), authorizations, scanThreads);
setupIterators(attempt, scanner, split.getTableName(), split);
if (null != classLoaderContext) {
scanner.setClassLoaderContext(classLoaderContext);
}
} catch (Exception e) {
e.printStackTrace();
throw new IOException(e);
}
scanner.setRanges(batchSplit.getRanges());
scannerBase = scanner;
} else {
Scanner scanner;
Boolean isOffline = split.isOffline();
if (null == isOffline) {
isOffline = tableConfig.isOfflineScan();
}
Boolean isIsolated = split.isIsolatedScan();
if (null == isIsolated) {
isIsolated = tableConfig.shouldUseIsolatedScanners();
}
Boolean usesLocalIterators = split.usesLocalIterators();
if (null == usesLocalIterators) {
usesLocalIterators = tableConfig.shouldUseLocalIterators();
}
try {
if (isOffline) {
scanner =
new OfflineScanner(
instance,
new Credentials(principal, token),
split.getTableId(),
authorizations);
} else if (DeprecationUtil.isMockInstance(instance)) {
scanner =
instance
.getConnector(principal, token)
.createScanner(split.getTableName(), authorizations);
} else {
ClientConfiguration clientConf = getClientConfiguration(attempt);
ClientContext context =
new ClientContext(instance, new Credentials(principal, token), clientConf);
scanner = new ScannerImpl(context, split.getTableId(), authorizations);
}
if (isIsolated) {
log.info("Creating isolated scanner");
scanner = new IsolatedScanner(scanner);
}
if (usesLocalIterators) {
log.info("Using local iterators");
scanner = new ClientSideIteratorScanner(scanner);
}
setupIterators(attempt, scanner, split.getTableName(), split);
} catch (Exception e) {
throw new IOException(e);
}
scanner.setRange(split.getRange());
scannerBase = scanner;
}
Collection<Pair<Text, Text>> columns = split.getFetchedColumns();
if (null == columns) {
columns = tableConfig.getFetchedColumns();
}
// setup a scanner within the bounds of this split
for (Pair<Text, Text> c : columns) {
if (c.getSecond() != null) {
log.debug("Fetching column " + c.getFirst() + ":" + c.getSecond());
scannerBase.fetchColumn(c.getFirst(), c.getSecond());
} else {
log.debug("Fetching column family " + c.getFirst());
scannerBase.fetchColumnFamily(c.getFirst());
}
}
SamplerConfiguration samplerConfig = split.getSamplerConfiguration();
if (null == samplerConfig) {
samplerConfig = tableConfig.getSamplerConfiguration();
}
if (samplerConfig != null) {
scannerBase.setSamplerConfiguration(samplerConfig);
}
scannerIterator = scannerBase.iterator();
numKeysRead = 0;
}