/**
* Test adding iterator options where the keys and values contain both the FIELD_SEPARATOR
* character (':') and ITERATOR_SEPARATOR (',') characters. There should be no exceptions thrown
* when trying to parse these types of option entries.
*
* <p>This test makes sure that the expected raw values, as appears in the Job, are equal to
* what's expected.
*/
@Test
public void testIteratorOptionEncoding() throws Throwable {
String key = "colon:delimited:key";
String value = "comma,delimited,value";
IteratorSetting someSetting = new IteratorSetting(1, "iterator", "Iterator.class");
someSetting.addOption(key, value);
@SuppressWarnings("deprecation")
Job job = new Job();
AccumuloInputFormat.addIterator(job, someSetting);
List<IteratorSetting> list = AccumuloInputFormat.getIterators(job);
assertEquals(1, list.size());
assertEquals(1, list.get(0).getOptions().size());
assertEquals(list.get(0).getOptions().get(key), value);
someSetting.addOption(key + "2", value);
someSetting.setPriority(2);
someSetting.setName("it2");
AccumuloInputFormat.addIterator(job, someSetting);
list = AccumuloInputFormat.getIterators(job);
assertEquals(2, list.size());
assertEquals(1, list.get(0).getOptions().size());
assertEquals(list.get(0).getOptions().get(key), value);
assertEquals(2, list.get(1).getOptions().size());
assertEquals(list.get(1).getOptions().get(key), value);
assertEquals(list.get(1).getOptions().get(key + "2"), value);
}
@Test
public void testIteratorNotInSplitsCompensation() throws Exception {
FileInputFormat.addInputPath(conf, new Path("unused"));
InputSplit[] splits = inputformat.getSplits(conf, 0);
assertEquals(1, splits.length);
InputSplit split = splits[0];
IteratorSetting is =
new IteratorSetting(
1, PrimitiveComparisonFilter.FILTER_PREFIX + 1, PrimitiveComparisonFilter.class);
is.addOption(PrimitiveComparisonFilter.P_COMPARE_CLASS, StringCompare.class.getName());
is.addOption(PrimitiveComparisonFilter.COMPARE_OPT_CLASS, Equal.class.getName());
is.addOption(
PrimitiveComparisonFilter.CONST_VAL, new String(Base64.encodeBase64(new byte[] {'0'})));
is.addOption(PrimitiveComparisonFilter.COLUMN, "cf:cq");
// Mock out the predicate handler because it's just easier
AccumuloPredicateHandler predicateHandler = Mockito.mock(AccumuloPredicateHandler.class);
Mockito.when(
predicateHandler.getIterators(
Mockito.any(JobConf.class), Mockito.any(ColumnMapper.class)))
.thenReturn(Arrays.asList(is));
// Set it on our inputformat
inputformat.predicateHandler = predicateHandler;
inputformat.getRecordReader(split, conf, null);
// The code should account for the bug and update the iterators on the split
List<IteratorSetting> settingsOnSplit = ((HiveAccumuloSplit) split).getSplit().getIterators();
assertEquals(1, settingsOnSplit.size());
assertEquals(is, settingsOnSplit.get(0));
}
@Test
public void testAllFieldsWritable() throws IOException {
Range[] ranges = new Range[] {new Range(new Key("a"), new Key("b"))};
BatchInputSplit split =
new BatchInputSplit("table", "1", Arrays.asList(ranges), new String[] {"localhost"});
Set<Pair<Text, Text>> fetchedColumns = new HashSet<>();
fetchedColumns.add(new Pair<>(new Text("colf1"), new Text("colq1")));
fetchedColumns.add(new Pair<>(new Text("colf2"), new Text("colq2")));
// Fake some iterators
ArrayList<IteratorSetting> iterators = new ArrayList<>();
IteratorSetting setting = new IteratorSetting(50, SummingCombiner.class);
setting.addOption("foo", "bar");
iterators.add(setting);
setting = new IteratorSetting(100, WholeRowIterator.class);
setting.addOption("bar", "foo");
iterators.add(setting);
split.setTableName("table");
split.setAuths(new Authorizations("foo"));
split.setFetchedColumns(fetchedColumns);
split.setToken(new PasswordToken("password"));
split.setPrincipal("root");
DeprecationUtil.setMockInstance(split, true);
split.setInstanceName("instance");
split.setZooKeepers("localhost");
split.setIterators(iterators);
split.setLogLevel(Level.WARN);
ByteArrayOutputStream baos = new ByteArrayOutputStream();
DataOutputStream dos = new DataOutputStream(baos);
split.write(dos);
BatchInputSplit newSplit = new BatchInputSplit();
ByteArrayInputStream bais = new ByteArrayInputStream(baos.toByteArray());
DataInputStream dis = new DataInputStream(bais);
newSplit.readFields(dis);
Assert.assertEquals(split.getRanges(), newSplit.getRanges());
Assert.assertArrayEquals(split.getLocations(), newSplit.getLocations());
Assert.assertEquals(split.getTableName(), newSplit.getTableName());
Assert.assertEquals(split.getAuths(), newSplit.getAuths());
Assert.assertEquals(split.getFetchedColumns(), newSplit.getFetchedColumns());
Assert.assertEquals(split.getToken(), newSplit.getToken());
Assert.assertEquals(split.getPrincipal(), newSplit.getPrincipal());
Assert.assertEquals(split.getInstanceName(), newSplit.getInstanceName());
Assert.assertEquals(
DeprecationUtil.isMockInstanceSet(split), DeprecationUtil.isMockInstanceSet(newSplit));
Assert.assertEquals(split.getZooKeepers(), newSplit.getZooKeepers());
Assert.assertEquals(split.getIterators(), newSplit.getIterators());
Assert.assertEquals(split.getLogLevel(), newSplit.getLogLevel());
}
@Test
public void testConfigureAccumuloInputFormatWithIterators() throws Exception {
AccumuloConnectionParameters accumuloParams = new AccumuloConnectionParameters(conf);
ColumnMapper columnMapper =
new ColumnMapper(
conf.get(AccumuloSerDeParameters.COLUMN_MAPPINGS),
conf.get(AccumuloSerDeParameters.DEFAULT_STORAGE_TYPE),
columnNames,
columnTypes);
Set<Pair<Text, Text>> cfCqPairs =
inputformat.getPairCollection(columnMapper.getColumnMappings());
List<IteratorSetting> iterators = new ArrayList<IteratorSetting>();
Set<Range> ranges = Collections.singleton(new Range());
String instanceName = "realInstance";
String zookeepers = "host1:2181,host2:2181,host3:2181";
IteratorSetting cfg = new IteratorSetting(50, PrimitiveComparisonFilter.class);
cfg.addOption(PrimitiveComparisonFilter.P_COMPARE_CLASS, StringCompare.class.getName());
cfg.addOption(PrimitiveComparisonFilter.COMPARE_OPT_CLASS, Equal.class.getName());
cfg.addOption(PrimitiveComparisonFilter.CONST_VAL, "dave");
cfg.addOption(PrimitiveComparisonFilter.COLUMN, "person:name");
iterators.add(cfg);
cfg = new IteratorSetting(50, PrimitiveComparisonFilter.class);
cfg.addOption(PrimitiveComparisonFilter.P_COMPARE_CLASS, IntCompare.class.getName());
cfg.addOption(PrimitiveComparisonFilter.COMPARE_OPT_CLASS, Equal.class.getName());
cfg.addOption(PrimitiveComparisonFilter.CONST_VAL, "50");
cfg.addOption(PrimitiveComparisonFilter.COLUMN, "person:age");
iterators.add(cfg);
ZooKeeperInstance zkInstance = Mockito.mock(ZooKeeperInstance.class);
HiveAccumuloTableInputFormat mockInputFormat = Mockito.mock(HiveAccumuloTableInputFormat.class);
// Stub out the ZKI mock
Mockito.when(zkInstance.getInstanceName()).thenReturn(instanceName);
Mockito.when(zkInstance.getZooKeepers()).thenReturn(zookeepers);
// Call out to the real configure method
Mockito.doCallRealMethod()
.when(mockInputFormat)
.configure(conf, zkInstance, con, accumuloParams, columnMapper, iterators, ranges);
// Also compute the correct cf:cq pairs so we can assert the right argument was passed
Mockito.doCallRealMethod()
.when(mockInputFormat)
.getPairCollection(columnMapper.getColumnMappings());
mockInputFormat.configure(
conf, zkInstance, con, accumuloParams, columnMapper, iterators, ranges);
// Verify that the correct methods are invoked on AccumuloInputFormat
Mockito.verify(mockInputFormat).setZooKeeperInstance(conf, instanceName, zookeepers, false);
Mockito.verify(mockInputFormat).setConnectorInfo(conf, USER, new PasswordToken(PASS));
Mockito.verify(mockInputFormat).setInputTableName(conf, TEST_TABLE);
Mockito.verify(mockInputFormat)
.setScanAuthorizations(conf, con.securityOperations().getUserAuthorizations(USER));
Mockito.verify(mockInputFormat).addIterators(conf, iterators);
Mockito.verify(mockInputFormat).setRanges(conf, ranges);
Mockito.verify(mockInputFormat).fetchColumns(conf, cfCqPairs);
}
/**
* Check that the iterator configuration is getting stored in the Job conf correctly.
*
* @throws IOException
*/
@Test
public void testSetIterator() throws IOException {
@SuppressWarnings("deprecation")
Job job = new Job();
IteratorSetting is =
new IteratorSetting(1, "WholeRow", "org.apache.accumulo.core.iterators.WholeRowIterator");
AccumuloInputFormat.addIterator(job, is);
Configuration conf = job.getConfiguration();
ByteArrayOutputStream baos = new ByteArrayOutputStream();
is.write(new DataOutputStream(baos));
String iterators = conf.get("AccumuloInputFormat.ScanOpts.Iterators");
assertEquals(new String(Base64.encodeBase64(baos.toByteArray())), iterators);
}
@Test
public void testGreaterThan1Sid() throws Exception {
Connector con = mockInstance.getConnector(USER, new PasswordToken(PASS.getBytes()));
Scanner scan = con.createScanner(TEST_TABLE, new Authorizations("blah"));
IteratorSetting is =
new IteratorSetting(
1, PrimitiveComparisonFilter.FILTER_PREFIX + 1, PrimitiveComparisonFilter.class);
is.addOption(PrimitiveComparisonFilter.P_COMPARE_CLASS, IntCompare.class.getName());
is.addOption(PrimitiveComparisonFilter.COMPARE_OPT_CLASS, GreaterThan.class.getName());
is.addOption(
PrimitiveComparisonFilter.CONST_VAL, new String(Base64.encodeBase64(parseIntBytes("1"))));
is.addOption(PrimitiveComparisonFilter.COLUMN, "cf:sid");
scan.addScanIterator(is);
boolean foundMark = false;
boolean foundDennis = false;
int totalCount = 0;
for (Map.Entry<Key, Value> kv : scan) {
boolean foundName = false;
boolean foundSid = false;
boolean foundDegrees = false;
boolean foundMillis = false;
SortedMap<Key, Value> items = PrimitiveComparisonFilter.decodeRow(kv.getKey(), kv.getValue());
for (Map.Entry<Key, Value> item : items.entrySet()) {
if (item.getKey().getRow().toString().equals("r2")) {
foundMark = true;
} else if (item.getKey().getRow().toString().equals("r3")) {
foundDennis = true;
}
if (item.getKey().getColumnQualifier().equals(NAME)) {
foundName = true;
} else if (item.getKey().getColumnQualifier().equals(SID)) {
foundSid = true;
} else if (item.getKey().getColumnQualifier().equals(DEGREES)) {
foundDegrees = true;
} else if (item.getKey().getColumnQualifier().equals(MILLIS)) {
foundMillis = true;
}
}
totalCount++;
assertTrue(foundDegrees & foundMillis & foundName & foundSid);
}
assertTrue(foundDennis & foundMark);
assertEquals(totalCount, 2);
}
@Test
public void testNameEqualBrian() throws Exception {
Connector con = mockInstance.getConnector(USER, new PasswordToken(PASS.getBytes()));
Scanner scan = con.createScanner(TEST_TABLE, new Authorizations("blah"));
IteratorSetting is =
new IteratorSetting(
1, PrimitiveComparisonFilter.FILTER_PREFIX + 1, PrimitiveComparisonFilter.class);
is.addOption(PrimitiveComparisonFilter.P_COMPARE_CLASS, StringCompare.class.getName());
is.addOption(PrimitiveComparisonFilter.COMPARE_OPT_CLASS, Equal.class.getName());
is.addOption(
PrimitiveComparisonFilter.CONST_VAL, new String(Base64.encodeBase64("brian".getBytes())));
is.addOption(PrimitiveComparisonFilter.COLUMN, "cf:name");
scan.addScanIterator(is);
boolean foundName = false;
boolean foundSid = false;
boolean foundDegrees = false;
boolean foundMillis = false;
for (Map.Entry<Key, Value> kv : scan) {
SortedMap<Key, Value> items = PrimitiveComparisonFilter.decodeRow(kv.getKey(), kv.getValue());
for (Map.Entry<Key, Value> item : items.entrySet()) {
assertEquals(item.getKey().getRow().toString(), "r1");
if (item.getKey().getColumnQualifier().equals(NAME)) {
foundName = true;
assertArrayEquals(item.getValue().get(), "brian".getBytes());
} else if (item.getKey().getColumnQualifier().equals(SID)) {
foundSid = true;
assertArrayEquals(item.getValue().get(), parseIntBytes("1"));
} else if (item.getKey().getColumnQualifier().equals(DEGREES)) {
foundDegrees = true;
assertArrayEquals(item.getValue().get(), parseDoubleBytes("44.5"));
} else if (item.getKey().getColumnQualifier().equals(MILLIS)) {
foundMillis = true;
assertArrayEquals(item.getValue().get(), parseLongBytes("555"));
}
}
}
assertTrue(foundDegrees & foundMillis & foundName & foundSid);
}
public ByteBuffer compress(IteratorSetting[] iterators) {
UnsynchronizedBuffer.Writer out = new UnsynchronizedBuffer.Writer(iterators.length * 8);
out.writeVInt(iterators.length);
for (IteratorSetting is : iterators) {
out.writeVInt(getSymbolID(is.getName()));
out.writeVInt(getSymbolID(is.getIteratorClass()));
out.writeVInt(is.getPriority());
Map<String, String> opts = is.getOptions();
out.writeVInt(opts.size());
for (Entry<String, String> entry : opts.entrySet()) {
out.writeVInt(getSymbolID(entry.getKey()));
out.writeVInt(getSymbolID(entry.getValue()));
}
}
return out.toByteBuffer();
}
@Override
protected void setTableProperties(
final CommandLine cl,
final Shell shellState,
final int priority,
final Map<String, String> options,
final String classname,
final String name)
throws AccumuloException, AccumuloSecurityException, ShellCommandException,
TableNotFoundException {
// instead of setting table properties, just put the options in a list to use at scan time
String profile = cl.getOptionValue(profileOpt.getOpt());
// instead of setting table properties, just put the options in a list to use at scan time
for (Iterator<Entry<String, String>> i = options.entrySet().iterator(); i.hasNext(); ) {
final Entry<String, String> entry = i.next();
if (entry.getValue() == null || entry.getValue().isEmpty()) {
i.remove();
}
}
List<IteratorSetting> tableScanIterators = shellState.iteratorProfiles.get(profile);
if (tableScanIterators == null) {
tableScanIterators = new ArrayList<>();
shellState.iteratorProfiles.put(profile, tableScanIterators);
}
final IteratorSetting setting = new IteratorSetting(priority, name, classname);
setting.addOptions(options);
Iterator<IteratorSetting> iter = tableScanIterators.iterator();
while (iter.hasNext()) {
if (iter.next().getName().equals(name)) {
iter.remove();
}
}
tableScanIterators.add(setting);
}
/**
* Test getting iterator settings for multiple iterators set
*
* @throws IOException
*/
@Test
public void testGetIteratorSettings() throws IOException {
@SuppressWarnings("deprecation")
Job job = new Job();
AccumuloInputFormat.addIterator(
job,
new IteratorSetting(1, "WholeRow", "org.apache.accumulo.core.iterators.WholeRowIterator"));
AccumuloInputFormat.addIterator(
job,
new IteratorSetting(
2, "Versions", "org.apache.accumulo.core.iterators.VersioningIterator"));
AccumuloInputFormat.addIterator(
job,
new IteratorSetting(3, "Count", "org.apache.accumulo.core.iterators.CountingIterator"));
List<IteratorSetting> list = AccumuloInputFormat.getIterators(job);
// Check the list size
assertTrue(list.size() == 3);
// Walk the list and make sure our settings are correct
IteratorSetting setting = list.get(0);
assertEquals(1, setting.getPriority());
assertEquals("org.apache.accumulo.core.iterators.WholeRowIterator", setting.getIteratorClass());
assertEquals("WholeRow", setting.getName());
setting = list.get(1);
assertEquals(2, setting.getPriority());
assertEquals(
"org.apache.accumulo.core.iterators.VersioningIterator", setting.getIteratorClass());
assertEquals("Versions", setting.getName());
setting = list.get(2);
assertEquals(3, setting.getPriority());
assertEquals("org.apache.accumulo.core.iterators.CountingIterator", setting.getIteratorClass());
assertEquals("Count", setting.getName());
}
/**
* Add the given Iterator to a table on scan, minc, majc scopes. If already present on a scope,
* does not re-add the iterator to that scope. Call it "plus".
*/
public static void applyIteratorSoft(IteratorSetting itset, TableOperations tops, String table) {
// checking if iterator already exists. Not checking for conflicts.
try {
IteratorSetting existing;
EnumSet<IteratorUtil.IteratorScope> enumSet =
EnumSet.noneOf(IteratorUtil.IteratorScope.class);
for (IteratorUtil.IteratorScope scope : IteratorUtil.IteratorScope.values()) {
existing = tops.getIteratorSetting(table, itset.getName(), scope);
if (existing == null) enumSet.add(scope);
}
tops.attachIterator(table, itset, enumSet);
} catch (AccumuloSecurityException | AccumuloException e) {
log.error("error trying to add " + itset + " to " + table, e);
throw new RuntimeException(e);
} catch (TableNotFoundException e) {
log.error("no table: " + table, e);
throw new RuntimeException(e);
}
}
public static <K extends WritableComparable<?>, V extends Writable>
SortedKeyValueIterator<K, V> loadIterators(
IteratorScope scope,
SortedKeyValueIterator<K, V> source,
KeyExtent extent,
AccumuloConfiguration conf,
List<IteratorSetting> iterators,
IteratorEnvironment env)
throws IOException {
List<IterInfo> ssiList = new ArrayList<>();
Map<String, Map<String, String>> ssio = new HashMap<>();
for (IteratorSetting is : iterators) {
ssiList.add(new IterInfo(is.getPriority(), is.getIteratorClass(), is.getName()));
ssio.put(is.getName(), is.getOptions());
}
return loadIterators(scope, source, extent, conf, ssiList, ssio, env, true);
}
/**
* A convenience method for setting the end timestamp accepted by the timestamp filter.
*
* @param is the iterator setting object to configure
* @param end the end timestamp
* @param endInclusive boolean indicating whether the end is inclusive
*/
public static void setEnd(IteratorSetting is, long end, boolean endInclusive) {
is.addOption(END, LONG_PREFIX + Long.toString(end));
is.addOption(END_INCL, Boolean.toString(endInclusive));
}
/**
* A convenience method for setting the start timestamp accepted by the timestamp filter.
*
* @param is the iterator setting object to configure
* @param start the start timestamp
* @param startInclusive boolean indicating whether the start is inclusive
*/
public static void setStart(IteratorSetting is, long start, boolean startInclusive) {
is.addOption(START, LONG_PREFIX + Long.toString(start));
is.addOption(START_INCL, Boolean.toString(startInclusive));
}
public static IteratorSetting iteratorSetting(int priority, int matrixSize, int numIterations) {
IteratorSetting itset = new IteratorSetting(priority, InverseMatrixIterator.class);
itset.addOption(MATRIX_SIZE, Integer.toString(matrixSize));
itset.addOption(NUMITERATIONS, Integer.toString(numIterations));
return itset;
}
/**
* A convenience method for setting the long encoding type.
*
* @param is IteratorSetting object to configure.
* @param type LongCombiner.Type specifying the encoding type.
*/
public static void setEncodingType(IteratorSetting is, Type type) {
is.addOption(TYPE, type.toString());
}
public static TIteratorSetting toTIteratorSetting(IteratorSetting is) {
return new TIteratorSetting(
is.getPriority(), is.getName(), is.getIteratorClass(), is.getOptions());
}