@Test
public void testIteratorNotInSplitsCompensation() throws Exception {
FileInputFormat.addInputPath(conf, new Path("unused"));
InputSplit[] splits = inputformat.getSplits(conf, 0);
assertEquals(1, splits.length);
InputSplit split = splits[0];
IteratorSetting is =
new IteratorSetting(
1, PrimitiveComparisonFilter.FILTER_PREFIX + 1, PrimitiveComparisonFilter.class);
is.addOption(PrimitiveComparisonFilter.P_COMPARE_CLASS, StringCompare.class.getName());
is.addOption(PrimitiveComparisonFilter.COMPARE_OPT_CLASS, Equal.class.getName());
is.addOption(
PrimitiveComparisonFilter.CONST_VAL, new String(Base64.encodeBase64(new byte[] {'0'})));
is.addOption(PrimitiveComparisonFilter.COLUMN, "cf:cq");
// Mock out the predicate handler because it's just easier
AccumuloPredicateHandler predicateHandler = Mockito.mock(AccumuloPredicateHandler.class);
Mockito.when(
predicateHandler.getIterators(
Mockito.any(JobConf.class), Mockito.any(ColumnMapper.class)))
.thenReturn(Arrays.asList(is));
// Set it on our inputformat
inputformat.predicateHandler = predicateHandler;
inputformat.getRecordReader(split, conf, null);
// The code should account for the bug and update the iterators on the split
List<IteratorSetting> settingsOnSplit = ((HiveAccumuloSplit) split).getSplit().getIterators();
assertEquals(1, settingsOnSplit.size());
assertEquals(is, settingsOnSplit.get(0));
}
/**
* Test adding iterator options where the keys and values contain both the FIELD_SEPARATOR
* character (':') and ITERATOR_SEPARATOR (',') characters. There should be no exceptions thrown
* when trying to parse these types of option entries.
*
* <p>This test makes sure that the expected raw values, as appears in the Job, are equal to
* what's expected.
*/
@Test
public void testIteratorOptionEncoding() throws Throwable {
String key = "colon:delimited:key";
String value = "comma,delimited,value";
IteratorSetting someSetting = new IteratorSetting(1, "iterator", "Iterator.class");
someSetting.addOption(key, value);
@SuppressWarnings("deprecation")
Job job = new Job();
AccumuloInputFormat.addIterator(job, someSetting);
List<IteratorSetting> list = AccumuloInputFormat.getIterators(job);
assertEquals(1, list.size());
assertEquals(1, list.get(0).getOptions().size());
assertEquals(list.get(0).getOptions().get(key), value);
someSetting.addOption(key + "2", value);
someSetting.setPriority(2);
someSetting.setName("it2");
AccumuloInputFormat.addIterator(job, someSetting);
list = AccumuloInputFormat.getIterators(job);
assertEquals(2, list.size());
assertEquals(1, list.get(0).getOptions().size());
assertEquals(list.get(0).getOptions().get(key), value);
assertEquals(2, list.get(1).getOptions().size());
assertEquals(list.get(1).getOptions().get(key), value);
assertEquals(list.get(1).getOptions().get(key + "2"), value);
}
@Test
public void testAllFieldsWritable() throws IOException {
Range[] ranges = new Range[] {new Range(new Key("a"), new Key("b"))};
BatchInputSplit split =
new BatchInputSplit("table", "1", Arrays.asList(ranges), new String[] {"localhost"});
Set<Pair<Text, Text>> fetchedColumns = new HashSet<>();
fetchedColumns.add(new Pair<>(new Text("colf1"), new Text("colq1")));
fetchedColumns.add(new Pair<>(new Text("colf2"), new Text("colq2")));
// Fake some iterators
ArrayList<IteratorSetting> iterators = new ArrayList<>();
IteratorSetting setting = new IteratorSetting(50, SummingCombiner.class);
setting.addOption("foo", "bar");
iterators.add(setting);
setting = new IteratorSetting(100, WholeRowIterator.class);
setting.addOption("bar", "foo");
iterators.add(setting);
split.setTableName("table");
split.setAuths(new Authorizations("foo"));
split.setFetchedColumns(fetchedColumns);
split.setToken(new PasswordToken("password"));
split.setPrincipal("root");
DeprecationUtil.setMockInstance(split, true);
split.setInstanceName("instance");
split.setZooKeepers("localhost");
split.setIterators(iterators);
split.setLogLevel(Level.WARN);
ByteArrayOutputStream baos = new ByteArrayOutputStream();
DataOutputStream dos = new DataOutputStream(baos);
split.write(dos);
BatchInputSplit newSplit = new BatchInputSplit();
ByteArrayInputStream bais = new ByteArrayInputStream(baos.toByteArray());
DataInputStream dis = new DataInputStream(bais);
newSplit.readFields(dis);
Assert.assertEquals(split.getRanges(), newSplit.getRanges());
Assert.assertArrayEquals(split.getLocations(), newSplit.getLocations());
Assert.assertEquals(split.getTableName(), newSplit.getTableName());
Assert.assertEquals(split.getAuths(), newSplit.getAuths());
Assert.assertEquals(split.getFetchedColumns(), newSplit.getFetchedColumns());
Assert.assertEquals(split.getToken(), newSplit.getToken());
Assert.assertEquals(split.getPrincipal(), newSplit.getPrincipal());
Assert.assertEquals(split.getInstanceName(), newSplit.getInstanceName());
Assert.assertEquals(
DeprecationUtil.isMockInstanceSet(split), DeprecationUtil.isMockInstanceSet(newSplit));
Assert.assertEquals(split.getZooKeepers(), newSplit.getZooKeepers());
Assert.assertEquals(split.getIterators(), newSplit.getIterators());
Assert.assertEquals(split.getLogLevel(), newSplit.getLogLevel());
}
@Test
public void testConfigureAccumuloInputFormatWithIterators() throws Exception {
AccumuloConnectionParameters accumuloParams = new AccumuloConnectionParameters(conf);
ColumnMapper columnMapper =
new ColumnMapper(
conf.get(AccumuloSerDeParameters.COLUMN_MAPPINGS),
conf.get(AccumuloSerDeParameters.DEFAULT_STORAGE_TYPE),
columnNames,
columnTypes);
Set<Pair<Text, Text>> cfCqPairs =
inputformat.getPairCollection(columnMapper.getColumnMappings());
List<IteratorSetting> iterators = new ArrayList<IteratorSetting>();
Set<Range> ranges = Collections.singleton(new Range());
String instanceName = "realInstance";
String zookeepers = "host1:2181,host2:2181,host3:2181";
IteratorSetting cfg = new IteratorSetting(50, PrimitiveComparisonFilter.class);
cfg.addOption(PrimitiveComparisonFilter.P_COMPARE_CLASS, StringCompare.class.getName());
cfg.addOption(PrimitiveComparisonFilter.COMPARE_OPT_CLASS, Equal.class.getName());
cfg.addOption(PrimitiveComparisonFilter.CONST_VAL, "dave");
cfg.addOption(PrimitiveComparisonFilter.COLUMN, "person:name");
iterators.add(cfg);
cfg = new IteratorSetting(50, PrimitiveComparisonFilter.class);
cfg.addOption(PrimitiveComparisonFilter.P_COMPARE_CLASS, IntCompare.class.getName());
cfg.addOption(PrimitiveComparisonFilter.COMPARE_OPT_CLASS, Equal.class.getName());
cfg.addOption(PrimitiveComparisonFilter.CONST_VAL, "50");
cfg.addOption(PrimitiveComparisonFilter.COLUMN, "person:age");
iterators.add(cfg);
ZooKeeperInstance zkInstance = Mockito.mock(ZooKeeperInstance.class);
HiveAccumuloTableInputFormat mockInputFormat = Mockito.mock(HiveAccumuloTableInputFormat.class);
// Stub out the ZKI mock
Mockito.when(zkInstance.getInstanceName()).thenReturn(instanceName);
Mockito.when(zkInstance.getZooKeepers()).thenReturn(zookeepers);
// Call out to the real configure method
Mockito.doCallRealMethod()
.when(mockInputFormat)
.configure(conf, zkInstance, con, accumuloParams, columnMapper, iterators, ranges);
// Also compute the correct cf:cq pairs so we can assert the right argument was passed
Mockito.doCallRealMethod()
.when(mockInputFormat)
.getPairCollection(columnMapper.getColumnMappings());
mockInputFormat.configure(
conf, zkInstance, con, accumuloParams, columnMapper, iterators, ranges);
// Verify that the correct methods are invoked on AccumuloInputFormat
Mockito.verify(mockInputFormat).setZooKeeperInstance(conf, instanceName, zookeepers, false);
Mockito.verify(mockInputFormat).setConnectorInfo(conf, USER, new PasswordToken(PASS));
Mockito.verify(mockInputFormat).setInputTableName(conf, TEST_TABLE);
Mockito.verify(mockInputFormat)
.setScanAuthorizations(conf, con.securityOperations().getUserAuthorizations(USER));
Mockito.verify(mockInputFormat).addIterators(conf, iterators);
Mockito.verify(mockInputFormat).setRanges(conf, ranges);
Mockito.verify(mockInputFormat).fetchColumns(conf, cfCqPairs);
}
@Test
public void testGreaterThan1Sid() throws Exception {
Connector con = mockInstance.getConnector(USER, new PasswordToken(PASS.getBytes()));
Scanner scan = con.createScanner(TEST_TABLE, new Authorizations("blah"));
IteratorSetting is =
new IteratorSetting(
1, PrimitiveComparisonFilter.FILTER_PREFIX + 1, PrimitiveComparisonFilter.class);
is.addOption(PrimitiveComparisonFilter.P_COMPARE_CLASS, IntCompare.class.getName());
is.addOption(PrimitiveComparisonFilter.COMPARE_OPT_CLASS, GreaterThan.class.getName());
is.addOption(
PrimitiveComparisonFilter.CONST_VAL, new String(Base64.encodeBase64(parseIntBytes("1"))));
is.addOption(PrimitiveComparisonFilter.COLUMN, "cf:sid");
scan.addScanIterator(is);
boolean foundMark = false;
boolean foundDennis = false;
int totalCount = 0;
for (Map.Entry<Key, Value> kv : scan) {
boolean foundName = false;
boolean foundSid = false;
boolean foundDegrees = false;
boolean foundMillis = false;
SortedMap<Key, Value> items = PrimitiveComparisonFilter.decodeRow(kv.getKey(), kv.getValue());
for (Map.Entry<Key, Value> item : items.entrySet()) {
if (item.getKey().getRow().toString().equals("r2")) {
foundMark = true;
} else if (item.getKey().getRow().toString().equals("r3")) {
foundDennis = true;
}
if (item.getKey().getColumnQualifier().equals(NAME)) {
foundName = true;
} else if (item.getKey().getColumnQualifier().equals(SID)) {
foundSid = true;
} else if (item.getKey().getColumnQualifier().equals(DEGREES)) {
foundDegrees = true;
} else if (item.getKey().getColumnQualifier().equals(MILLIS)) {
foundMillis = true;
}
}
totalCount++;
assertTrue(foundDegrees & foundMillis & foundName & foundSid);
}
assertTrue(foundDennis & foundMark);
assertEquals(totalCount, 2);
}
@Test
public void testAddIterator() throws IOException {
@SuppressWarnings("deprecation")
Job job = new Job();
AccumuloInputFormat.addIterator(
job, new IteratorSetting(1, "WholeRow", WholeRowIterator.class));
AccumuloInputFormat.addIterator(
job,
new IteratorSetting(
2, "Versions", "org.apache.accumulo.core.iterators.VersioningIterator"));
IteratorSetting iter =
new IteratorSetting(3, "Count", "org.apache.accumulo.core.iterators.CountingIterator");
iter.addOption("v1", "1");
iter.addOption("junk", "\0omg:!\\xyzzy");
AccumuloInputFormat.addIterator(job, iter);
List<IteratorSetting> list = AccumuloInputFormat.getIterators(job);
// Check the list size
assertTrue(list.size() == 3);
// Walk the list and make sure our settings are correct
IteratorSetting setting = list.get(0);
assertEquals(1, setting.getPriority());
assertEquals(
"org.apache.accumulo.core.iterators.user.WholeRowIterator", setting.getIteratorClass());
assertEquals("WholeRow", setting.getName());
assertEquals(0, setting.getOptions().size());
setting = list.get(1);
assertEquals(2, setting.getPriority());
assertEquals(
"org.apache.accumulo.core.iterators.VersioningIterator", setting.getIteratorClass());
assertEquals("Versions", setting.getName());
assertEquals(0, setting.getOptions().size());
setting = list.get(2);
assertEquals(3, setting.getPriority());
assertEquals("org.apache.accumulo.core.iterators.CountingIterator", setting.getIteratorClass());
assertEquals("Count", setting.getName());
assertEquals(2, setting.getOptions().size());
assertEquals("1", setting.getOptions().get("v1"));
assertEquals("\0omg:!\\xyzzy", setting.getOptions().get("junk"));
}
@Test
public void testNameEqualBrian() throws Exception {
Connector con = mockInstance.getConnector(USER, new PasswordToken(PASS.getBytes()));
Scanner scan = con.createScanner(TEST_TABLE, new Authorizations("blah"));
IteratorSetting is =
new IteratorSetting(
1, PrimitiveComparisonFilter.FILTER_PREFIX + 1, PrimitiveComparisonFilter.class);
is.addOption(PrimitiveComparisonFilter.P_COMPARE_CLASS, StringCompare.class.getName());
is.addOption(PrimitiveComparisonFilter.COMPARE_OPT_CLASS, Equal.class.getName());
is.addOption(
PrimitiveComparisonFilter.CONST_VAL, new String(Base64.encodeBase64("brian".getBytes())));
is.addOption(PrimitiveComparisonFilter.COLUMN, "cf:name");
scan.addScanIterator(is);
boolean foundName = false;
boolean foundSid = false;
boolean foundDegrees = false;
boolean foundMillis = false;
for (Map.Entry<Key, Value> kv : scan) {
SortedMap<Key, Value> items = PrimitiveComparisonFilter.decodeRow(kv.getKey(), kv.getValue());
for (Map.Entry<Key, Value> item : items.entrySet()) {
assertEquals(item.getKey().getRow().toString(), "r1");
if (item.getKey().getColumnQualifier().equals(NAME)) {
foundName = true;
assertArrayEquals(item.getValue().get(), "brian".getBytes());
} else if (item.getKey().getColumnQualifier().equals(SID)) {
foundSid = true;
assertArrayEquals(item.getValue().get(), parseIntBytes("1"));
} else if (item.getKey().getColumnQualifier().equals(DEGREES)) {
foundDegrees = true;
assertArrayEquals(item.getValue().get(), parseDoubleBytes("44.5"));
} else if (item.getKey().getColumnQualifier().equals(MILLIS)) {
foundMillis = true;
assertArrayEquals(item.getValue().get(), parseLongBytes("555"));
}
}
}
assertTrue(foundDegrees & foundMillis & foundName & foundSid);
}
/**
* A convenience method for setting the end timestamp accepted by the timestamp filter.
*
* @param is the iterator setting object to configure
* @param end the end timestamp
* @param endInclusive boolean indicating whether the end is inclusive
*/
public static void setEnd(IteratorSetting is, long end, boolean endInclusive) {
is.addOption(END, LONG_PREFIX + Long.toString(end));
is.addOption(END_INCL, Boolean.toString(endInclusive));
}
/**
* A convenience method for setting the start timestamp accepted by the timestamp filter.
*
* @param is the iterator setting object to configure
* @param start the start timestamp
* @param startInclusive boolean indicating whether the start is inclusive
*/
public static void setStart(IteratorSetting is, long start, boolean startInclusive) {
is.addOption(START, LONG_PREFIX + Long.toString(start));
is.addOption(START_INCL, Boolean.toString(startInclusive));
}
public static IteratorSetting iteratorSetting(int priority, int matrixSize, int numIterations) {
IteratorSetting itset = new IteratorSetting(priority, InverseMatrixIterator.class);
itset.addOption(MATRIX_SIZE, Integer.toString(matrixSize));
itset.addOption(NUMITERATIONS, Integer.toString(numIterations));
return itset;
}
/**
* A convenience method for setting the long encoding type.
*
* @param is IteratorSetting object to configure.
* @param type LongCombiner.Type specifying the encoding type.
*/
public static void setEncodingType(IteratorSetting is, Type type) {
is.addOption(TYPE, type.toString());
}