@Test
public void testConfigureAccumuloInputFormatWithIterators() throws Exception {
AccumuloConnectionParameters accumuloParams = new AccumuloConnectionParameters(conf);
ColumnMapper columnMapper =
new ColumnMapper(
conf.get(AccumuloSerDeParameters.COLUMN_MAPPINGS),
conf.get(AccumuloSerDeParameters.DEFAULT_STORAGE_TYPE),
columnNames,
columnTypes);
Set<Pair<Text, Text>> cfCqPairs =
inputformat.getPairCollection(columnMapper.getColumnMappings());
List<IteratorSetting> iterators = new ArrayList<IteratorSetting>();
Set<Range> ranges = Collections.singleton(new Range());
String instanceName = "realInstance";
String zookeepers = "host1:2181,host2:2181,host3:2181";
IteratorSetting cfg = new IteratorSetting(50, PrimitiveComparisonFilter.class);
cfg.addOption(PrimitiveComparisonFilter.P_COMPARE_CLASS, StringCompare.class.getName());
cfg.addOption(PrimitiveComparisonFilter.COMPARE_OPT_CLASS, Equal.class.getName());
cfg.addOption(PrimitiveComparisonFilter.CONST_VAL, "dave");
cfg.addOption(PrimitiveComparisonFilter.COLUMN, "person:name");
iterators.add(cfg);
cfg = new IteratorSetting(50, PrimitiveComparisonFilter.class);
cfg.addOption(PrimitiveComparisonFilter.P_COMPARE_CLASS, IntCompare.class.getName());
cfg.addOption(PrimitiveComparisonFilter.COMPARE_OPT_CLASS, Equal.class.getName());
cfg.addOption(PrimitiveComparisonFilter.CONST_VAL, "50");
cfg.addOption(PrimitiveComparisonFilter.COLUMN, "person:age");
iterators.add(cfg);
ZooKeeperInstance zkInstance = Mockito.mock(ZooKeeperInstance.class);
HiveAccumuloTableInputFormat mockInputFormat = Mockito.mock(HiveAccumuloTableInputFormat.class);
// Stub out the ZKI mock
Mockito.when(zkInstance.getInstanceName()).thenReturn(instanceName);
Mockito.when(zkInstance.getZooKeepers()).thenReturn(zookeepers);
// Call out to the real configure method
Mockito.doCallRealMethod()
.when(mockInputFormat)
.configure(conf, zkInstance, con, accumuloParams, columnMapper, iterators, ranges);
// Also compute the correct cf:cq pairs so we can assert the right argument was passed
Mockito.doCallRealMethod()
.when(mockInputFormat)
.getPairCollection(columnMapper.getColumnMappings());
mockInputFormat.configure(
conf, zkInstance, con, accumuloParams, columnMapper, iterators, ranges);
// Verify that the correct methods are invoked on AccumuloInputFormat
Mockito.verify(mockInputFormat).setZooKeeperInstance(conf, instanceName, zookeepers, false);
Mockito.verify(mockInputFormat).setConnectorInfo(conf, USER, new PasswordToken(PASS));
Mockito.verify(mockInputFormat).setInputTableName(conf, TEST_TABLE);
Mockito.verify(mockInputFormat)
.setScanAuthorizations(conf, con.securityOperations().getUserAuthorizations(USER));
Mockito.verify(mockInputFormat).addIterators(conf, iterators);
Mockito.verify(mockInputFormat).setRanges(conf, ranges);
Mockito.verify(mockInputFormat).fetchColumns(conf, cfCqPairs);
}
@Test
public void testIteratorNotInSplitsCompensation() throws Exception {
FileInputFormat.addInputPath(conf, new Path("unused"));
InputSplit[] splits = inputformat.getSplits(conf, 0);
assertEquals(1, splits.length);
InputSplit split = splits[0];
IteratorSetting is =
new IteratorSetting(
1, PrimitiveComparisonFilter.FILTER_PREFIX + 1, PrimitiveComparisonFilter.class);
is.addOption(PrimitiveComparisonFilter.P_COMPARE_CLASS, StringCompare.class.getName());
is.addOption(PrimitiveComparisonFilter.COMPARE_OPT_CLASS, Equal.class.getName());
is.addOption(
PrimitiveComparisonFilter.CONST_VAL, new String(Base64.encodeBase64(new byte[] {'0'})));
is.addOption(PrimitiveComparisonFilter.COLUMN, "cf:cq");
// Mock out the predicate handler because it's just easier
AccumuloPredicateHandler predicateHandler = Mockito.mock(AccumuloPredicateHandler.class);
Mockito.when(
predicateHandler.getIterators(
Mockito.any(JobConf.class), Mockito.any(ColumnMapper.class)))
.thenReturn(Arrays.asList(is));
// Set it on our inputformat
inputformat.predicateHandler = predicateHandler;
inputformat.getRecordReader(split, conf, null);
// The code should account for the bug and update the iterators on the split
List<IteratorSetting> settingsOnSplit = ((HiveAccumuloSplit) split).getSplit().getIterators();
assertEquals(1, settingsOnSplit.size());
assertEquals(is, settingsOnSplit.get(0));
}
@Test
public void testGetOnlyName() throws Exception {
FileInputFormat.addInputPath(conf, new Path("unused"));
InputSplit[] splits = inputformat.getSplits(conf, 0);
assertEquals(splits.length, 1);
RecordReader<Text, AccumuloHiveRow> reader = inputformat.getRecordReader(splits[0], conf, null);
Text rowId = new Text("r1");
AccumuloHiveRow row = new AccumuloHiveRow();
assertTrue(reader.next(rowId, row));
assertEquals(row.getRowId(), rowId.toString());
assertTrue(row.hasFamAndQual(COLUMN_FAMILY, NAME));
assertArrayEquals(row.getValue(COLUMN_FAMILY, NAME), "brian".getBytes());
rowId = new Text("r2");
assertTrue(reader.next(rowId, row));
assertEquals(row.getRowId(), rowId.toString());
assertTrue(row.hasFamAndQual(COLUMN_FAMILY, NAME));
assertArrayEquals(row.getValue(COLUMN_FAMILY, NAME), "mark".getBytes());
rowId = new Text("r3");
assertTrue(reader.next(rowId, row));
assertEquals(row.getRowId(), rowId.toString());
assertTrue(row.hasFamAndQual(COLUMN_FAMILY, NAME));
assertArrayEquals(row.getValue(COLUMN_FAMILY, NAME), "dennis".getBytes());
assertFalse(reader.next(rowId, row));
}
@Test
public void testGetNone() throws Exception {
FileInputFormat.addInputPath(conf, new Path("unused"));
conf.set(AccumuloSerDeParameters.COLUMN_MAPPINGS, "cf:f1");
InputSplit[] splits = inputformat.getSplits(conf, 0);
assertEquals(splits.length, 1);
RecordReader<Text, AccumuloHiveRow> reader = inputformat.getRecordReader(splits[0], conf, null);
Text rowId = new Text("r1");
AccumuloHiveRow row = new AccumuloHiveRow();
row.setRowId("r1");
assertFalse(reader.next(rowId, row));
}
@Test
public void testGetProtectedField() throws Exception {
FileInputFormat.addInputPath(conf, new Path("unused"));
BatchWriterConfig writerConf = new BatchWriterConfig();
BatchWriter writer = con.createBatchWriter(TEST_TABLE, writerConf);
Authorizations origAuths = con.securityOperations().getUserAuthorizations(USER);
con.securityOperations()
.changeUserAuthorizations(USER, new Authorizations(origAuths.toString() + ",foo"));
Mutation m = new Mutation("r4");
m.put(COLUMN_FAMILY, NAME, new ColumnVisibility("foo"), new Value("frank".getBytes()));
m.put(COLUMN_FAMILY, SID, new ColumnVisibility("foo"), new Value(parseIntBytes("4")));
m.put(COLUMN_FAMILY, DEGREES, new ColumnVisibility("foo"), new Value(parseDoubleBytes("60.6")));
m.put(COLUMN_FAMILY, MILLIS, new ColumnVisibility("foo"), new Value(parseLongBytes("777")));
writer.addMutation(m);
writer.close();
conf.set(AccumuloSerDeParameters.AUTHORIZATIONS_KEY, "foo");
InputSplit[] splits = inputformat.getSplits(conf, 0);
assertEquals(splits.length, 1);
RecordReader<Text, AccumuloHiveRow> reader = inputformat.getRecordReader(splits[0], conf, null);
Text rowId = new Text("r1");
AccumuloHiveRow row = new AccumuloHiveRow();
assertTrue(reader.next(rowId, row));
assertEquals(row.getRowId(), rowId.toString());
assertTrue(row.hasFamAndQual(COLUMN_FAMILY, NAME));
assertArrayEquals(row.getValue(COLUMN_FAMILY, NAME), "brian".getBytes());
rowId = new Text("r2");
assertTrue(reader.next(rowId, row));
assertEquals(row.getRowId(), rowId.toString());
assertTrue(row.hasFamAndQual(COLUMN_FAMILY, NAME));
assertArrayEquals(row.getValue(COLUMN_FAMILY, NAME), "mark".getBytes());
rowId = new Text("r3");
assertTrue(reader.next(rowId, row));
assertEquals(row.getRowId(), rowId.toString());
assertTrue(row.hasFamAndQual(COLUMN_FAMILY, NAME));
assertArrayEquals(row.getValue(COLUMN_FAMILY, NAME), "dennis".getBytes());
rowId = new Text("r4");
assertTrue(reader.next(rowId, row));
assertEquals(row.getRowId(), rowId.toString());
assertTrue(row.hasFamAndQual(COLUMN_FAMILY, NAME));
assertArrayEquals(row.getValue(COLUMN_FAMILY, NAME), "frank".getBytes());
assertFalse(reader.next(rowId, row));
}
@Test
public void testConfigureAccumuloInputFormatWithAuthorizations() throws Exception {
AccumuloConnectionParameters accumuloParams = new AccumuloConnectionParameters(conf);
conf.set(AccumuloSerDeParameters.AUTHORIZATIONS_KEY, "foo,bar");
ColumnMapper columnMapper =
new ColumnMapper(
conf.get(AccumuloSerDeParameters.COLUMN_MAPPINGS),
conf.get(AccumuloSerDeParameters.DEFAULT_STORAGE_TYPE),
columnNames,
columnTypes);
Set<Pair<Text, Text>> cfCqPairs =
inputformat.getPairCollection(columnMapper.getColumnMappings());
List<IteratorSetting> iterators = Collections.emptyList();
Set<Range> ranges = Collections.singleton(new Range());
String instanceName = "realInstance";
String zookeepers = "host1:2181,host2:2181,host3:2181";
ZooKeeperInstance zkInstance = Mockito.mock(ZooKeeperInstance.class);
HiveAccumuloTableInputFormat mockInputFormat = Mockito.mock(HiveAccumuloTableInputFormat.class);
// Stub out the ZKI mock
Mockito.when(zkInstance.getInstanceName()).thenReturn(instanceName);
Mockito.when(zkInstance.getZooKeepers()).thenReturn(zookeepers);
// Call out to the real configure method
Mockito.doCallRealMethod()
.when(mockInputFormat)
.configure(conf, zkInstance, con, accumuloParams, columnMapper, iterators, ranges);
// Also compute the correct cf:cq pairs so we can assert the right argument was passed
Mockito.doCallRealMethod()
.when(mockInputFormat)
.getPairCollection(columnMapper.getColumnMappings());
mockInputFormat.configure(
conf, zkInstance, con, accumuloParams, columnMapper, iterators, ranges);
// Verify that the correct methods are invoked on AccumuloInputFormat
Mockito.verify(mockInputFormat).setZooKeeperInstance(conf, instanceName, zookeepers, false);
Mockito.verify(mockInputFormat).setConnectorInfo(conf, USER, new PasswordToken(PASS));
Mockito.verify(mockInputFormat).setInputTableName(conf, TEST_TABLE);
Mockito.verify(mockInputFormat).setScanAuthorizations(conf, new Authorizations("foo,bar"));
Mockito.verify(mockInputFormat).addIterators(conf, iterators);
Mockito.verify(mockInputFormat).setRanges(conf, ranges);
Mockito.verify(mockInputFormat).fetchColumns(conf, cfCqPairs);
}
@Test
public void testColumnMappingsToPairs() {
List<ColumnMapping> mappings = new ArrayList<ColumnMapping>();
Set<Pair<Text, Text>> columns = new HashSet<Pair<Text, Text>>();
// Row ID
mappings.add(
new HiveAccumuloRowIdColumnMapping(
AccumuloHiveConstants.ROWID,
ColumnEncoding.STRING,
"row",
TypeInfoFactory.stringTypeInfo.toString()));
// Some cf:cq
mappings.add(
new HiveAccumuloColumnMapping(
"person",
"name",
ColumnEncoding.STRING,
"col1",
TypeInfoFactory.stringTypeInfo.toString()));
mappings.add(
new HiveAccumuloColumnMapping(
"person",
"age",
ColumnEncoding.STRING,
"col2",
TypeInfoFactory.stringTypeInfo.toString()));
mappings.add(
new HiveAccumuloColumnMapping(
"person",
"height",
ColumnEncoding.STRING,
"col3",
TypeInfoFactory.stringTypeInfo.toString()));
// Bare cf
mappings.add(
new HiveAccumuloColumnMapping(
"city",
"name",
ColumnEncoding.STRING,
"col4",
TypeInfoFactory.stringTypeInfo.toString()));
columns.add(new Pair<Text, Text>(new Text("person"), new Text("name")));
columns.add(new Pair<Text, Text>(new Text("person"), new Text("age")));
columns.add(new Pair<Text, Text>(new Text("person"), new Text("height")));
// Null qualifier would mean all qualifiers in that family, want an empty qualifier
columns.add(new Pair<Text, Text>(new Text("city"), new Text("name")));
assertEquals(columns, inputformat.getPairCollection(mappings));
}
@Test
public void testConfigureMockAccumuloInputFormat() throws Exception {
AccumuloConnectionParameters accumuloParams = new AccumuloConnectionParameters(conf);
ColumnMapper columnMapper =
new ColumnMapper(
conf.get(AccumuloSerDeParameters.COLUMN_MAPPINGS),
conf.get(AccumuloSerDeParameters.DEFAULT_STORAGE_TYPE),
columnNames,
columnTypes);
Set<Pair<Text, Text>> cfCqPairs =
inputformat.getPairCollection(columnMapper.getColumnMappings());
List<IteratorSetting> iterators = Collections.emptyList();
Set<Range> ranges = Collections.singleton(new Range());
HiveAccumuloTableInputFormat mockInputFormat = Mockito.mock(HiveAccumuloTableInputFormat.class);
// Call out to the real configure method
Mockito.doCallRealMethod()
.when(mockInputFormat)
.configure(conf, mockInstance, con, accumuloParams, columnMapper, iterators, ranges);
// Also compute the correct cf:cq pairs so we can assert the right argument was passed
Mockito.doCallRealMethod()
.when(mockInputFormat)
.getPairCollection(columnMapper.getColumnMappings());
mockInputFormat.configure(
conf, mockInstance, con, accumuloParams, columnMapper, iterators, ranges);
// Verify that the correct methods are invoked on AccumuloInputFormat
Mockito.verify(mockInputFormat).setMockInstance(conf, mockInstance.getInstanceName());
Mockito.verify(mockInputFormat).setConnectorInfo(conf, USER, new PasswordToken(PASS));
Mockito.verify(mockInputFormat).setInputTableName(conf, TEST_TABLE);
Mockito.verify(mockInputFormat)
.setScanAuthorizations(conf, con.securityOperations().getUserAuthorizations(USER));
Mockito.verify(mockInputFormat).addIterators(conf, iterators);
Mockito.verify(mockInputFormat).setRanges(conf, ranges);
Mockito.verify(mockInputFormat).fetchColumns(conf, cfCqPairs);
}
@Test
public void testHiveAccumuloRecord() throws Exception {
FileInputFormat.addInputPath(conf, new Path("unused"));
InputSplit[] splits = inputformat.getSplits(conf, 0);
assertEquals(splits.length, 1);
RecordReader<Text, AccumuloHiveRow> reader = inputformat.getRecordReader(splits[0], conf, null);
Text rowId = new Text("r1");
AccumuloHiveRow row = new AccumuloHiveRow();
row.add(COLUMN_FAMILY.toString(), NAME.toString(), "brian".getBytes());
row.add(COLUMN_FAMILY.toString(), SID.toString(), parseIntBytes("1"));
row.add(COLUMN_FAMILY.toString(), DEGREES.toString(), parseDoubleBytes("44.5"));
row.add(COLUMN_FAMILY.toString(), MILLIS.toString(), parseLongBytes("555"));
assertTrue(reader.next(rowId, row));
assertEquals(rowId.toString(), row.getRowId());
assertTrue(row.hasFamAndQual(COLUMN_FAMILY, NAME));
assertArrayEquals("brian".getBytes(), row.getValue(COLUMN_FAMILY, NAME));
assertTrue(row.hasFamAndQual(COLUMN_FAMILY, SID));
assertArrayEquals(parseIntBytes("1"), row.getValue(COLUMN_FAMILY, SID));
assertTrue(row.hasFamAndQual(COLUMN_FAMILY, DEGREES));
assertArrayEquals(parseDoubleBytes("44.5"), row.getValue(COLUMN_FAMILY, DEGREES));
assertTrue(row.hasFamAndQual(COLUMN_FAMILY, MILLIS));
assertArrayEquals(parseLongBytes("555"), row.getValue(COLUMN_FAMILY, MILLIS));
}
@Test
public void testMapColumnPairs() throws TooManyAccumuloColumnsException {
ColumnMapper columnMapper =
new ColumnMapper(
":rowID,cf:*",
conf.get(AccumuloSerDeParameters.DEFAULT_STORAGE_TYPE),
Arrays.asList("row", "col"),
Arrays.<TypeInfo>asList(
TypeInfoFactory.stringTypeInfo,
TypeInfoFactory.getMapTypeInfo(
TypeInfoFactory.stringTypeInfo, TypeInfoFactory.stringTypeInfo)));
Set<Pair<Text, Text>> pairs = inputformat.getPairCollection(columnMapper.getColumnMappings());
Assert.assertEquals(1, pairs.size());
Pair<Text, Text> cfCq = pairs.iterator().next();
Assert.assertEquals("cf", cfCq.getFirst().toString());
Assert.assertNull(cfCq.getSecond());
}