@Test public void testIteratorNotInSplitsCompensation() throws Exception { FileInputFormat.addInputPath(conf, new Path("unused")); InputSplit[] splits = inputformat.getSplits(conf, 0); assertEquals(1, splits.length); InputSplit split = splits[0]; IteratorSetting is = new IteratorSetting( 1, PrimitiveComparisonFilter.FILTER_PREFIX + 1, PrimitiveComparisonFilter.class); is.addOption(PrimitiveComparisonFilter.P_COMPARE_CLASS, StringCompare.class.getName()); is.addOption(PrimitiveComparisonFilter.COMPARE_OPT_CLASS, Equal.class.getName()); is.addOption( PrimitiveComparisonFilter.CONST_VAL, new String(Base64.encodeBase64(new byte[] {'0'}))); is.addOption(PrimitiveComparisonFilter.COLUMN, "cf:cq"); // Mock out the predicate handler because it's just easier AccumuloPredicateHandler predicateHandler = Mockito.mock(AccumuloPredicateHandler.class); Mockito.when( predicateHandler.getIterators( Mockito.any(JobConf.class), Mockito.any(ColumnMapper.class))) .thenReturn(Arrays.asList(is)); // Set it on our inputformat inputformat.predicateHandler = predicateHandler; inputformat.getRecordReader(split, conf, null); // The code should account for the bug and update the iterators on the split List<IteratorSetting> settingsOnSplit = ((HiveAccumuloSplit) split).getSplit().getIterators(); assertEquals(1, settingsOnSplit.size()); assertEquals(is, settingsOnSplit.get(0)); }
/** * Test adding iterator options where the keys and values contain both the FIELD_SEPARATOR * character (':') and ITERATOR_SEPARATOR (',') characters. There should be no exceptions thrown * when trying to parse these types of option entries. * * <p>This test makes sure that the expected raw values, as appears in the Job, are equal to * what's expected. */ @Test public void testIteratorOptionEncoding() throws Throwable { String key = "colon:delimited:key"; String value = "comma,delimited,value"; IteratorSetting someSetting = new IteratorSetting(1, "iterator", "Iterator.class"); someSetting.addOption(key, value); @SuppressWarnings("deprecation") Job job = new Job(); AccumuloInputFormat.addIterator(job, someSetting); List<IteratorSetting> list = AccumuloInputFormat.getIterators(job); assertEquals(1, list.size()); assertEquals(1, list.get(0).getOptions().size()); assertEquals(list.get(0).getOptions().get(key), value); someSetting.addOption(key + "2", value); someSetting.setPriority(2); someSetting.setName("it2"); AccumuloInputFormat.addIterator(job, someSetting); list = AccumuloInputFormat.getIterators(job); assertEquals(2, list.size()); assertEquals(1, list.get(0).getOptions().size()); assertEquals(list.get(0).getOptions().get(key), value); assertEquals(2, list.get(1).getOptions().size()); assertEquals(list.get(1).getOptions().get(key), value); assertEquals(list.get(1).getOptions().get(key + "2"), value); }
@Test public void testAllFieldsWritable() throws IOException { Range[] ranges = new Range[] {new Range(new Key("a"), new Key("b"))}; BatchInputSplit split = new BatchInputSplit("table", "1", Arrays.asList(ranges), new String[] {"localhost"}); Set<Pair<Text, Text>> fetchedColumns = new HashSet<>(); fetchedColumns.add(new Pair<>(new Text("colf1"), new Text("colq1"))); fetchedColumns.add(new Pair<>(new Text("colf2"), new Text("colq2"))); // Fake some iterators ArrayList<IteratorSetting> iterators = new ArrayList<>(); IteratorSetting setting = new IteratorSetting(50, SummingCombiner.class); setting.addOption("foo", "bar"); iterators.add(setting); setting = new IteratorSetting(100, WholeRowIterator.class); setting.addOption("bar", "foo"); iterators.add(setting); split.setTableName("table"); split.setAuths(new Authorizations("foo")); split.setFetchedColumns(fetchedColumns); split.setToken(new PasswordToken("password")); split.setPrincipal("root"); DeprecationUtil.setMockInstance(split, true); split.setInstanceName("instance"); split.setZooKeepers("localhost"); split.setIterators(iterators); split.setLogLevel(Level.WARN); ByteArrayOutputStream baos = new ByteArrayOutputStream(); DataOutputStream dos = new DataOutputStream(baos); split.write(dos); BatchInputSplit newSplit = new BatchInputSplit(); ByteArrayInputStream bais = new ByteArrayInputStream(baos.toByteArray()); DataInputStream dis = new DataInputStream(bais); newSplit.readFields(dis); Assert.assertEquals(split.getRanges(), newSplit.getRanges()); Assert.assertArrayEquals(split.getLocations(), newSplit.getLocations()); Assert.assertEquals(split.getTableName(), newSplit.getTableName()); Assert.assertEquals(split.getAuths(), newSplit.getAuths()); Assert.assertEquals(split.getFetchedColumns(), newSplit.getFetchedColumns()); Assert.assertEquals(split.getToken(), newSplit.getToken()); Assert.assertEquals(split.getPrincipal(), newSplit.getPrincipal()); Assert.assertEquals(split.getInstanceName(), newSplit.getInstanceName()); Assert.assertEquals( DeprecationUtil.isMockInstanceSet(split), DeprecationUtil.isMockInstanceSet(newSplit)); Assert.assertEquals(split.getZooKeepers(), newSplit.getZooKeepers()); Assert.assertEquals(split.getIterators(), newSplit.getIterators()); Assert.assertEquals(split.getLogLevel(), newSplit.getLogLevel()); }
@Test public void testConfigureAccumuloInputFormatWithIterators() throws Exception { AccumuloConnectionParameters accumuloParams = new AccumuloConnectionParameters(conf); ColumnMapper columnMapper = new ColumnMapper( conf.get(AccumuloSerDeParameters.COLUMN_MAPPINGS), conf.get(AccumuloSerDeParameters.DEFAULT_STORAGE_TYPE), columnNames, columnTypes); Set<Pair<Text, Text>> cfCqPairs = inputformat.getPairCollection(columnMapper.getColumnMappings()); List<IteratorSetting> iterators = new ArrayList<IteratorSetting>(); Set<Range> ranges = Collections.singleton(new Range()); String instanceName = "realInstance"; String zookeepers = "host1:2181,host2:2181,host3:2181"; IteratorSetting cfg = new IteratorSetting(50, PrimitiveComparisonFilter.class); cfg.addOption(PrimitiveComparisonFilter.P_COMPARE_CLASS, StringCompare.class.getName()); cfg.addOption(PrimitiveComparisonFilter.COMPARE_OPT_CLASS, Equal.class.getName()); cfg.addOption(PrimitiveComparisonFilter.CONST_VAL, "dave"); cfg.addOption(PrimitiveComparisonFilter.COLUMN, "person:name"); iterators.add(cfg); cfg = new IteratorSetting(50, PrimitiveComparisonFilter.class); cfg.addOption(PrimitiveComparisonFilter.P_COMPARE_CLASS, IntCompare.class.getName()); cfg.addOption(PrimitiveComparisonFilter.COMPARE_OPT_CLASS, Equal.class.getName()); cfg.addOption(PrimitiveComparisonFilter.CONST_VAL, "50"); cfg.addOption(PrimitiveComparisonFilter.COLUMN, "person:age"); iterators.add(cfg); ZooKeeperInstance zkInstance = Mockito.mock(ZooKeeperInstance.class); HiveAccumuloTableInputFormat mockInputFormat = Mockito.mock(HiveAccumuloTableInputFormat.class); // Stub out the ZKI mock Mockito.when(zkInstance.getInstanceName()).thenReturn(instanceName); Mockito.when(zkInstance.getZooKeepers()).thenReturn(zookeepers); // Call out to the real configure method Mockito.doCallRealMethod() .when(mockInputFormat) .configure(conf, zkInstance, con, accumuloParams, columnMapper, iterators, ranges); // Also compute the correct cf:cq pairs so we can assert the right argument was passed Mockito.doCallRealMethod() .when(mockInputFormat) .getPairCollection(columnMapper.getColumnMappings()); mockInputFormat.configure( conf, zkInstance, con, accumuloParams, columnMapper, iterators, ranges); // Verify that the correct methods are invoked on AccumuloInputFormat Mockito.verify(mockInputFormat).setZooKeeperInstance(conf, instanceName, zookeepers, false); Mockito.verify(mockInputFormat).setConnectorInfo(conf, USER, new PasswordToken(PASS)); Mockito.verify(mockInputFormat).setInputTableName(conf, TEST_TABLE); Mockito.verify(mockInputFormat) .setScanAuthorizations(conf, con.securityOperations().getUserAuthorizations(USER)); Mockito.verify(mockInputFormat).addIterators(conf, iterators); Mockito.verify(mockInputFormat).setRanges(conf, ranges); Mockito.verify(mockInputFormat).fetchColumns(conf, cfCqPairs); }
@Test public void testGreaterThan1Sid() throws Exception { Connector con = mockInstance.getConnector(USER, new PasswordToken(PASS.getBytes())); Scanner scan = con.createScanner(TEST_TABLE, new Authorizations("blah")); IteratorSetting is = new IteratorSetting( 1, PrimitiveComparisonFilter.FILTER_PREFIX + 1, PrimitiveComparisonFilter.class); is.addOption(PrimitiveComparisonFilter.P_COMPARE_CLASS, IntCompare.class.getName()); is.addOption(PrimitiveComparisonFilter.COMPARE_OPT_CLASS, GreaterThan.class.getName()); is.addOption( PrimitiveComparisonFilter.CONST_VAL, new String(Base64.encodeBase64(parseIntBytes("1")))); is.addOption(PrimitiveComparisonFilter.COLUMN, "cf:sid"); scan.addScanIterator(is); boolean foundMark = false; boolean foundDennis = false; int totalCount = 0; for (Map.Entry<Key, Value> kv : scan) { boolean foundName = false; boolean foundSid = false; boolean foundDegrees = false; boolean foundMillis = false; SortedMap<Key, Value> items = PrimitiveComparisonFilter.decodeRow(kv.getKey(), kv.getValue()); for (Map.Entry<Key, Value> item : items.entrySet()) { if (item.getKey().getRow().toString().equals("r2")) { foundMark = true; } else if (item.getKey().getRow().toString().equals("r3")) { foundDennis = true; } if (item.getKey().getColumnQualifier().equals(NAME)) { foundName = true; } else if (item.getKey().getColumnQualifier().equals(SID)) { foundSid = true; } else if (item.getKey().getColumnQualifier().equals(DEGREES)) { foundDegrees = true; } else if (item.getKey().getColumnQualifier().equals(MILLIS)) { foundMillis = true; } } totalCount++; assertTrue(foundDegrees & foundMillis & foundName & foundSid); } assertTrue(foundDennis & foundMark); assertEquals(totalCount, 2); }
@Test public void testAddIterator() throws IOException { @SuppressWarnings("deprecation") Job job = new Job(); AccumuloInputFormat.addIterator( job, new IteratorSetting(1, "WholeRow", WholeRowIterator.class)); AccumuloInputFormat.addIterator( job, new IteratorSetting( 2, "Versions", "org.apache.accumulo.core.iterators.VersioningIterator")); IteratorSetting iter = new IteratorSetting(3, "Count", "org.apache.accumulo.core.iterators.CountingIterator"); iter.addOption("v1", "1"); iter.addOption("junk", "\0omg:!\\xyzzy"); AccumuloInputFormat.addIterator(job, iter); List<IteratorSetting> list = AccumuloInputFormat.getIterators(job); // Check the list size assertTrue(list.size() == 3); // Walk the list and make sure our settings are correct IteratorSetting setting = list.get(0); assertEquals(1, setting.getPriority()); assertEquals( "org.apache.accumulo.core.iterators.user.WholeRowIterator", setting.getIteratorClass()); assertEquals("WholeRow", setting.getName()); assertEquals(0, setting.getOptions().size()); setting = list.get(1); assertEquals(2, setting.getPriority()); assertEquals( "org.apache.accumulo.core.iterators.VersioningIterator", setting.getIteratorClass()); assertEquals("Versions", setting.getName()); assertEquals(0, setting.getOptions().size()); setting = list.get(2); assertEquals(3, setting.getPriority()); assertEquals("org.apache.accumulo.core.iterators.CountingIterator", setting.getIteratorClass()); assertEquals("Count", setting.getName()); assertEquals(2, setting.getOptions().size()); assertEquals("1", setting.getOptions().get("v1")); assertEquals("\0omg:!\\xyzzy", setting.getOptions().get("junk")); }
@Test public void testNameEqualBrian() throws Exception { Connector con = mockInstance.getConnector(USER, new PasswordToken(PASS.getBytes())); Scanner scan = con.createScanner(TEST_TABLE, new Authorizations("blah")); IteratorSetting is = new IteratorSetting( 1, PrimitiveComparisonFilter.FILTER_PREFIX + 1, PrimitiveComparisonFilter.class); is.addOption(PrimitiveComparisonFilter.P_COMPARE_CLASS, StringCompare.class.getName()); is.addOption(PrimitiveComparisonFilter.COMPARE_OPT_CLASS, Equal.class.getName()); is.addOption( PrimitiveComparisonFilter.CONST_VAL, new String(Base64.encodeBase64("brian".getBytes()))); is.addOption(PrimitiveComparisonFilter.COLUMN, "cf:name"); scan.addScanIterator(is); boolean foundName = false; boolean foundSid = false; boolean foundDegrees = false; boolean foundMillis = false; for (Map.Entry<Key, Value> kv : scan) { SortedMap<Key, Value> items = PrimitiveComparisonFilter.decodeRow(kv.getKey(), kv.getValue()); for (Map.Entry<Key, Value> item : items.entrySet()) { assertEquals(item.getKey().getRow().toString(), "r1"); if (item.getKey().getColumnQualifier().equals(NAME)) { foundName = true; assertArrayEquals(item.getValue().get(), "brian".getBytes()); } else if (item.getKey().getColumnQualifier().equals(SID)) { foundSid = true; assertArrayEquals(item.getValue().get(), parseIntBytes("1")); } else if (item.getKey().getColumnQualifier().equals(DEGREES)) { foundDegrees = true; assertArrayEquals(item.getValue().get(), parseDoubleBytes("44.5")); } else if (item.getKey().getColumnQualifier().equals(MILLIS)) { foundMillis = true; assertArrayEquals(item.getValue().get(), parseLongBytes("555")); } } } assertTrue(foundDegrees & foundMillis & foundName & foundSid); }
/** * A convenience method for setting the end timestamp accepted by the timestamp filter. * * @param is the iterator setting object to configure * @param end the end timestamp * @param endInclusive boolean indicating whether the end is inclusive */ public static void setEnd(IteratorSetting is, long end, boolean endInclusive) { is.addOption(END, LONG_PREFIX + Long.toString(end)); is.addOption(END_INCL, Boolean.toString(endInclusive)); }
/** * A convenience method for setting the start timestamp accepted by the timestamp filter. * * @param is the iterator setting object to configure * @param start the start timestamp * @param startInclusive boolean indicating whether the start is inclusive */ public static void setStart(IteratorSetting is, long start, boolean startInclusive) { is.addOption(START, LONG_PREFIX + Long.toString(start)); is.addOption(START_INCL, Boolean.toString(startInclusive)); }
public static IteratorSetting iteratorSetting(int priority, int matrixSize, int numIterations) { IteratorSetting itset = new IteratorSetting(priority, InverseMatrixIterator.class); itset.addOption(MATRIX_SIZE, Integer.toString(matrixSize)); itset.addOption(NUMITERATIONS, Integer.toString(numIterations)); return itset; }
/** * A convenience method for setting the long encoding type. * * @param is IteratorSetting object to configure. * @param type LongCombiner.Type specifying the encoding type. */ public static void setEncodingType(IteratorSetting is, Type type) { is.addOption(TYPE, type.toString()); }