/**
   * Test adding iterator options where the keys and values contain both the FIELD_SEPARATOR
   * character (':') and ITERATOR_SEPARATOR (',') characters. There should be no exceptions thrown
   * when trying to parse these types of option entries.
   *
   * <p>This test makes sure that the expected raw values, as appears in the Job, are equal to
   * what's expected.
   */
  @Test
  public void testIteratorOptionEncoding() throws Throwable {
    String key = "colon:delimited:key";
    String value = "comma,delimited,value";
    IteratorSetting someSetting = new IteratorSetting(1, "iterator", "Iterator.class");
    someSetting.addOption(key, value);
    @SuppressWarnings("deprecation")
    Job job = new Job();
    AccumuloInputFormat.addIterator(job, someSetting);

    List<IteratorSetting> list = AccumuloInputFormat.getIterators(job);
    assertEquals(1, list.size());
    assertEquals(1, list.get(0).getOptions().size());
    assertEquals(list.get(0).getOptions().get(key), value);

    someSetting.addOption(key + "2", value);
    someSetting.setPriority(2);
    someSetting.setName("it2");
    AccumuloInputFormat.addIterator(job, someSetting);
    list = AccumuloInputFormat.getIterators(job);
    assertEquals(2, list.size());
    assertEquals(1, list.get(0).getOptions().size());
    assertEquals(list.get(0).getOptions().get(key), value);
    assertEquals(2, list.get(1).getOptions().size());
    assertEquals(list.get(1).getOptions().get(key), value);
    assertEquals(list.get(1).getOptions().get(key + "2"), value);
  }
  @Test
  public void testIteratorNotInSplitsCompensation() throws Exception {
    FileInputFormat.addInputPath(conf, new Path("unused"));
    InputSplit[] splits = inputformat.getSplits(conf, 0);

    assertEquals(1, splits.length);
    InputSplit split = splits[0];

    IteratorSetting is =
        new IteratorSetting(
            1, PrimitiveComparisonFilter.FILTER_PREFIX + 1, PrimitiveComparisonFilter.class);

    is.addOption(PrimitiveComparisonFilter.P_COMPARE_CLASS, StringCompare.class.getName());
    is.addOption(PrimitiveComparisonFilter.COMPARE_OPT_CLASS, Equal.class.getName());
    is.addOption(
        PrimitiveComparisonFilter.CONST_VAL, new String(Base64.encodeBase64(new byte[] {'0'})));
    is.addOption(PrimitiveComparisonFilter.COLUMN, "cf:cq");

    // Mock out the predicate handler because it's just easier
    AccumuloPredicateHandler predicateHandler = Mockito.mock(AccumuloPredicateHandler.class);
    Mockito.when(
            predicateHandler.getIterators(
                Mockito.any(JobConf.class), Mockito.any(ColumnMapper.class)))
        .thenReturn(Arrays.asList(is));

    // Set it on our inputformat
    inputformat.predicateHandler = predicateHandler;

    inputformat.getRecordReader(split, conf, null);

    // The code should account for the bug and update the iterators on the split
    List<IteratorSetting> settingsOnSplit = ((HiveAccumuloSplit) split).getSplit().getIterators();
    assertEquals(1, settingsOnSplit.size());
    assertEquals(is, settingsOnSplit.get(0));
  }
Пример #3
0
  @Test
  public void testAllFieldsWritable() throws IOException {
    Range[] ranges = new Range[] {new Range(new Key("a"), new Key("b"))};
    BatchInputSplit split =
        new BatchInputSplit("table", "1", Arrays.asList(ranges), new String[] {"localhost"});

    Set<Pair<Text, Text>> fetchedColumns = new HashSet<>();

    fetchedColumns.add(new Pair<>(new Text("colf1"), new Text("colq1")));
    fetchedColumns.add(new Pair<>(new Text("colf2"), new Text("colq2")));

    // Fake some iterators
    ArrayList<IteratorSetting> iterators = new ArrayList<>();
    IteratorSetting setting = new IteratorSetting(50, SummingCombiner.class);
    setting.addOption("foo", "bar");
    iterators.add(setting);

    setting = new IteratorSetting(100, WholeRowIterator.class);
    setting.addOption("bar", "foo");
    iterators.add(setting);

    split.setTableName("table");
    split.setAuths(new Authorizations("foo"));
    split.setFetchedColumns(fetchedColumns);
    split.setToken(new PasswordToken("password"));
    split.setPrincipal("root");
    DeprecationUtil.setMockInstance(split, true);
    split.setInstanceName("instance");
    split.setZooKeepers("localhost");
    split.setIterators(iterators);
    split.setLogLevel(Level.WARN);

    ByteArrayOutputStream baos = new ByteArrayOutputStream();
    DataOutputStream dos = new DataOutputStream(baos);
    split.write(dos);

    BatchInputSplit newSplit = new BatchInputSplit();

    ByteArrayInputStream bais = new ByteArrayInputStream(baos.toByteArray());
    DataInputStream dis = new DataInputStream(bais);
    newSplit.readFields(dis);

    Assert.assertEquals(split.getRanges(), newSplit.getRanges());
    Assert.assertArrayEquals(split.getLocations(), newSplit.getLocations());

    Assert.assertEquals(split.getTableName(), newSplit.getTableName());
    Assert.assertEquals(split.getAuths(), newSplit.getAuths());
    Assert.assertEquals(split.getFetchedColumns(), newSplit.getFetchedColumns());
    Assert.assertEquals(split.getToken(), newSplit.getToken());
    Assert.assertEquals(split.getPrincipal(), newSplit.getPrincipal());
    Assert.assertEquals(split.getInstanceName(), newSplit.getInstanceName());
    Assert.assertEquals(
        DeprecationUtil.isMockInstanceSet(split), DeprecationUtil.isMockInstanceSet(newSplit));
    Assert.assertEquals(split.getZooKeepers(), newSplit.getZooKeepers());
    Assert.assertEquals(split.getIterators(), newSplit.getIterators());
    Assert.assertEquals(split.getLogLevel(), newSplit.getLogLevel());
  }
  @Test
  public void testConfigureAccumuloInputFormatWithIterators() throws Exception {
    AccumuloConnectionParameters accumuloParams = new AccumuloConnectionParameters(conf);
    ColumnMapper columnMapper =
        new ColumnMapper(
            conf.get(AccumuloSerDeParameters.COLUMN_MAPPINGS),
            conf.get(AccumuloSerDeParameters.DEFAULT_STORAGE_TYPE),
            columnNames,
            columnTypes);
    Set<Pair<Text, Text>> cfCqPairs =
        inputformat.getPairCollection(columnMapper.getColumnMappings());
    List<IteratorSetting> iterators = new ArrayList<IteratorSetting>();
    Set<Range> ranges = Collections.singleton(new Range());
    String instanceName = "realInstance";
    String zookeepers = "host1:2181,host2:2181,host3:2181";

    IteratorSetting cfg = new IteratorSetting(50, PrimitiveComparisonFilter.class);
    cfg.addOption(PrimitiveComparisonFilter.P_COMPARE_CLASS, StringCompare.class.getName());
    cfg.addOption(PrimitiveComparisonFilter.COMPARE_OPT_CLASS, Equal.class.getName());
    cfg.addOption(PrimitiveComparisonFilter.CONST_VAL, "dave");
    cfg.addOption(PrimitiveComparisonFilter.COLUMN, "person:name");
    iterators.add(cfg);

    cfg = new IteratorSetting(50, PrimitiveComparisonFilter.class);
    cfg.addOption(PrimitiveComparisonFilter.P_COMPARE_CLASS, IntCompare.class.getName());
    cfg.addOption(PrimitiveComparisonFilter.COMPARE_OPT_CLASS, Equal.class.getName());
    cfg.addOption(PrimitiveComparisonFilter.CONST_VAL, "50");
    cfg.addOption(PrimitiveComparisonFilter.COLUMN, "person:age");
    iterators.add(cfg);

    ZooKeeperInstance zkInstance = Mockito.mock(ZooKeeperInstance.class);
    HiveAccumuloTableInputFormat mockInputFormat = Mockito.mock(HiveAccumuloTableInputFormat.class);

    // Stub out the ZKI mock
    Mockito.when(zkInstance.getInstanceName()).thenReturn(instanceName);
    Mockito.when(zkInstance.getZooKeepers()).thenReturn(zookeepers);

    // Call out to the real configure method
    Mockito.doCallRealMethod()
        .when(mockInputFormat)
        .configure(conf, zkInstance, con, accumuloParams, columnMapper, iterators, ranges);

    // Also compute the correct cf:cq pairs so we can assert the right argument was passed
    Mockito.doCallRealMethod()
        .when(mockInputFormat)
        .getPairCollection(columnMapper.getColumnMappings());

    mockInputFormat.configure(
        conf, zkInstance, con, accumuloParams, columnMapper, iterators, ranges);

    // Verify that the correct methods are invoked on AccumuloInputFormat
    Mockito.verify(mockInputFormat).setZooKeeperInstance(conf, instanceName, zookeepers, false);
    Mockito.verify(mockInputFormat).setConnectorInfo(conf, USER, new PasswordToken(PASS));
    Mockito.verify(mockInputFormat).setInputTableName(conf, TEST_TABLE);
    Mockito.verify(mockInputFormat)
        .setScanAuthorizations(conf, con.securityOperations().getUserAuthorizations(USER));
    Mockito.verify(mockInputFormat).addIterators(conf, iterators);
    Mockito.verify(mockInputFormat).setRanges(conf, ranges);
    Mockito.verify(mockInputFormat).fetchColumns(conf, cfCqPairs);
  }
  /**
   * Check that the iterator configuration is getting stored in the Job conf correctly.
   *
   * @throws IOException
   */
  @Test
  public void testSetIterator() throws IOException {
    @SuppressWarnings("deprecation")
    Job job = new Job();

    IteratorSetting is =
        new IteratorSetting(1, "WholeRow", "org.apache.accumulo.core.iterators.WholeRowIterator");
    AccumuloInputFormat.addIterator(job, is);
    Configuration conf = job.getConfiguration();
    ByteArrayOutputStream baos = new ByteArrayOutputStream();
    is.write(new DataOutputStream(baos));
    String iterators = conf.get("AccumuloInputFormat.ScanOpts.Iterators");
    assertEquals(new String(Base64.encodeBase64(baos.toByteArray())), iterators);
  }
  @Test
  public void testGreaterThan1Sid() throws Exception {
    Connector con = mockInstance.getConnector(USER, new PasswordToken(PASS.getBytes()));
    Scanner scan = con.createScanner(TEST_TABLE, new Authorizations("blah"));
    IteratorSetting is =
        new IteratorSetting(
            1, PrimitiveComparisonFilter.FILTER_PREFIX + 1, PrimitiveComparisonFilter.class);

    is.addOption(PrimitiveComparisonFilter.P_COMPARE_CLASS, IntCompare.class.getName());
    is.addOption(PrimitiveComparisonFilter.COMPARE_OPT_CLASS, GreaterThan.class.getName());
    is.addOption(
        PrimitiveComparisonFilter.CONST_VAL, new String(Base64.encodeBase64(parseIntBytes("1"))));
    is.addOption(PrimitiveComparisonFilter.COLUMN, "cf:sid");
    scan.addScanIterator(is);
    boolean foundMark = false;
    boolean foundDennis = false;
    int totalCount = 0;
    for (Map.Entry<Key, Value> kv : scan) {
      boolean foundName = false;
      boolean foundSid = false;
      boolean foundDegrees = false;
      boolean foundMillis = false;
      SortedMap<Key, Value> items = PrimitiveComparisonFilter.decodeRow(kv.getKey(), kv.getValue());
      for (Map.Entry<Key, Value> item : items.entrySet()) {
        if (item.getKey().getRow().toString().equals("r2")) {
          foundMark = true;
        } else if (item.getKey().getRow().toString().equals("r3")) {
          foundDennis = true;
        }
        if (item.getKey().getColumnQualifier().equals(NAME)) {
          foundName = true;
        } else if (item.getKey().getColumnQualifier().equals(SID)) {
          foundSid = true;
        } else if (item.getKey().getColumnQualifier().equals(DEGREES)) {
          foundDegrees = true;
        } else if (item.getKey().getColumnQualifier().equals(MILLIS)) {
          foundMillis = true;
        }
      }
      totalCount++;
      assertTrue(foundDegrees & foundMillis & foundName & foundSid);
    }
    assertTrue(foundDennis & foundMark);
    assertEquals(totalCount, 2);
  }
  @Test
  public void testNameEqualBrian() throws Exception {
    Connector con = mockInstance.getConnector(USER, new PasswordToken(PASS.getBytes()));
    Scanner scan = con.createScanner(TEST_TABLE, new Authorizations("blah"));
    IteratorSetting is =
        new IteratorSetting(
            1, PrimitiveComparisonFilter.FILTER_PREFIX + 1, PrimitiveComparisonFilter.class);

    is.addOption(PrimitiveComparisonFilter.P_COMPARE_CLASS, StringCompare.class.getName());
    is.addOption(PrimitiveComparisonFilter.COMPARE_OPT_CLASS, Equal.class.getName());
    is.addOption(
        PrimitiveComparisonFilter.CONST_VAL, new String(Base64.encodeBase64("brian".getBytes())));
    is.addOption(PrimitiveComparisonFilter.COLUMN, "cf:name");
    scan.addScanIterator(is);
    boolean foundName = false;
    boolean foundSid = false;
    boolean foundDegrees = false;
    boolean foundMillis = false;
    for (Map.Entry<Key, Value> kv : scan) {
      SortedMap<Key, Value> items = PrimitiveComparisonFilter.decodeRow(kv.getKey(), kv.getValue());
      for (Map.Entry<Key, Value> item : items.entrySet()) {
        assertEquals(item.getKey().getRow().toString(), "r1");
        if (item.getKey().getColumnQualifier().equals(NAME)) {
          foundName = true;
          assertArrayEquals(item.getValue().get(), "brian".getBytes());
        } else if (item.getKey().getColumnQualifier().equals(SID)) {
          foundSid = true;
          assertArrayEquals(item.getValue().get(), parseIntBytes("1"));
        } else if (item.getKey().getColumnQualifier().equals(DEGREES)) {
          foundDegrees = true;
          assertArrayEquals(item.getValue().get(), parseDoubleBytes("44.5"));
        } else if (item.getKey().getColumnQualifier().equals(MILLIS)) {
          foundMillis = true;
          assertArrayEquals(item.getValue().get(), parseLongBytes("555"));
        }
      }
    }
    assertTrue(foundDegrees & foundMillis & foundName & foundSid);
  }
  public ByteBuffer compress(IteratorSetting[] iterators) {

    UnsynchronizedBuffer.Writer out = new UnsynchronizedBuffer.Writer(iterators.length * 8);

    out.writeVInt(iterators.length);

    for (IteratorSetting is : iterators) {
      out.writeVInt(getSymbolID(is.getName()));
      out.writeVInt(getSymbolID(is.getIteratorClass()));
      out.writeVInt(is.getPriority());

      Map<String, String> opts = is.getOptions();
      out.writeVInt(opts.size());

      for (Entry<String, String> entry : opts.entrySet()) {
        out.writeVInt(getSymbolID(entry.getKey()));
        out.writeVInt(getSymbolID(entry.getValue()));
      }
    }

    return out.toByteBuffer();
  }
Пример #9
0
  @Override
  protected void setTableProperties(
      final CommandLine cl,
      final Shell shellState,
      final int priority,
      final Map<String, String> options,
      final String classname,
      final String name)
      throws AccumuloException, AccumuloSecurityException, ShellCommandException,
          TableNotFoundException {
    // instead of setting table properties, just put the options in a list to use at scan time

    String profile = cl.getOptionValue(profileOpt.getOpt());

    // instead of setting table properties, just put the options in a list to use at scan time
    for (Iterator<Entry<String, String>> i = options.entrySet().iterator(); i.hasNext(); ) {
      final Entry<String, String> entry = i.next();
      if (entry.getValue() == null || entry.getValue().isEmpty()) {
        i.remove();
      }
    }

    List<IteratorSetting> tableScanIterators = shellState.iteratorProfiles.get(profile);
    if (tableScanIterators == null) {
      tableScanIterators = new ArrayList<>();
      shellState.iteratorProfiles.put(profile, tableScanIterators);
    }
    final IteratorSetting setting = new IteratorSetting(priority, name, classname);
    setting.addOptions(options);

    Iterator<IteratorSetting> iter = tableScanIterators.iterator();
    while (iter.hasNext()) {
      if (iter.next().getName().equals(name)) {
        iter.remove();
      }
    }

    tableScanIterators.add(setting);
  }
  /**
   * Test getting iterator settings for multiple iterators set
   *
   * @throws IOException
   */
  @Test
  public void testGetIteratorSettings() throws IOException {
    @SuppressWarnings("deprecation")
    Job job = new Job();

    AccumuloInputFormat.addIterator(
        job,
        new IteratorSetting(1, "WholeRow", "org.apache.accumulo.core.iterators.WholeRowIterator"));
    AccumuloInputFormat.addIterator(
        job,
        new IteratorSetting(
            2, "Versions", "org.apache.accumulo.core.iterators.VersioningIterator"));
    AccumuloInputFormat.addIterator(
        job,
        new IteratorSetting(3, "Count", "org.apache.accumulo.core.iterators.CountingIterator"));

    List<IteratorSetting> list = AccumuloInputFormat.getIterators(job);

    // Check the list size
    assertTrue(list.size() == 3);

    // Walk the list and make sure our settings are correct
    IteratorSetting setting = list.get(0);
    assertEquals(1, setting.getPriority());
    assertEquals("org.apache.accumulo.core.iterators.WholeRowIterator", setting.getIteratorClass());
    assertEquals("WholeRow", setting.getName());

    setting = list.get(1);
    assertEquals(2, setting.getPriority());
    assertEquals(
        "org.apache.accumulo.core.iterators.VersioningIterator", setting.getIteratorClass());
    assertEquals("Versions", setting.getName());

    setting = list.get(2);
    assertEquals(3, setting.getPriority());
    assertEquals("org.apache.accumulo.core.iterators.CountingIterator", setting.getIteratorClass());
    assertEquals("Count", setting.getName());
  }
Пример #11
0
 /**
  * Add the given Iterator to a table on scan, minc, majc scopes. If already present on a scope,
  * does not re-add the iterator to that scope. Call it "plus".
  */
 public static void applyIteratorSoft(IteratorSetting itset, TableOperations tops, String table) {
   // checking if iterator already exists. Not checking for conflicts.
   try {
     IteratorSetting existing;
     EnumSet<IteratorUtil.IteratorScope> enumSet =
         EnumSet.noneOf(IteratorUtil.IteratorScope.class);
     for (IteratorUtil.IteratorScope scope : IteratorUtil.IteratorScope.values()) {
       existing = tops.getIteratorSetting(table, itset.getName(), scope);
       if (existing == null) enumSet.add(scope);
     }
     tops.attachIterator(table, itset, enumSet);
   } catch (AccumuloSecurityException | AccumuloException e) {
     log.error("error trying to add " + itset + " to " + table, e);
     throw new RuntimeException(e);
   } catch (TableNotFoundException e) {
     log.error("no table: " + table, e);
     throw new RuntimeException(e);
   }
 }
Пример #12
0
  public static <K extends WritableComparable<?>, V extends Writable>
      SortedKeyValueIterator<K, V> loadIterators(
          IteratorScope scope,
          SortedKeyValueIterator<K, V> source,
          KeyExtent extent,
          AccumuloConfiguration conf,
          List<IteratorSetting> iterators,
          IteratorEnvironment env)
          throws IOException {

    List<IterInfo> ssiList = new ArrayList<>();
    Map<String, Map<String, String>> ssio = new HashMap<>();

    for (IteratorSetting is : iterators) {
      ssiList.add(new IterInfo(is.getPriority(), is.getIteratorClass(), is.getName()));
      ssio.put(is.getName(), is.getOptions());
    }

    return loadIterators(scope, source, extent, conf, ssiList, ssio, env, true);
  }
Пример #13
0
 /**
  * A convenience method for setting the end timestamp accepted by the timestamp filter.
  *
  * @param is the iterator setting object to configure
  * @param end the end timestamp
  * @param endInclusive boolean indicating whether the end is inclusive
  */
 public static void setEnd(IteratorSetting is, long end, boolean endInclusive) {
   is.addOption(END, LONG_PREFIX + Long.toString(end));
   is.addOption(END_INCL, Boolean.toString(endInclusive));
 }
Пример #14
0
 /**
  * A convenience method for setting the start timestamp accepted by the timestamp filter.
  *
  * @param is the iterator setting object to configure
  * @param start the start timestamp
  * @param startInclusive boolean indicating whether the start is inclusive
  */
 public static void setStart(IteratorSetting is, long start, boolean startInclusive) {
   is.addOption(START, LONG_PREFIX + Long.toString(start));
   is.addOption(START_INCL, Boolean.toString(startInclusive));
 }
 public static IteratorSetting iteratorSetting(int priority, int matrixSize, int numIterations) {
   IteratorSetting itset = new IteratorSetting(priority, InverseMatrixIterator.class);
   itset.addOption(MATRIX_SIZE, Integer.toString(matrixSize));
   itset.addOption(NUMITERATIONS, Integer.toString(numIterations));
   return itset;
 }
 /**
  * A convenience method for setting the long encoding type.
  *
  * @param is IteratorSetting object to configure.
  * @param type LongCombiner.Type specifying the encoding type.
  */
 public static void setEncodingType(IteratorSetting is, Type type) {
   is.addOption(TYPE, type.toString());
 }
Пример #17
0
 public static TIteratorSetting toTIteratorSetting(IteratorSetting is) {
   return new TIteratorSetting(
       is.getPriority(), is.getName(), is.getIteratorClass(), is.getOptions());
 }