Esempio n. 1
0
  private static void runTest(
      Connector conn, int numlg, ArrayList<byte[]> cfset, Map<String, Stat> stats)
      throws Exception {
    String table = "immlgb";

    try {
      conn.tableOperations().delete(table);
    } catch (TableNotFoundException tnfe) {
    }

    conn.tableOperations().create(table);
    conn.tableOperations()
        .setProperty(table, Property.TABLE_FILE_COMPRESSION_TYPE.getKey(), "snappy");

    setupLocalityGroups(conn, numlg, cfset, table);

    addStat(stats, "write", write(conn, cfset, table));
    addStat(stats, "scan cf", scan(conn, cfset, table, false));
    addStat(stats, "scan cf:cq", scan(conn, cfset, table, true));
    // TODO time reading all data

    long t1 = System.currentTimeMillis();
    conn.tableOperations().flush(table, null, null, true);
    long t2 = System.currentTimeMillis();

    addStat(stats, "flush", t2 - t1);
  }
 @Test(timeout = 5 * 60 * 1000)
 public void testConcurrentAssignmentPerformance() throws Exception {
   // make a table with a lot of splits
   String tableName = getUniqueNames(1)[0];
   Connector c = getConnector();
   c.tableOperations().create(tableName);
   SortedSet<Text> splits = new TreeSet<Text>();
   for (int i = 0; i < 4000; i++) {
     splits.add(new Text(randomHex(8)));
   }
   c.tableOperations().addSplits(tableName, splits);
   c.tableOperations().offline(tableName, true);
   // time how long it takes to load
   long now = System.currentTimeMillis();
   c.tableOperations().online(tableName, true);
   long diff = System.currentTimeMillis() - now;
   log.debug("Loaded " + splits.size() + " tablets in " + diff + " ms");
   c.instanceOperations().setProperty(Property.TSERV_ASSIGNMENT_MAXCONCURRENT.getKey(), "20");
   now = System.currentTimeMillis();
   c.tableOperations().offline(tableName, true);
   // wait >10 seconds for thread pool to update
   UtilWaitThread.sleep(Math.max(0, now + 11 * 1000 - System.currentTimeMillis()));
   now = System.currentTimeMillis();
   c.tableOperations().online(tableName, true);
   long diff2 = System.currentTimeMillis() - now;
   log.debug("Loaded " + splits.size() + " tablets in " + diff2 + " ms");
   assertTrue(diff2 < diff);
 }
Esempio n. 3
0
  @Override
  public void setup() {

    Connector conn = getConnector();
    String tableName = getTestProperty("TABLE");

    // delete existing table
    if (conn.tableOperations().exists(tableName)) {
      System.out.println("Deleting existing table: " + tableName);
      try {
        conn.tableOperations().delete(tableName);
      } catch (Exception e) {
        log.error("Failed to delete table '" + tableName + "'.", e);
      }
    }

    // create table
    try {
      conn.tableOperations().create(tableName);
      conn.tableOperations().addSplits(tableName, calculateSplits());
      conn.tableOperations().setProperty(tableName, "table.split.threshold", "256M");
    } catch (Exception e) {
      log.error("Failed to create table '" + tableName + "'.", e);
    }
  }
  @Before
  public void init()
      throws AccumuloException, AccumuloSecurityException, TableNotFoundException,
          TableExistsException {

    mock = new MockInstance("accumulo");
    PasswordToken pToken = new PasswordToken("pass".getBytes());
    conn = mock.getConnector("user", pToken);

    config = new BatchWriterConfig();
    config.setMaxMemory(1000);
    config.setMaxLatency(1000, TimeUnit.SECONDS);
    config.setMaxWriteThreads(10);

    if (conn.tableOperations().exists("rya_prospects")) {
      conn.tableOperations().delete("rya_prospects");
    }
    if (conn.tableOperations().exists("rya_selectivity")) {
      conn.tableOperations().delete("rya_selectivity");
    }

    arc = new AccumuloRdfConfiguration();
    arc.setTableLayoutStrategy(new TablePrefixLayoutStrategy());
    arc.setMaxRangesForScanner(300);
    res = new ProspectorServiceEvalStatsDAO(conn, arc);
  }
    public void addTable(Text tableName) throws AccumuloException, AccumuloSecurityException {
      if (simulate) {
        log.info("Simulating adding table: " + tableName);
        return;
      }

      log.debug("Adding table: " + tableName);
      BatchWriter bw = null;
      String table = tableName.toString();

      if (createTables && !conn.tableOperations().exists(table)) {
        try {
          conn.tableOperations().create(table);
        } catch (AccumuloSecurityException e) {
          log.error("Accumulo security violation creating " + table, e);
          throw e;
        } catch (TableExistsException e) {
          // Shouldn't happen
        }
      }

      try {
        bw = mtbw.getBatchWriter(table);
      } catch (TableNotFoundException e) {
        log.error("Accumulo table " + table + " doesn't exist and cannot be created.", e);
        throw new AccumuloException(e);
      } catch (AccumuloException e) {
        throw e;
      } catch (AccumuloSecurityException e) {
        throw e;
      }

      if (bw != null) bws.put(tableName, bw);
    }
Esempio n. 6
0
 @Test
 public void test() throws Exception {
   Connector c = getConnector();
   String tableName = getUniqueNames(1)[0];
   c.tableOperations().create(tableName);
   BatchWriter bw = c.createBatchWriter(tableName, new BatchWriterConfig());
   Mutation m = new Mutation("row1");
   m.put("cf", "col1", "Test");
   bw.addMutation(m);
   bw.close();
   scanCheck(c, tableName, "Test");
   FileSystem fs = getCluster().getFileSystem();
   Path jarPath = new Path(rootPath + "/lib/ext/Test.jar");
   copyStreamToFileSystem(fs, "/TestCombinerX.jar", jarPath);
   sleepUninterruptibly(1, TimeUnit.SECONDS);
   IteratorSetting is =
       new IteratorSetting(10, "TestCombiner", "org.apache.accumulo.test.functional.TestCombiner");
   Combiner.setColumns(is, Collections.singletonList(new IteratorSetting.Column("cf")));
   c.tableOperations().attachIterator(tableName, is, EnumSet.of(IteratorScope.scan));
   sleepUninterruptibly(ZOOKEEPER_PROPAGATION_TIME, TimeUnit.MILLISECONDS);
   scanCheck(c, tableName, "TestX");
   fs.delete(jarPath, true);
   copyStreamToFileSystem(fs, "/TestCombinerY.jar", jarPath);
   sleepUninterruptibly(5, TimeUnit.SECONDS);
   scanCheck(c, tableName, "TestY");
   fs.delete(jarPath, true);
 }
Esempio n. 7
0
  @Test
  public void test() throws Exception {
    Connector c = getConnector();
    // make a table
    String tableName = getUniqueNames(1)[0];
    c.tableOperations().create(tableName);
    // write to it
    BatchWriter bw = c.createBatchWriter(tableName, new BatchWriterConfig());
    Mutation m = new Mutation("row");
    m.put("cf", "cq", "value");
    bw.addMutation(m);
    bw.close();

    // create a fake _tmp file in its directory
    String id = c.tableOperations().tableIdMap().get(tableName);
    FileSystem fs = getCluster().getFileSystem();
    Path tmp = new Path("/accumulo/tables/" + id + "/default_tablet/junk.rf_tmp");
    fs.create(tmp).close();
    for (ProcessReference tserver : getCluster().getProcesses().get(ServerType.TABLET_SERVER)) {
      getCluster().killProcess(ServerType.TABLET_SERVER, tserver);
    }
    getCluster().start();

    Scanner scanner = c.createScanner(tableName, Authorizations.EMPTY);
    FunctionalTestUtils.count(scanner);
    assertFalse(fs.exists(tmp));
  }
Esempio n. 8
0
  @Override
  public int run(String[] args) {
    Opts opts = new Opts();
    opts.parseArgs(BulkIngestExample.class.getName(), args);

    Configuration conf = getConf();
    PrintStream out = null;
    try {
      Job job = JobUtil.getJob(conf);
      job.setJobName("bulk ingest example");
      job.setJarByClass(this.getClass());

      job.setInputFormatClass(TextInputFormat.class);

      job.setMapperClass(MapClass.class);
      job.setMapOutputKeyClass(Text.class);
      job.setMapOutputValueClass(Text.class);

      job.setReducerClass(ReduceClass.class);
      job.setOutputFormatClass(AccumuloFileOutputFormat.class);
      opts.setAccumuloConfigs(job);

      Connector connector = opts.getConnector();

      TextInputFormat.setInputPaths(job, new Path(opts.inputDir));
      AccumuloFileOutputFormat.setOutputPath(job, new Path(opts.workDir + "/files"));

      FileSystem fs = FileSystem.get(conf);
      out =
          new PrintStream(
              new BufferedOutputStream(fs.create(new Path(opts.workDir + "/splits.txt"))));

      Collection<Text> splits = connector.tableOperations().listSplits(opts.getTableName(), 100);
      for (Text split : splits)
        out.println(new String(Base64.encodeBase64(TextUtil.getBytes(split))));

      job.setNumReduceTasks(splits.size() + 1);
      out.close();

      job.setPartitionerClass(RangePartitioner.class);
      RangePartitioner.setSplitFile(job, opts.workDir + "/splits.txt");

      job.waitForCompletion(true);
      Path failures = new Path(opts.workDir, "failures");
      fs.delete(failures, true);
      fs.mkdirs(new Path(opts.workDir, "failures"));
      connector
          .tableOperations()
          .importDirectory(
              opts.getTableName(), opts.workDir + "/files", opts.workDir + "/failures", false);

    } catch (Exception e) {
      throw new RuntimeException(e);
    } finally {
      if (out != null) out.close();
    }

    return 0;
  }
  @Test
  public void testBulkSplitOptimization() throws Exception {
    final Connector c = getConnector();
    final String tableName = getUniqueNames(1)[0];
    c.tableOperations().create(tableName);
    c.tableOperations().setProperty(tableName, Property.TABLE_MAJC_RATIO.getKey(), "1000");
    c.tableOperations().setProperty(tableName, Property.TABLE_FILE_MAX.getKey(), "1000");
    c.tableOperations().setProperty(tableName, Property.TABLE_SPLIT_THRESHOLD.getKey(), "1G");
    FileSystem fs = cluster.getFileSystem();
    Path testDir = new Path(getUsableDir(), "testmf");
    FunctionalTestUtils.createRFiles(c, fs, testDir.toString(), ROWS, SPLITS, 8);
    FileStatus[] stats = fs.listStatus(testDir);

    System.out.println("Number of generated files: " + stats.length);
    FunctionalTestUtils.bulkImport(c, fs, tableName, testDir.toString());
    FunctionalTestUtils.checkSplits(c, tableName, 0, 0);
    FunctionalTestUtils.checkRFiles(c, tableName, 1, 1, 100, 100);

    // initiate splits
    getConnector()
        .tableOperations()
        .setProperty(tableName, Property.TABLE_SPLIT_THRESHOLD.getKey(), "100K");

    sleepUninterruptibly(2, TimeUnit.SECONDS);

    // wait until over split threshold -- should be 78 splits
    while (getConnector().tableOperations().listSplits(tableName).size() < 75) {
      sleepUninterruptibly(500, TimeUnit.MILLISECONDS);
    }

    FunctionalTestUtils.checkSplits(c, tableName, 50, 100);
    VerifyIngest.Opts opts = new VerifyIngest.Opts();
    opts.timestamp = 1;
    opts.dataSize = 50;
    opts.random = 56;
    opts.rows = 100000;
    opts.startRow = 0;
    opts.cols = 1;
    opts.setTableName(tableName);

    AuthenticationToken adminToken = getAdminToken();
    if (adminToken instanceof PasswordToken) {
      PasswordToken token = (PasswordToken) getAdminToken();
      opts.setPassword(new Password(new String(token.getPassword(), UTF_8)));
      opts.setPrincipal(getAdminPrincipal());
    } else if (adminToken instanceof KerberosToken) {
      ClientConfiguration clientConf = cluster.getClientConfig();
      opts.updateKerberosCredentials(clientConf);
    } else {
      Assert.fail("Unknown token type");
    }

    VerifyIngest.verifyIngest(c, opts, new ScannerOpts());

    // ensure each tablet does not have all map files, should be ~2.5 files per tablet
    FunctionalTestUtils.checkRFiles(c, tableName, 50, 100, 1, 4);
  }
Esempio n. 10
0
  private void runMergeTest(
      Connector conn,
      String table,
      String[] splits,
      String[] expectedSplits,
      String[] inserts,
      String start,
      String end)
      throws Exception {
    System.out.println(
        "Running merge test " + table + " " + Arrays.asList(splits) + " " + start + " " + end);

    conn.tableOperations().create(table, new NewTableConfiguration().setTimeType(TimeType.LOGICAL));
    TreeSet<Text> splitSet = new TreeSet<Text>();
    for (String split : splits) {
      splitSet.add(new Text(split));
    }
    conn.tableOperations().addSplits(table, splitSet);

    BatchWriter bw = conn.createBatchWriter(table, null);
    HashSet<String> expected = new HashSet<String>();
    for (String row : inserts) {
      Mutation m = new Mutation(row);
      m.put("cf", "cq", row);
      bw.addMutation(m);
      expected.add(row);
    }

    bw.close();

    conn.tableOperations()
        .merge(table, start == null ? null : new Text(start), end == null ? null : new Text(end));

    Scanner scanner = conn.createScanner(table, Authorizations.EMPTY);

    HashSet<String> observed = new HashSet<String>();
    for (Entry<Key, Value> entry : scanner) {
      String row = entry.getKey().getRowData().toString();
      if (!observed.add(row)) {
        throw new Exception("Saw data twice " + table + " " + row);
      }
    }

    if (!observed.equals(expected)) {
      throw new Exception("data inconsistency " + table + " " + observed + " != " + expected);
    }

    HashSet<Text> currentSplits = new HashSet<Text>(conn.tableOperations().listSplits(table));
    HashSet<Text> ess = new HashSet<Text>();
    for (String es : expectedSplits) {
      ess.add(new Text(es));
    }

    if (!currentSplits.equals(ess)) {
      throw new Exception("split inconsistency " + table + " " + currentSplits + " != " + ess);
    }
  }
Esempio n. 11
0
 @Test
 public void interleaveSplit() throws Exception {
   Connector c = getConnector();
   c.tableOperations().create("test_ingest");
   c.tableOperations().setProperty("test_ingest", Property.TABLE_SPLIT_THRESHOLD.getKey(), "10K");
   c.tableOperations()
       .setProperty("test_ingest", Property.TABLE_FILE_COMPRESSION_TYPE.getKey(), "none");
   ReadWriteIT.interleaveTest(c);
   UtilWaitThread.sleep(5 * 1000);
   assertTrue(c.tableOperations().listSplits("test_ingest").size() > 20);
 }
Esempio n. 12
0
 @Test(timeout = 4 * 60 * 1000)
 public void binaryStressTest() throws Exception {
   Connector c = getConnector();
   c.tableOperations().create("bt");
   c.tableOperations().setProperty("bt", Property.TABLE_SPLIT_THRESHOLD.getKey(), "10K");
   BinaryIT.runTest(c);
   String id = c.tableOperations().tableIdMap().get("bt");
   FileSystem fs = FileSystem.get(CachedConfiguration.getInstance());
   FileStatus[] dir =
       fs.listStatus(new Path(cluster.getConfig().getDir() + "/accumulo/tables/" + id));
   assertTrue(dir.length > 7);
 }
Esempio n. 13
0
 @Test
 public void deleteSplit() throws Exception {
   Connector c = getConnector();
   c.tableOperations().create("test_ingest");
   c.tableOperations().setProperty("test_ingest", Property.TABLE_SPLIT_THRESHOLD.getKey(), "10K");
   DeleteIT.deleteTest(c, cluster);
   c.tableOperations().flush("test_ingest", null, null, true);
   for (int i = 0; i < 5; i++) {
     UtilWaitThread.sleep(10 * 1000);
     if (c.tableOperations().listSplits("test_ingest").size() > 20) break;
   }
   assertTrue(c.tableOperations().listSplits("test_ingest").size() > 20);
 }
Esempio n. 14
0
  /**
   * Insert the contents of an NDSI data file into an Accumulo table.
   *
   * @param file Input file
   * @param Atable Table to insert into
   * @return Nummber of entries inserted into Atable; equal to 2x the number of rows.
   * @throws IOException
   */
  public long ingestFile(File file, String Atable, boolean deleteIfExists) throws IOException {
    if (deleteIfExists && connector.tableOperations().exists(Atable))
      try {
        connector.tableOperations().delete(Atable);
      } catch (AccumuloException | AccumuloSecurityException e) {
        log.warn("trouble deleting table " + Atable, e);
        throw new RuntimeException(e);
      } catch (TableNotFoundException e) {
        throw new RuntimeException(e);
      }
    if (!connector.tableOperations().exists(Atable))
      try {
        connector.tableOperations().create(Atable);
      } catch (AccumuloException | AccumuloSecurityException e) {
        log.warn("trouble creating table " + Atable, e);
        throw new RuntimeException(e);
      } catch (TableExistsException e) {
        throw new RuntimeException(e);
      }

    BatchWriter bw = null;
    String line = null;
    long entriesProcessed = 0;

    try (BufferedReader fo = new BufferedReader(new FileReader(file))) {
      BatchWriterConfig config = new BatchWriterConfig();
      bw = connector.createBatchWriter(Atable, config);

      // Skip header line
      fo.readLine();

      while ((line = fo.readLine()) != null)
        if (!line.isEmpty()) entriesProcessed += ingestLine(bw, line);

    } catch (TableNotFoundException e) {
      throw new RuntimeException(e);
    } catch (MutationsRejectedException e) {
      log.warn("Mutation rejected on line " + line, e);
    } finally {
      if (bw != null)
        try {
          bw.close();
        } catch (MutationsRejectedException e) {
          log.warn("Mutation rejected at close() on line " + line, e);
        }
    }
    return entriesProcessed;
  }
Esempio n. 15
0
  /** Write entries to a table. */
  public static void writeEntries(
      Connector connector, Map<Key, Value> map, String table, boolean createIfNotExist) {
    if (createIfNotExist && !connector.tableOperations().exists(table))
      try {
        connector.tableOperations().create(table);
      } catch (AccumuloException | AccumuloSecurityException e) {
        log.error("trouble creating " + table, e);
        throw new RuntimeException(e);
      } catch (TableExistsException e) {
        log.error("crazy", e);
        throw new RuntimeException(e);
      }

    BatchWriterConfig bwc = new BatchWriterConfig();
    BatchWriter bw;
    try {
      bw = connector.createBatchWriter(table, bwc);
    } catch (TableNotFoundException e) {
      log.error("tried to write to a non-existant table " + table, e);
      throw new RuntimeException(e);
    }

    try {
      for (Map.Entry<Key, Value> entry : map.entrySet()) {
        Key k = entry.getKey();
        ByteSequence rowData = k.getRowData(),
            cfData = k.getColumnFamilyData(),
            cqData = k.getColumnQualifierData();
        Mutation m = new Mutation(rowData.getBackingArray(), rowData.offset(), rowData.length());
        m.put(
            cfData.getBackingArray(),
            cqData.getBackingArray(),
            k.getColumnVisibilityParsed(),
            entry.getValue().get());
        bw.addMutation(m);
      }

    } catch (MutationsRejectedException e) {
      log.error("mutations rejected", e);
      throw new RuntimeException(e);
    } finally {
      try {
        bw.close();
      } catch (MutationsRejectedException e) {
        log.error("mutations rejected while trying to close BatchWriter", e);
      }
    }
  }
Esempio n. 16
0
  public static synchronized void configureMetadataTable(Connector conn, String tableName) {
    TableOperations tops = conn.tableOperations();
    Map<String, EnumSet<IteratorScope>> iterators = null;
    try {
      iterators = tops.listIterators(tableName);
    } catch (AccumuloSecurityException | AccumuloException | TableNotFoundException e) {
      throw new RuntimeException(e);
    }

    if (!iterators.containsKey(COMBINER_NAME)) {
      // Set our combiner and combine all columns
      // Need to set the combiner beneath versioning since we don't want to turn it off
      IteratorSetting setting = new IteratorSetting(9, COMBINER_NAME, StatusCombiner.class);
      Combiner.setColumns(
          setting, Collections.singletonList(new Column(MetadataSchema.ReplicationSection.COLF)));
      try {
        tops.attachIterator(tableName, setting);
      } catch (AccumuloSecurityException | AccumuloException | TableNotFoundException e) {
        throw new RuntimeException(e);
      }
    }

    // Make sure the StatusFormatter is set on the metadata table
    Iterable<Entry<String, String>> properties;
    try {
      properties = tops.getProperties(tableName);
    } catch (AccumuloException | TableNotFoundException e) {
      throw new RuntimeException(e);
    }

    for (Entry<String, String> property : properties) {
      if (Property.TABLE_FORMATTER_CLASS.getKey().equals(property.getKey())) {
        if (!STATUS_FORMATTER_CLASS_NAME.equals(property.getValue())) {
          log.info(
              "Setting formatter for {} from {} to {}",
              tableName,
              property.getValue(),
              STATUS_FORMATTER_CLASS_NAME);
          try {
            tops.setProperty(
                tableName, Property.TABLE_FORMATTER_CLASS.getKey(), STATUS_FORMATTER_CLASS_NAME);
          } catch (AccumuloException | AccumuloSecurityException e) {
            throw new RuntimeException(e);
          }
        }

        // Don't need to keep iterating over the properties after we found the one we were looking
        // for
        return;
      }
    }

    // Set the formatter on the table because it wasn't already there
    try {
      tops.setProperty(
          tableName, Property.TABLE_FORMATTER_CLASS.getKey(), STATUS_FORMATTER_CLASS_NAME);
    } catch (AccumuloException | AccumuloSecurityException e) {
      throw new RuntimeException(e);
    }
  }
Esempio n. 17
0
  @Override
  public void tearDown(State state, Environment env) throws Exception {
    // We have resources we need to clean up
    if (env.isMultiTableBatchWriterInitialized()) {
      MultiTableBatchWriter mtbw = env.getMultiTableBatchWriter();
      try {
        mtbw.close();
      } catch (MutationsRejectedException e) {
        log.error("Ignoring mutations that weren't flushed", e);
      }

      // Reset the MTBW on the state to null
      env.resetMultiTableBatchWriter();
    }

    // Now we can safely delete the tables
    log.debug("Dropping tables: " + imageTableName + " " + indexTableName);

    Connector conn = env.getConnector();

    conn.tableOperations().delete(imageTableName);
    conn.tableOperations().delete(indexTableName);

    log.debug("Final total of writes: " + state.getLong("totalWrites"));
  }
Esempio n. 18
0
  public static void printDiskUsage(
      AccumuloConfiguration acuConf,
      Collection<String> tables,
      VolumeManager fs,
      Connector conn,
      Printer printer,
      boolean humanReadable)
      throws TableNotFoundException, IOException {

    HashSet<String> tableIds = new HashSet<String>();

    // Get table IDs for all tables requested to be 'du'
    for (String tableName : tables) {
      String tableId = conn.tableOperations().tableIdMap().get(tableName);
      if (tableId == null)
        throw new TableNotFoundException(null, tableName, "Table " + tableName + " not found");

      tableIds.add(tableId);
    }

    Map<TreeSet<String>, Long> usage = getDiskUsage(acuConf, tableIds, fs, conn);

    String valueFormat = humanReadable ? "%9s" : "%,24d";
    for (Entry<TreeSet<String>, Long> entry : usage.entrySet()) {
      Object value = humanReadable ? NumUtil.bigNumberForSize(entry.getValue()) : entry.getValue();
      printer.print(String.format(valueFormat + " %s", value, entry.getKey()));
    }
  }
Esempio n. 19
0
 @Test(timeout = 2 * 60 * 1000)
 public void test() throws Exception {
   Connector c = getConnector();
   c.tableOperations().create("test_ingest");
   deleteTest(c, cluster);
   assertEquals(0, cluster.exec(Admin.class, "stopAll").waitFor());
 }
Esempio n. 20
0
  @Override
  public void visit(State state, Properties props) throws Exception {
    Connector conn = state.getConnector();

    Random rand = (Random) state.get("rand");

    @SuppressWarnings("unchecked")
    List<String> tableNames = (List<String>) state.get("tables");

    String tableName = tableNames.get(rand.nextInt(tableNames.size()));

    // TODO need to sometimes do null start and end ranges

    TreeSet<Text> range = new TreeSet<Text>();
    range.add(new Text(String.format("%016x", Math.abs(rand.nextLong()))));
    range.add(new Text(String.format("%016x", Math.abs(rand.nextLong()))));

    try {
      boolean wait = rand.nextBoolean();
      conn.tableOperations().compact(tableName, range.first(), range.last(), false, wait);
      log.debug((wait ? "compacted " : "initiated compaction ") + tableName);
    } catch (TableNotFoundException tne) {
      log.debug("compact " + tableName + " failed, doesnt exist");
    } catch (TableOfflineException toe) {
      log.debug("compact " + tableName + " failed, offline");
    }
  }
Esempio n. 21
0
  public static void run(
      String instanceName, String zookeepers, AuthenticationToken rootPassword, String args[])
      throws Exception {
    // edit this method to play with Accumulo

    Instance instance = new ZooKeeperInstance(instanceName, zookeepers);

    Connector conn = instance.getConnector("root", rootPassword);

    conn.tableOperations().create("foo");

    BatchWriterConfig bwConfig = new BatchWriterConfig();
    bwConfig.setMaxLatency(60000l, java.util.concurrent.TimeUnit.MILLISECONDS);
    bwConfig.setMaxWriteThreads(3);
    bwConfig.setMaxMemory(50000000);
    BatchWriter bw = conn.createBatchWriter("foo", bwConfig);
    Mutation m = new Mutation("r1");
    m.put("cf1", "cq1", "v1");
    m.put("cf1", "cq2", "v3");
    bw.addMutation(m);
    bw.close();

    Scanner scanner = conn.createScanner("foo", Constants.NO_AUTHS);
    for (Entry<Key, Value> entry : scanner) {
      System.out.println(entry.getKey() + " " + entry.getValue());
    }
  }
Esempio n. 22
0
 @Test
 public void merge() throws Exception {
   Connector c = getConnector();
   String tableName = getUniqueNames(1)[0];
   c.tableOperations().create(tableName);
   c.tableOperations().addSplits(tableName, splits("a b c d e f g h i j k".split(" ")));
   BatchWriter bw = c.createBatchWriter(tableName, null);
   for (String row : "a b c d e f g h i j k".split(" ")) {
     Mutation m = new Mutation(row);
     m.put("cf", "cq", "value");
     bw.addMutation(m);
   }
   bw.close();
   c.tableOperations().flush(tableName, null, null, true);
   c.tableOperations().merge(tableName, new Text("c1"), new Text("f1"));
   assertEquals(8, c.tableOperations().listSplits(tableName).size());
 }
 @Test
 public void run() throws Exception {
   Connector c = getConnector();
   String tableName = getUniqueNames(1)[0];
   c.tableOperations().create(tableName);
   for (int i = 0; i < 100000; i++) {
     c.createScanner(tableName, Authorizations.EMPTY);
   }
 }
  @Before
  public void createMockKeyValues() throws Exception {
    // Make a MockInstance here, by setting the instance name to be the same as this mock instance
    // we can "trick" the InputFormat into using a MockInstance
    mockInstance = new MockInstance(test.getMethodName());
    inputformat = new HiveAccumuloTableInputFormat();
    conf = new JobConf();
    conf.set(AccumuloSerDeParameters.TABLE_NAME, TEST_TABLE);
    conf.set(AccumuloSerDeParameters.USE_MOCK_INSTANCE, "true");
    conf.set(AccumuloSerDeParameters.INSTANCE_NAME, test.getMethodName());
    conf.set(AccumuloSerDeParameters.USER_NAME, USER);
    conf.set(AccumuloSerDeParameters.USER_PASS, PASS);
    conf.set(AccumuloSerDeParameters.ZOOKEEPERS, "localhost:2181"); // not used for mock, but
    // required by input format.

    columnNames = Arrays.asList("name", "sid", "dgrs", "mills");
    columnTypes =
        Arrays.<TypeInfo>asList(
            TypeInfoFactory.stringTypeInfo,
            TypeInfoFactory.intTypeInfo,
            TypeInfoFactory.doubleTypeInfo,
            TypeInfoFactory.longTypeInfo);
    conf.set(AccumuloSerDeParameters.COLUMN_MAPPINGS, "cf:name,cf:sid,cf:dgrs,cf:mills");
    conf.set(serdeConstants.LIST_COLUMNS, "name,sid,dgrs,mills");
    conf.set(serdeConstants.LIST_COLUMN_TYPES, "string,int,double,bigint");

    con = mockInstance.getConnector(USER, new PasswordToken(PASS.getBytes()));
    con.tableOperations().create(TEST_TABLE);
    con.securityOperations().changeUserAuthorizations(USER, new Authorizations("blah"));
    BatchWriterConfig writerConf = new BatchWriterConfig();
    BatchWriter writer = con.createBatchWriter(TEST_TABLE, writerConf);

    Mutation m1 = new Mutation(new Text("r1"));
    m1.put(COLUMN_FAMILY, NAME, new Value("brian".getBytes()));
    m1.put(COLUMN_FAMILY, SID, new Value(parseIntBytes("1")));
    m1.put(COLUMN_FAMILY, DEGREES, new Value(parseDoubleBytes("44.5")));
    m1.put(COLUMN_FAMILY, MILLIS, new Value(parseLongBytes("555")));

    Mutation m2 = new Mutation(new Text("r2"));
    m2.put(COLUMN_FAMILY, NAME, new Value("mark".getBytes()));
    m2.put(COLUMN_FAMILY, SID, new Value(parseIntBytes("2")));
    m2.put(COLUMN_FAMILY, DEGREES, new Value(parseDoubleBytes("55.5")));
    m2.put(COLUMN_FAMILY, MILLIS, new Value(parseLongBytes("666")));

    Mutation m3 = new Mutation(new Text("r3"));
    m3.put(COLUMN_FAMILY, NAME, new Value("dennis".getBytes()));
    m3.put(COLUMN_FAMILY, SID, new Value(parseIntBytes("3")));
    m3.put(COLUMN_FAMILY, DEGREES, new Value(parseDoubleBytes("65.5")));
    m3.put(COLUMN_FAMILY, MILLIS, new Value(parseLongBytes("777")));

    writer.addMutation(m1);
    writer.addMutation(m2);
    writer.addMutation(m3);

    writer.close();
  }
Esempio n. 25
0
 @Test
 public void tabletShouldSplit() throws Exception {
   Connector c = getConnector();
   c.tableOperations().create("test_ingest");
   c.tableOperations().setProperty("test_ingest", Property.TABLE_SPLIT_THRESHOLD.getKey(), "256K");
   c.tableOperations()
       .setProperty("test_ingest", Property.TABLE_FILE_COMPRESSED_BLOCK_SIZE.getKey(), "1K");
   TestIngest.Opts opts = new TestIngest.Opts();
   opts.rows = 100000;
   TestIngest.ingest(c, opts, new BatchWriterOpts());
   VerifyIngest.Opts vopts = new VerifyIngest.Opts();
   vopts.rows = opts.rows;
   VerifyIngest.verifyIngest(c, vopts, new ScannerOpts());
   UtilWaitThread.sleep(15 * 1000);
   String id = c.tableOperations().tableIdMap().get("test_ingest");
   Scanner s = c.createScanner(MetadataTable.NAME, Authorizations.EMPTY);
   KeyExtent extent = new KeyExtent(new Text(id), null, null);
   s.setRange(extent.toMetadataRange());
   MetadataSchema.TabletsSection.TabletColumnFamily.PREV_ROW_COLUMN.fetch(s);
   int count = 0;
   int shortened = 0;
   for (Entry<Key, Value> entry : s) {
     extent = new KeyExtent(entry.getKey().getRow(), entry.getValue());
     if (extent.getEndRow() != null && extent.getEndRow().toString().length() < 14) shortened++;
     count++;
   }
   assertTrue(shortened > 0);
   assertTrue(count > 10);
   assertEquals(
       0,
       cluster
           .exec(
               CheckForMetadataProblems.class,
               "-i",
               cluster.getInstanceName(),
               "-u",
               "root",
               "-p",
               ROOT_PASSWORD,
               "-z",
               cluster.getZooKeepers())
           .waitFor());
 }
Esempio n. 26
0
 @Test
 public void aggregationTest() throws Exception {
   Connector c = getConnector();
   String tableName = getUniqueNames(1)[0];
   c.tableOperations().create(tableName);
   IteratorSetting setting = new IteratorSetting(10, SummingCombiner.class);
   SummingCombiner.setEncodingType(setting, Type.STRING);
   SummingCombiner.setColumns(
       setting, Collections.singletonList(new IteratorSetting.Column("cf")));
   c.tableOperations().attachIterator(tableName, setting);
   BatchWriter bw = c.createBatchWriter(tableName, new BatchWriterConfig());
   for (int i = 0; i < 10; i++) {
     Mutation m = new Mutation("row1");
     m.put("cf".getBytes(), "col1".getBytes(), ("" + i).getBytes());
     bw.addMutation(m);
   }
   bw.close();
   checkSum(tableName, c);
 }
Esempio n. 27
0
  @Override
  public void visit(State state, Properties props) throws Exception {
    Connector conn = state.getConnector();

    Random rand = (Random) state.get("rand");

    @SuppressWarnings("unchecked")
    List<String> tableNames = (List<String>) state.get("tables");

    String tableName = tableNames.get(rand.nextInt(tableNames.size()));

    Configuration conf = CachedConfiguration.getInstance();
    FileSystem fs = FileSystem.get(conf);

    String bulkDir = "/tmp/concurrent_bulk/b_" + String.format("%016x", Math.abs(rand.nextLong()));

    fs.mkdirs(new Path(bulkDir));
    fs.mkdirs(new Path(bulkDir + "_f"));

    try {
      BatchWriter bw = new RFileBatchWriter(conf, fs, bulkDir + "/file01.rf");
      try {
        TreeSet<Long> rows = new TreeSet<Long>();
        int numRows = rand.nextInt(100000);
        for (int i = 0; i < numRows; i++) {
          rows.add(Math.abs(rand.nextLong()));
        }

        for (Long row : rows) {
          Mutation m = new Mutation(String.format("%016x", row));
          long val = Math.abs(rand.nextLong());
          for (int j = 0; j < 10; j++) {
            m.put("cf", "cq" + j, new Value(String.format("%016x", val).getBytes()));
          }

          bw.addMutation(m);
        }
      } finally {
        bw.close();
      }

      conn.tableOperations()
          .importDirectory(tableName, bulkDir, bulkDir + "_f", rand.nextBoolean());

      log.debug("BulkImported to " + tableName);
    } catch (TableNotFoundException e) {
      log.debug("BulkImport " + tableName + " failed, doesnt exist");
    } catch (TableOfflineException toe) {
      log.debug("BulkImport " + tableName + " failed, offline");
    } finally {
      fs.delete(new Path(bulkDir), true);
      fs.delete(new Path(bulkDir + "_f"), true);
    }
  }
Esempio n. 28
0
  @Override
  public void teardown() {

    Connector conn = getConnector();
    String tableName = getTestProperty("TABLE");

    try {
      conn.tableOperations().delete(tableName);
    } catch (Exception e) {
      log.error("Failed to delete table '" + tableName + "'", e);
    }
  }
Esempio n. 29
0
  public static void bulkImport(Connector c, FileSystem fs, String table, String dir)
      throws Exception {
    String failDir = dir + "_failures";
    Path failPath = new Path(failDir);
    fs.delete(failPath, true);
    fs.mkdirs(failPath);

    c.tableOperations().importDirectory(table, dir, failDir, false);

    if (fs.listStatus(failPath).length > 0) {
      throw new Exception("Some files failed to bulk import");
    }
  }
Esempio n. 30
0
 public static void checkSplits(Connector c, String table, int min, int max) throws Exception {
   Collection<Text> splits = c.tableOperations().listSplits(table);
   if (splits.size() < min || splits.size() > max) {
     throw new Exception(
         "# of table splits points out of range, #splits="
             + splits.size()
             + " table="
             + table
             + " min="
             + min
             + " max="
             + max);
   }
 }