private static void runTest( Connector conn, int numlg, ArrayList<byte[]> cfset, Map<String, Stat> stats) throws Exception { String table = "immlgb"; try { conn.tableOperations().delete(table); } catch (TableNotFoundException tnfe) { } conn.tableOperations().create(table); conn.tableOperations() .setProperty(table, Property.TABLE_FILE_COMPRESSION_TYPE.getKey(), "snappy"); setupLocalityGroups(conn, numlg, cfset, table); addStat(stats, "write", write(conn, cfset, table)); addStat(stats, "scan cf", scan(conn, cfset, table, false)); addStat(stats, "scan cf:cq", scan(conn, cfset, table, true)); // TODO time reading all data long t1 = System.currentTimeMillis(); conn.tableOperations().flush(table, null, null, true); long t2 = System.currentTimeMillis(); addStat(stats, "flush", t2 - t1); }
@Test(timeout = 5 * 60 * 1000) public void testConcurrentAssignmentPerformance() throws Exception { // make a table with a lot of splits String tableName = getUniqueNames(1)[0]; Connector c = getConnector(); c.tableOperations().create(tableName); SortedSet<Text> splits = new TreeSet<Text>(); for (int i = 0; i < 4000; i++) { splits.add(new Text(randomHex(8))); } c.tableOperations().addSplits(tableName, splits); c.tableOperations().offline(tableName, true); // time how long it takes to load long now = System.currentTimeMillis(); c.tableOperations().online(tableName, true); long diff = System.currentTimeMillis() - now; log.debug("Loaded " + splits.size() + " tablets in " + diff + " ms"); c.instanceOperations().setProperty(Property.TSERV_ASSIGNMENT_MAXCONCURRENT.getKey(), "20"); now = System.currentTimeMillis(); c.tableOperations().offline(tableName, true); // wait >10 seconds for thread pool to update UtilWaitThread.sleep(Math.max(0, now + 11 * 1000 - System.currentTimeMillis())); now = System.currentTimeMillis(); c.tableOperations().online(tableName, true); long diff2 = System.currentTimeMillis() - now; log.debug("Loaded " + splits.size() + " tablets in " + diff2 + " ms"); assertTrue(diff2 < diff); }
@Override public void setup() { Connector conn = getConnector(); String tableName = getTestProperty("TABLE"); // delete existing table if (conn.tableOperations().exists(tableName)) { System.out.println("Deleting existing table: " + tableName); try { conn.tableOperations().delete(tableName); } catch (Exception e) { log.error("Failed to delete table '" + tableName + "'.", e); } } // create table try { conn.tableOperations().create(tableName); conn.tableOperations().addSplits(tableName, calculateSplits()); conn.tableOperations().setProperty(tableName, "table.split.threshold", "256M"); } catch (Exception e) { log.error("Failed to create table '" + tableName + "'.", e); } }
@Before public void init() throws AccumuloException, AccumuloSecurityException, TableNotFoundException, TableExistsException { mock = new MockInstance("accumulo"); PasswordToken pToken = new PasswordToken("pass".getBytes()); conn = mock.getConnector("user", pToken); config = new BatchWriterConfig(); config.setMaxMemory(1000); config.setMaxLatency(1000, TimeUnit.SECONDS); config.setMaxWriteThreads(10); if (conn.tableOperations().exists("rya_prospects")) { conn.tableOperations().delete("rya_prospects"); } if (conn.tableOperations().exists("rya_selectivity")) { conn.tableOperations().delete("rya_selectivity"); } arc = new AccumuloRdfConfiguration(); arc.setTableLayoutStrategy(new TablePrefixLayoutStrategy()); arc.setMaxRangesForScanner(300); res = new ProspectorServiceEvalStatsDAO(conn, arc); }
public void addTable(Text tableName) throws AccumuloException, AccumuloSecurityException { if (simulate) { log.info("Simulating adding table: " + tableName); return; } log.debug("Adding table: " + tableName); BatchWriter bw = null; String table = tableName.toString(); if (createTables && !conn.tableOperations().exists(table)) { try { conn.tableOperations().create(table); } catch (AccumuloSecurityException e) { log.error("Accumulo security violation creating " + table, e); throw e; } catch (TableExistsException e) { // Shouldn't happen } } try { bw = mtbw.getBatchWriter(table); } catch (TableNotFoundException e) { log.error("Accumulo table " + table + " doesn't exist and cannot be created.", e); throw new AccumuloException(e); } catch (AccumuloException e) { throw e; } catch (AccumuloSecurityException e) { throw e; } if (bw != null) bws.put(tableName, bw); }
@Test public void test() throws Exception { Connector c = getConnector(); String tableName = getUniqueNames(1)[0]; c.tableOperations().create(tableName); BatchWriter bw = c.createBatchWriter(tableName, new BatchWriterConfig()); Mutation m = new Mutation("row1"); m.put("cf", "col1", "Test"); bw.addMutation(m); bw.close(); scanCheck(c, tableName, "Test"); FileSystem fs = getCluster().getFileSystem(); Path jarPath = new Path(rootPath + "/lib/ext/Test.jar"); copyStreamToFileSystem(fs, "/TestCombinerX.jar", jarPath); sleepUninterruptibly(1, TimeUnit.SECONDS); IteratorSetting is = new IteratorSetting(10, "TestCombiner", "org.apache.accumulo.test.functional.TestCombiner"); Combiner.setColumns(is, Collections.singletonList(new IteratorSetting.Column("cf"))); c.tableOperations().attachIterator(tableName, is, EnumSet.of(IteratorScope.scan)); sleepUninterruptibly(ZOOKEEPER_PROPAGATION_TIME, TimeUnit.MILLISECONDS); scanCheck(c, tableName, "TestX"); fs.delete(jarPath, true); copyStreamToFileSystem(fs, "/TestCombinerY.jar", jarPath); sleepUninterruptibly(5, TimeUnit.SECONDS); scanCheck(c, tableName, "TestY"); fs.delete(jarPath, true); }
@Test public void test() throws Exception { Connector c = getConnector(); // make a table String tableName = getUniqueNames(1)[0]; c.tableOperations().create(tableName); // write to it BatchWriter bw = c.createBatchWriter(tableName, new BatchWriterConfig()); Mutation m = new Mutation("row"); m.put("cf", "cq", "value"); bw.addMutation(m); bw.close(); // create a fake _tmp file in its directory String id = c.tableOperations().tableIdMap().get(tableName); FileSystem fs = getCluster().getFileSystem(); Path tmp = new Path("/accumulo/tables/" + id + "/default_tablet/junk.rf_tmp"); fs.create(tmp).close(); for (ProcessReference tserver : getCluster().getProcesses().get(ServerType.TABLET_SERVER)) { getCluster().killProcess(ServerType.TABLET_SERVER, tserver); } getCluster().start(); Scanner scanner = c.createScanner(tableName, Authorizations.EMPTY); FunctionalTestUtils.count(scanner); assertFalse(fs.exists(tmp)); }
@Override public int run(String[] args) { Opts opts = new Opts(); opts.parseArgs(BulkIngestExample.class.getName(), args); Configuration conf = getConf(); PrintStream out = null; try { Job job = JobUtil.getJob(conf); job.setJobName("bulk ingest example"); job.setJarByClass(this.getClass()); job.setInputFormatClass(TextInputFormat.class); job.setMapperClass(MapClass.class); job.setMapOutputKeyClass(Text.class); job.setMapOutputValueClass(Text.class); job.setReducerClass(ReduceClass.class); job.setOutputFormatClass(AccumuloFileOutputFormat.class); opts.setAccumuloConfigs(job); Connector connector = opts.getConnector(); TextInputFormat.setInputPaths(job, new Path(opts.inputDir)); AccumuloFileOutputFormat.setOutputPath(job, new Path(opts.workDir + "/files")); FileSystem fs = FileSystem.get(conf); out = new PrintStream( new BufferedOutputStream(fs.create(new Path(opts.workDir + "/splits.txt")))); Collection<Text> splits = connector.tableOperations().listSplits(opts.getTableName(), 100); for (Text split : splits) out.println(new String(Base64.encodeBase64(TextUtil.getBytes(split)))); job.setNumReduceTasks(splits.size() + 1); out.close(); job.setPartitionerClass(RangePartitioner.class); RangePartitioner.setSplitFile(job, opts.workDir + "/splits.txt"); job.waitForCompletion(true); Path failures = new Path(opts.workDir, "failures"); fs.delete(failures, true); fs.mkdirs(new Path(opts.workDir, "failures")); connector .tableOperations() .importDirectory( opts.getTableName(), opts.workDir + "/files", opts.workDir + "/failures", false); } catch (Exception e) { throw new RuntimeException(e); } finally { if (out != null) out.close(); } return 0; }
@Test public void testBulkSplitOptimization() throws Exception { final Connector c = getConnector(); final String tableName = getUniqueNames(1)[0]; c.tableOperations().create(tableName); c.tableOperations().setProperty(tableName, Property.TABLE_MAJC_RATIO.getKey(), "1000"); c.tableOperations().setProperty(tableName, Property.TABLE_FILE_MAX.getKey(), "1000"); c.tableOperations().setProperty(tableName, Property.TABLE_SPLIT_THRESHOLD.getKey(), "1G"); FileSystem fs = cluster.getFileSystem(); Path testDir = new Path(getUsableDir(), "testmf"); FunctionalTestUtils.createRFiles(c, fs, testDir.toString(), ROWS, SPLITS, 8); FileStatus[] stats = fs.listStatus(testDir); System.out.println("Number of generated files: " + stats.length); FunctionalTestUtils.bulkImport(c, fs, tableName, testDir.toString()); FunctionalTestUtils.checkSplits(c, tableName, 0, 0); FunctionalTestUtils.checkRFiles(c, tableName, 1, 1, 100, 100); // initiate splits getConnector() .tableOperations() .setProperty(tableName, Property.TABLE_SPLIT_THRESHOLD.getKey(), "100K"); sleepUninterruptibly(2, TimeUnit.SECONDS); // wait until over split threshold -- should be 78 splits while (getConnector().tableOperations().listSplits(tableName).size() < 75) { sleepUninterruptibly(500, TimeUnit.MILLISECONDS); } FunctionalTestUtils.checkSplits(c, tableName, 50, 100); VerifyIngest.Opts opts = new VerifyIngest.Opts(); opts.timestamp = 1; opts.dataSize = 50; opts.random = 56; opts.rows = 100000; opts.startRow = 0; opts.cols = 1; opts.setTableName(tableName); AuthenticationToken adminToken = getAdminToken(); if (adminToken instanceof PasswordToken) { PasswordToken token = (PasswordToken) getAdminToken(); opts.setPassword(new Password(new String(token.getPassword(), UTF_8))); opts.setPrincipal(getAdminPrincipal()); } else if (adminToken instanceof KerberosToken) { ClientConfiguration clientConf = cluster.getClientConfig(); opts.updateKerberosCredentials(clientConf); } else { Assert.fail("Unknown token type"); } VerifyIngest.verifyIngest(c, opts, new ScannerOpts()); // ensure each tablet does not have all map files, should be ~2.5 files per tablet FunctionalTestUtils.checkRFiles(c, tableName, 50, 100, 1, 4); }
private void runMergeTest( Connector conn, String table, String[] splits, String[] expectedSplits, String[] inserts, String start, String end) throws Exception { System.out.println( "Running merge test " + table + " " + Arrays.asList(splits) + " " + start + " " + end); conn.tableOperations().create(table, new NewTableConfiguration().setTimeType(TimeType.LOGICAL)); TreeSet<Text> splitSet = new TreeSet<Text>(); for (String split : splits) { splitSet.add(new Text(split)); } conn.tableOperations().addSplits(table, splitSet); BatchWriter bw = conn.createBatchWriter(table, null); HashSet<String> expected = new HashSet<String>(); for (String row : inserts) { Mutation m = new Mutation(row); m.put("cf", "cq", row); bw.addMutation(m); expected.add(row); } bw.close(); conn.tableOperations() .merge(table, start == null ? null : new Text(start), end == null ? null : new Text(end)); Scanner scanner = conn.createScanner(table, Authorizations.EMPTY); HashSet<String> observed = new HashSet<String>(); for (Entry<Key, Value> entry : scanner) { String row = entry.getKey().getRowData().toString(); if (!observed.add(row)) { throw new Exception("Saw data twice " + table + " " + row); } } if (!observed.equals(expected)) { throw new Exception("data inconsistency " + table + " " + observed + " != " + expected); } HashSet<Text> currentSplits = new HashSet<Text>(conn.tableOperations().listSplits(table)); HashSet<Text> ess = new HashSet<Text>(); for (String es : expectedSplits) { ess.add(new Text(es)); } if (!currentSplits.equals(ess)) { throw new Exception("split inconsistency " + table + " " + currentSplits + " != " + ess); } }
@Test public void interleaveSplit() throws Exception { Connector c = getConnector(); c.tableOperations().create("test_ingest"); c.tableOperations().setProperty("test_ingest", Property.TABLE_SPLIT_THRESHOLD.getKey(), "10K"); c.tableOperations() .setProperty("test_ingest", Property.TABLE_FILE_COMPRESSION_TYPE.getKey(), "none"); ReadWriteIT.interleaveTest(c); UtilWaitThread.sleep(5 * 1000); assertTrue(c.tableOperations().listSplits("test_ingest").size() > 20); }
@Test(timeout = 4 * 60 * 1000) public void binaryStressTest() throws Exception { Connector c = getConnector(); c.tableOperations().create("bt"); c.tableOperations().setProperty("bt", Property.TABLE_SPLIT_THRESHOLD.getKey(), "10K"); BinaryIT.runTest(c); String id = c.tableOperations().tableIdMap().get("bt"); FileSystem fs = FileSystem.get(CachedConfiguration.getInstance()); FileStatus[] dir = fs.listStatus(new Path(cluster.getConfig().getDir() + "/accumulo/tables/" + id)); assertTrue(dir.length > 7); }
@Test public void deleteSplit() throws Exception { Connector c = getConnector(); c.tableOperations().create("test_ingest"); c.tableOperations().setProperty("test_ingest", Property.TABLE_SPLIT_THRESHOLD.getKey(), "10K"); DeleteIT.deleteTest(c, cluster); c.tableOperations().flush("test_ingest", null, null, true); for (int i = 0; i < 5; i++) { UtilWaitThread.sleep(10 * 1000); if (c.tableOperations().listSplits("test_ingest").size() > 20) break; } assertTrue(c.tableOperations().listSplits("test_ingest").size() > 20); }
/** * Insert the contents of an NDSI data file into an Accumulo table. * * @param file Input file * @param Atable Table to insert into * @return Nummber of entries inserted into Atable; equal to 2x the number of rows. * @throws IOException */ public long ingestFile(File file, String Atable, boolean deleteIfExists) throws IOException { if (deleteIfExists && connector.tableOperations().exists(Atable)) try { connector.tableOperations().delete(Atable); } catch (AccumuloException | AccumuloSecurityException e) { log.warn("trouble deleting table " + Atable, e); throw new RuntimeException(e); } catch (TableNotFoundException e) { throw new RuntimeException(e); } if (!connector.tableOperations().exists(Atable)) try { connector.tableOperations().create(Atable); } catch (AccumuloException | AccumuloSecurityException e) { log.warn("trouble creating table " + Atable, e); throw new RuntimeException(e); } catch (TableExistsException e) { throw new RuntimeException(e); } BatchWriter bw = null; String line = null; long entriesProcessed = 0; try (BufferedReader fo = new BufferedReader(new FileReader(file))) { BatchWriterConfig config = new BatchWriterConfig(); bw = connector.createBatchWriter(Atable, config); // Skip header line fo.readLine(); while ((line = fo.readLine()) != null) if (!line.isEmpty()) entriesProcessed += ingestLine(bw, line); } catch (TableNotFoundException e) { throw new RuntimeException(e); } catch (MutationsRejectedException e) { log.warn("Mutation rejected on line " + line, e); } finally { if (bw != null) try { bw.close(); } catch (MutationsRejectedException e) { log.warn("Mutation rejected at close() on line " + line, e); } } return entriesProcessed; }
/** Write entries to a table. */ public static void writeEntries( Connector connector, Map<Key, Value> map, String table, boolean createIfNotExist) { if (createIfNotExist && !connector.tableOperations().exists(table)) try { connector.tableOperations().create(table); } catch (AccumuloException | AccumuloSecurityException e) { log.error("trouble creating " + table, e); throw new RuntimeException(e); } catch (TableExistsException e) { log.error("crazy", e); throw new RuntimeException(e); } BatchWriterConfig bwc = new BatchWriterConfig(); BatchWriter bw; try { bw = connector.createBatchWriter(table, bwc); } catch (TableNotFoundException e) { log.error("tried to write to a non-existant table " + table, e); throw new RuntimeException(e); } try { for (Map.Entry<Key, Value> entry : map.entrySet()) { Key k = entry.getKey(); ByteSequence rowData = k.getRowData(), cfData = k.getColumnFamilyData(), cqData = k.getColumnQualifierData(); Mutation m = new Mutation(rowData.getBackingArray(), rowData.offset(), rowData.length()); m.put( cfData.getBackingArray(), cqData.getBackingArray(), k.getColumnVisibilityParsed(), entry.getValue().get()); bw.addMutation(m); } } catch (MutationsRejectedException e) { log.error("mutations rejected", e); throw new RuntimeException(e); } finally { try { bw.close(); } catch (MutationsRejectedException e) { log.error("mutations rejected while trying to close BatchWriter", e); } } }
public static synchronized void configureMetadataTable(Connector conn, String tableName) { TableOperations tops = conn.tableOperations(); Map<String, EnumSet<IteratorScope>> iterators = null; try { iterators = tops.listIterators(tableName); } catch (AccumuloSecurityException | AccumuloException | TableNotFoundException e) { throw new RuntimeException(e); } if (!iterators.containsKey(COMBINER_NAME)) { // Set our combiner and combine all columns // Need to set the combiner beneath versioning since we don't want to turn it off IteratorSetting setting = new IteratorSetting(9, COMBINER_NAME, StatusCombiner.class); Combiner.setColumns( setting, Collections.singletonList(new Column(MetadataSchema.ReplicationSection.COLF))); try { tops.attachIterator(tableName, setting); } catch (AccumuloSecurityException | AccumuloException | TableNotFoundException e) { throw new RuntimeException(e); } } // Make sure the StatusFormatter is set on the metadata table Iterable<Entry<String, String>> properties; try { properties = tops.getProperties(tableName); } catch (AccumuloException | TableNotFoundException e) { throw new RuntimeException(e); } for (Entry<String, String> property : properties) { if (Property.TABLE_FORMATTER_CLASS.getKey().equals(property.getKey())) { if (!STATUS_FORMATTER_CLASS_NAME.equals(property.getValue())) { log.info( "Setting formatter for {} from {} to {}", tableName, property.getValue(), STATUS_FORMATTER_CLASS_NAME); try { tops.setProperty( tableName, Property.TABLE_FORMATTER_CLASS.getKey(), STATUS_FORMATTER_CLASS_NAME); } catch (AccumuloException | AccumuloSecurityException e) { throw new RuntimeException(e); } } // Don't need to keep iterating over the properties after we found the one we were looking // for return; } } // Set the formatter on the table because it wasn't already there try { tops.setProperty( tableName, Property.TABLE_FORMATTER_CLASS.getKey(), STATUS_FORMATTER_CLASS_NAME); } catch (AccumuloException | AccumuloSecurityException e) { throw new RuntimeException(e); } }
@Override public void tearDown(State state, Environment env) throws Exception { // We have resources we need to clean up if (env.isMultiTableBatchWriterInitialized()) { MultiTableBatchWriter mtbw = env.getMultiTableBatchWriter(); try { mtbw.close(); } catch (MutationsRejectedException e) { log.error("Ignoring mutations that weren't flushed", e); } // Reset the MTBW on the state to null env.resetMultiTableBatchWriter(); } // Now we can safely delete the tables log.debug("Dropping tables: " + imageTableName + " " + indexTableName); Connector conn = env.getConnector(); conn.tableOperations().delete(imageTableName); conn.tableOperations().delete(indexTableName); log.debug("Final total of writes: " + state.getLong("totalWrites")); }
public static void printDiskUsage( AccumuloConfiguration acuConf, Collection<String> tables, VolumeManager fs, Connector conn, Printer printer, boolean humanReadable) throws TableNotFoundException, IOException { HashSet<String> tableIds = new HashSet<String>(); // Get table IDs for all tables requested to be 'du' for (String tableName : tables) { String tableId = conn.tableOperations().tableIdMap().get(tableName); if (tableId == null) throw new TableNotFoundException(null, tableName, "Table " + tableName + " not found"); tableIds.add(tableId); } Map<TreeSet<String>, Long> usage = getDiskUsage(acuConf, tableIds, fs, conn); String valueFormat = humanReadable ? "%9s" : "%,24d"; for (Entry<TreeSet<String>, Long> entry : usage.entrySet()) { Object value = humanReadable ? NumUtil.bigNumberForSize(entry.getValue()) : entry.getValue(); printer.print(String.format(valueFormat + " %s", value, entry.getKey())); } }
@Test(timeout = 2 * 60 * 1000) public void test() throws Exception { Connector c = getConnector(); c.tableOperations().create("test_ingest"); deleteTest(c, cluster); assertEquals(0, cluster.exec(Admin.class, "stopAll").waitFor()); }
@Override public void visit(State state, Properties props) throws Exception { Connector conn = state.getConnector(); Random rand = (Random) state.get("rand"); @SuppressWarnings("unchecked") List<String> tableNames = (List<String>) state.get("tables"); String tableName = tableNames.get(rand.nextInt(tableNames.size())); // TODO need to sometimes do null start and end ranges TreeSet<Text> range = new TreeSet<Text>(); range.add(new Text(String.format("%016x", Math.abs(rand.nextLong())))); range.add(new Text(String.format("%016x", Math.abs(rand.nextLong())))); try { boolean wait = rand.nextBoolean(); conn.tableOperations().compact(tableName, range.first(), range.last(), false, wait); log.debug((wait ? "compacted " : "initiated compaction ") + tableName); } catch (TableNotFoundException tne) { log.debug("compact " + tableName + " failed, doesnt exist"); } catch (TableOfflineException toe) { log.debug("compact " + tableName + " failed, offline"); } }
public static void run( String instanceName, String zookeepers, AuthenticationToken rootPassword, String args[]) throws Exception { // edit this method to play with Accumulo Instance instance = new ZooKeeperInstance(instanceName, zookeepers); Connector conn = instance.getConnector("root", rootPassword); conn.tableOperations().create("foo"); BatchWriterConfig bwConfig = new BatchWriterConfig(); bwConfig.setMaxLatency(60000l, java.util.concurrent.TimeUnit.MILLISECONDS); bwConfig.setMaxWriteThreads(3); bwConfig.setMaxMemory(50000000); BatchWriter bw = conn.createBatchWriter("foo", bwConfig); Mutation m = new Mutation("r1"); m.put("cf1", "cq1", "v1"); m.put("cf1", "cq2", "v3"); bw.addMutation(m); bw.close(); Scanner scanner = conn.createScanner("foo", Constants.NO_AUTHS); for (Entry<Key, Value> entry : scanner) { System.out.println(entry.getKey() + " " + entry.getValue()); } }
@Test public void merge() throws Exception { Connector c = getConnector(); String tableName = getUniqueNames(1)[0]; c.tableOperations().create(tableName); c.tableOperations().addSplits(tableName, splits("a b c d e f g h i j k".split(" "))); BatchWriter bw = c.createBatchWriter(tableName, null); for (String row : "a b c d e f g h i j k".split(" ")) { Mutation m = new Mutation(row); m.put("cf", "cq", "value"); bw.addMutation(m); } bw.close(); c.tableOperations().flush(tableName, null, null, true); c.tableOperations().merge(tableName, new Text("c1"), new Text("f1")); assertEquals(8, c.tableOperations().listSplits(tableName).size()); }
@Test public void run() throws Exception { Connector c = getConnector(); String tableName = getUniqueNames(1)[0]; c.tableOperations().create(tableName); for (int i = 0; i < 100000; i++) { c.createScanner(tableName, Authorizations.EMPTY); } }
@Before public void createMockKeyValues() throws Exception { // Make a MockInstance here, by setting the instance name to be the same as this mock instance // we can "trick" the InputFormat into using a MockInstance mockInstance = new MockInstance(test.getMethodName()); inputformat = new HiveAccumuloTableInputFormat(); conf = new JobConf(); conf.set(AccumuloSerDeParameters.TABLE_NAME, TEST_TABLE); conf.set(AccumuloSerDeParameters.USE_MOCK_INSTANCE, "true"); conf.set(AccumuloSerDeParameters.INSTANCE_NAME, test.getMethodName()); conf.set(AccumuloSerDeParameters.USER_NAME, USER); conf.set(AccumuloSerDeParameters.USER_PASS, PASS); conf.set(AccumuloSerDeParameters.ZOOKEEPERS, "localhost:2181"); // not used for mock, but // required by input format. columnNames = Arrays.asList("name", "sid", "dgrs", "mills"); columnTypes = Arrays.<TypeInfo>asList( TypeInfoFactory.stringTypeInfo, TypeInfoFactory.intTypeInfo, TypeInfoFactory.doubleTypeInfo, TypeInfoFactory.longTypeInfo); conf.set(AccumuloSerDeParameters.COLUMN_MAPPINGS, "cf:name,cf:sid,cf:dgrs,cf:mills"); conf.set(serdeConstants.LIST_COLUMNS, "name,sid,dgrs,mills"); conf.set(serdeConstants.LIST_COLUMN_TYPES, "string,int,double,bigint"); con = mockInstance.getConnector(USER, new PasswordToken(PASS.getBytes())); con.tableOperations().create(TEST_TABLE); con.securityOperations().changeUserAuthorizations(USER, new Authorizations("blah")); BatchWriterConfig writerConf = new BatchWriterConfig(); BatchWriter writer = con.createBatchWriter(TEST_TABLE, writerConf); Mutation m1 = new Mutation(new Text("r1")); m1.put(COLUMN_FAMILY, NAME, new Value("brian".getBytes())); m1.put(COLUMN_FAMILY, SID, new Value(parseIntBytes("1"))); m1.put(COLUMN_FAMILY, DEGREES, new Value(parseDoubleBytes("44.5"))); m1.put(COLUMN_FAMILY, MILLIS, new Value(parseLongBytes("555"))); Mutation m2 = new Mutation(new Text("r2")); m2.put(COLUMN_FAMILY, NAME, new Value("mark".getBytes())); m2.put(COLUMN_FAMILY, SID, new Value(parseIntBytes("2"))); m2.put(COLUMN_FAMILY, DEGREES, new Value(parseDoubleBytes("55.5"))); m2.put(COLUMN_FAMILY, MILLIS, new Value(parseLongBytes("666"))); Mutation m3 = new Mutation(new Text("r3")); m3.put(COLUMN_FAMILY, NAME, new Value("dennis".getBytes())); m3.put(COLUMN_FAMILY, SID, new Value(parseIntBytes("3"))); m3.put(COLUMN_FAMILY, DEGREES, new Value(parseDoubleBytes("65.5"))); m3.put(COLUMN_FAMILY, MILLIS, new Value(parseLongBytes("777"))); writer.addMutation(m1); writer.addMutation(m2); writer.addMutation(m3); writer.close(); }
@Test public void tabletShouldSplit() throws Exception { Connector c = getConnector(); c.tableOperations().create("test_ingest"); c.tableOperations().setProperty("test_ingest", Property.TABLE_SPLIT_THRESHOLD.getKey(), "256K"); c.tableOperations() .setProperty("test_ingest", Property.TABLE_FILE_COMPRESSED_BLOCK_SIZE.getKey(), "1K"); TestIngest.Opts opts = new TestIngest.Opts(); opts.rows = 100000; TestIngest.ingest(c, opts, new BatchWriterOpts()); VerifyIngest.Opts vopts = new VerifyIngest.Opts(); vopts.rows = opts.rows; VerifyIngest.verifyIngest(c, vopts, new ScannerOpts()); UtilWaitThread.sleep(15 * 1000); String id = c.tableOperations().tableIdMap().get("test_ingest"); Scanner s = c.createScanner(MetadataTable.NAME, Authorizations.EMPTY); KeyExtent extent = new KeyExtent(new Text(id), null, null); s.setRange(extent.toMetadataRange()); MetadataSchema.TabletsSection.TabletColumnFamily.PREV_ROW_COLUMN.fetch(s); int count = 0; int shortened = 0; for (Entry<Key, Value> entry : s) { extent = new KeyExtent(entry.getKey().getRow(), entry.getValue()); if (extent.getEndRow() != null && extent.getEndRow().toString().length() < 14) shortened++; count++; } assertTrue(shortened > 0); assertTrue(count > 10); assertEquals( 0, cluster .exec( CheckForMetadataProblems.class, "-i", cluster.getInstanceName(), "-u", "root", "-p", ROOT_PASSWORD, "-z", cluster.getZooKeepers()) .waitFor()); }
@Test public void aggregationTest() throws Exception { Connector c = getConnector(); String tableName = getUniqueNames(1)[0]; c.tableOperations().create(tableName); IteratorSetting setting = new IteratorSetting(10, SummingCombiner.class); SummingCombiner.setEncodingType(setting, Type.STRING); SummingCombiner.setColumns( setting, Collections.singletonList(new IteratorSetting.Column("cf"))); c.tableOperations().attachIterator(tableName, setting); BatchWriter bw = c.createBatchWriter(tableName, new BatchWriterConfig()); for (int i = 0; i < 10; i++) { Mutation m = new Mutation("row1"); m.put("cf".getBytes(), "col1".getBytes(), ("" + i).getBytes()); bw.addMutation(m); } bw.close(); checkSum(tableName, c); }
@Override public void visit(State state, Properties props) throws Exception { Connector conn = state.getConnector(); Random rand = (Random) state.get("rand"); @SuppressWarnings("unchecked") List<String> tableNames = (List<String>) state.get("tables"); String tableName = tableNames.get(rand.nextInt(tableNames.size())); Configuration conf = CachedConfiguration.getInstance(); FileSystem fs = FileSystem.get(conf); String bulkDir = "/tmp/concurrent_bulk/b_" + String.format("%016x", Math.abs(rand.nextLong())); fs.mkdirs(new Path(bulkDir)); fs.mkdirs(new Path(bulkDir + "_f")); try { BatchWriter bw = new RFileBatchWriter(conf, fs, bulkDir + "/file01.rf"); try { TreeSet<Long> rows = new TreeSet<Long>(); int numRows = rand.nextInt(100000); for (int i = 0; i < numRows; i++) { rows.add(Math.abs(rand.nextLong())); } for (Long row : rows) { Mutation m = new Mutation(String.format("%016x", row)); long val = Math.abs(rand.nextLong()); for (int j = 0; j < 10; j++) { m.put("cf", "cq" + j, new Value(String.format("%016x", val).getBytes())); } bw.addMutation(m); } } finally { bw.close(); } conn.tableOperations() .importDirectory(tableName, bulkDir, bulkDir + "_f", rand.nextBoolean()); log.debug("BulkImported to " + tableName); } catch (TableNotFoundException e) { log.debug("BulkImport " + tableName + " failed, doesnt exist"); } catch (TableOfflineException toe) { log.debug("BulkImport " + tableName + " failed, offline"); } finally { fs.delete(new Path(bulkDir), true); fs.delete(new Path(bulkDir + "_f"), true); } }
@Override public void teardown() { Connector conn = getConnector(); String tableName = getTestProperty("TABLE"); try { conn.tableOperations().delete(tableName); } catch (Exception e) { log.error("Failed to delete table '" + tableName + "'", e); } }
public static void bulkImport(Connector c, FileSystem fs, String table, String dir) throws Exception { String failDir = dir + "_failures"; Path failPath = new Path(failDir); fs.delete(failPath, true); fs.mkdirs(failPath); c.tableOperations().importDirectory(table, dir, failDir, false); if (fs.listStatus(failPath).length > 0) { throw new Exception("Some files failed to bulk import"); } }
public static void checkSplits(Connector c, String table, int min, int max) throws Exception { Collection<Text> splits = c.tableOperations().listSplits(table); if (splits.size() < min || splits.size() > max) { throw new Exception( "# of table splits points out of range, #splits=" + splits.size() + " table=" + table + " min=" + min + " max=" + max); } }