private HBaseConfiguration getConf() { HBaseConfiguration conf = new HBaseConfiguration(); // disable compactions in this test. conf.setInt("hbase.hstore.compactionThreshold", 10000); return conf; }
private static void createTable() throws Exception { try { Configuration configuration = HBaseConfiguration.create(); HBaseAdmin.checkHBaseAvailable(configuration); Connection connection = ConnectionFactory.createConnection(configuration); // Instantiating HbaseAdmin class Admin admin = connection.getAdmin(); // Instantiating table descriptor class HTableDescriptor stockTableDesc = new HTableDescriptor(TableName.valueOf(Constants.STOCK_DATES_TABLE)); // Adding column families to table descriptor HColumnDescriptor stock_0414 = new HColumnDescriptor(Constants.STOCK_DATES_CF); stockTableDesc.addFamily(stock_0414); // Execute the table through admin if (!admin.tableExists(stockTableDesc.getTableName())) { admin.createTable(stockTableDesc); System.out.println("Stock table created !!!"); } // Load hbase-site.xml HBaseConfiguration.addHbaseResources(configuration); } catch (ServiceException e) { log.error("Error occurred while creating HBase tables", e); throw new Exception("Error occurred while creating HBase tables", e); } }
public static void main(String[] args) throws IOException { // TODO Auto-generated method stub @SuppressWarnings("deprecation") HBaseConfiguration conf = new HBaseConfiguration(); conf.addResource("/usr/local/hbase-0.98.2/conf/hbase-site.xml"); conf.set("hbase.master", "localhost:60000"); CreateTable(conf); SeedData(conf); }
@Test public void testV1CodecV2Compat() throws Exception { long now = System.currentTimeMillis(); // NOTE: set visibilityUpperBound to 0 as this is expected default for decoding older version // that doesn't store it TreeMap<Long, TransactionManager.InProgressTx> inProgress = Maps.newTreeMap( ImmutableSortedMap.of( 16L, new TransactionManager.InProgressTx(0L, now + 1000), 17L, new TransactionManager.InProgressTx(0L, now + 1000))); TransactionSnapshot snapshot = new TransactionSnapshot( now, 15, 18, Lists.newArrayList(5L, 7L), inProgress, ImmutableMap.<Long, Set<ChangeId>>of( 17L, Sets.newHashSet( new ChangeId(Bytes.toBytes("ch1")), new ChangeId(Bytes.toBytes("ch2")))), ImmutableMap.<Long, Set<ChangeId>>of( 16L, Sets.newHashSet( new ChangeId(Bytes.toBytes("ch2")), new ChangeId(Bytes.toBytes("ch3"))))); Configuration configV1 = HBaseConfiguration.create(); configV1.setStrings( TxConstants.Persist.CFG_TX_SNAPHOT_CODEC_CLASSES, SnapshotCodecV1.class.getName()); SnapshotCodecProvider codecV1 = new SnapshotCodecProvider(configV1); // encoding with codec of v1 ByteArrayOutputStream out = new ByteArrayOutputStream(); try { codecV1.encode(out, snapshot); } finally { out.close(); } // decoding Configuration configV1V2 = HBaseConfiguration.create(); configV1V2.setStrings( TxConstants.Persist.CFG_TX_SNAPHOT_CODEC_CLASSES, SnapshotCodecV1.class.getName(), SnapshotCodecV2.class.getName()); SnapshotCodecProvider codecV1V2 = new SnapshotCodecProvider(configV1V2); TransactionSnapshot decoded = codecV1V2.decode(new ByteArrayInputStream(out.toByteArray())); assertEquals(snapshot, decoded); }
/** * @param args : * <ol> * <li>table name * </ol> */ public static void main(String[] args) { try { ToolRunner.run(HBaseConfiguration.create(), new CountRowsMR(), args); } catch (Exception e) { e.printStackTrace(); } }
public static void main(String args[]) { if (args.length == 0) { System.out.println("JavaHBaseBulkPutExample {master} {tableName} {columnFamily}"); } String master = args[0]; String tableName = args[1]; String columnFamily = args[2]; JavaSparkContext jsc = new JavaSparkContext(master, "JavaHBaseBulkPutExample"); List<String> list = new ArrayList<String>(); list.add("1," + columnFamily + ",a,1"); list.add("2," + columnFamily + ",a,2"); list.add("3," + columnFamily + ",a,3"); list.add("4," + columnFamily + ",a,4"); list.add("5," + columnFamily + ",a,5"); JavaRDD<String> rdd = jsc.parallelize(list); Configuration conf = HBaseConfiguration.create(); conf.addResource(new Path("/opt/hadoop-2.6.0/etc/hadoop/core-site.xml")); conf.addResource(new Path("/opt/hbase/conf/hbase-site.xml")); JavaHBaseContext hbaseContext = new JavaHBaseContext(jsc, conf); hbaseContext.bulkPut(rdd, tableName, new PutFunction(), true); }
/** * Check duplicated tweet IDs in <b>tweetIdDir</b>, and output the duplicates to stdout. * * @param tweetIdDir * @throws Exception */ public static void checkTidDuplicates(String tweetIdDir) throws Exception { // First change path strings to URI strings starting with 'file:' or 'hdfs:' tweetIdDir = MultiFileFolderWriter.getUriStrForPath(tweetIdDir); Set<String> tidSet = new HashSet<String>(); Configuration conf = HBaseConfiguration.create(); FileSystem fs = FileSystem.get(new URI(tweetIdDir), conf); int dupCount = 0; for (FileStatus srcFileStatus : fs.listStatus(new Path(tweetIdDir))) { String srcFileName = srcFileStatus.getPath().getName(); if (srcFileName.endsWith(".txt") && srcFileName.contains("tweetIds")) { BufferedReader brTid = new BufferedReader(new InputStreamReader(fs.open(srcFileStatus.getPath()))); String tid = brTid.readLine(); while (tid != null) { if (tidSet.contains(tid)) { System.out.println("Duplicated tweet ID: " + tid); dupCount++; } else { tidSet.add(tid); } tid = brTid.readLine(); } brTid.close(); } } System.out.println( "Number of unique tweet IDs: " + tidSet.size() + ", number of duplicates: " + dupCount); }
public static void deleteTest(String tableStr) { try { Configuration conf = HBaseConfiguration.create(); byte[] tableName = Bytes.toBytes(tableStr); HConnection hConnection = HConnectionManager.createConnection(conf); HTableInterface table = hConnection.getTable(tableName); byte[] startRow = Bytes.toBytes("rowKey_1"); byte[] stopRow = Bytes.toBytes("rowKey_3"); byte[] family = f0; Scan scan = new Scan(); scan.addFamily(family); scan.setMaxVersions(1); // scan.setStartRow(startRow); // scan.setStopRow(stopRow); ResultScanner scanner = table.getScanner(scan); Result result = scanner.next(); List<Delete> delete = new ArrayList<Delete>(); while (result != null) { Delete del = new Delete(result.getRow()); delete.add(del); result = scanner.next(); } table.delete(delete); System.out.println("delete done"); table.close(); // very important } catch (IOException e) { e.printStackTrace(); } }
public static void main(String[] args) throws IOException, NoSuchAlgorithmException { Configuration config = HBaseConfiguration.create(); config.set("hbase.zookeeper.quorum", "distillery"); HTable meadcam = new HTable(config, "meadcam"); if (args[0].equals("delta")) { MeadRows meadrows = new MeadRows(meadcam); System.out.println("Average Delta: " + averageDeltas(meadrows)); } if (args[0].equals("get")) { MeadRow row = new MeadRow(meadcam, args[1]); System.out.println("==== " + row.getBrewId() + " ===="); System.out.println("Date:\t " + row.getDate()); System.out.println("Delta:\t " + row.getDelta()); System.out.println("RMS:\t " + row.getRMS()); } if (args[0].equals("latest")) { System.out.println("Getting latest..."); String brewId = args[1]; // long minutes = Long.parseLong(args[2]); MeadRows meadrows = new MeadRows(meadcam, brewId); System.out.println("Average: " + Float.toString(averageDeltas(meadrows))); } meadcam.close(); }
public static void main(String... args) throws IOException { Configuration config = HBaseConfiguration.create(); // Add any necessary configuration files (hbase-site.xml, core-site.xml) // config.addResource(new Path(System.getenv("HBASE_CONF_DIR"), "hbase-site.xml")); // config.addResource(new Path(System.getenv("HADOOP_CONF_DIR"), "core-site.xml")); System.out.println("Argumentos: " + args.length); if (args.length == 2) { CF_DEFAULT = args[0]; TABLE_NAME = args[1]; TABLE_NAME1 = TableName.valueOf(TABLE_NAME); CF = Bytes.toBytes(CF_DEFAULT); } else { System.out.println("Error de argumentos validos"); System.out.println( "Ejemplo de ejecucion: hadoop jar /home/cloudera/Desktop/loadtohbase-0.0.1-SNAPSHOT.jar loadtohbase.ToHBase csvFile bdu"); System.exit(0); } // putTable(config); try { scanTable(config); } catch (URISyntaxException e) { // TODO Auto-generated catch block e.printStackTrace(); System.out.println(e.getMessage()); } }
public static void main(String[] args) throws Exception { final Configuration configuration = HBaseConfiguration.create(); configuration.addResource("grade.xml"); String tables = configuration.get("hbase.cdn.tables"); if (Strings.isNullOrEmpty(tables)) { return; } List<String> list = Lists.newArrayList(Splitter.on(",").split(tables)); List<String> results = Lists.transform( list, new Function<String, String>() { @Override public String apply(@Nullable java.lang.String input) { return String.format( configuration.get("hdfs.directory.base.db"), new Date(), input); } }); String[] arrays = new String[] { Joiner.on(",").join(results), String.format(configuration.get("hdfs.directory.num.middle"), new Date()), String.format(configuration.get("hdfs.directory.num.result"), new Date()) }; AbstractJob job = new TopNJob(); // job.setStart(true); int i = ToolRunner.run(configuration, job, arrays); System.exit(i); }
/** * Test that operation timeout prevails over rpc default timeout and retries, etc. * * @throws IOException */ @Test public void testRocTimeout() throws IOException { Configuration localConfig = HBaseConfiguration.create(this.conf); // This override mocks up our exists/get call to throw a RegionServerStoppedException. localConfig.set("hbase.client.connection.impl", RpcTimeoutConnection.class.getName()); int pause = 10; localConfig.setInt("hbase.client.pause", pause); localConfig.setInt(HConstants.HBASE_CLIENT_RETRIES_NUMBER, 10); // Set the operation timeout to be < the pause. Expectation is that after first pause, we will // fail out of the rpc because the rpc timeout will have been set to the operation tiemout // and it has expired. Otherwise, if this functionality is broke, all retries will be run -- // all ten of them -- and we'll get the RetriesExhaustedException exception. localConfig.setInt(HConstants.HBASE_CLIENT_META_OPERATION_TIMEOUT, pause - 1); HTable table = new HTable(localConfig, TableName.META_TABLE_NAME); Throwable t = null; try { // An exists call turns into a get w/ a flag. table.exists(new Get(Bytes.toBytes("abc"))); } catch (SocketTimeoutException e) { // I expect this exception. LOG.info("Got expected exception", e); t = e; } catch (RetriesExhaustedException e) { // This is the old, unwanted behavior. If we get here FAIL!!! fail(); } finally { table.close(); } assertTrue(t != null); }
private static void addDirectoryToClassPath(File directory) { try { // Get the classloader actually used by HBaseConfiguration ClassLoader classLoader = HBaseConfiguration.create().getClassLoader(); if (!(classLoader instanceof URLClassLoader)) { fail("We should get a URLClassLoader"); } // Make the addURL method accessible Method method = URLClassLoader.class.getDeclaredMethod("addURL", URL.class); method.setAccessible(true); // Add the directory where we put the hbase-site.xml to the classpath method.invoke(classLoader, directory.toURI().toURL()); } catch (MalformedURLException | NoSuchMethodException | IllegalAccessException | InvocationTargetException e) { fail( "Unable to add " + directory + " to classpath because of this exception: " + e.getMessage()); } }
@Override public void init(Context context) throws IOException { super.init(context); this.conf = HBaseConfiguration.create(context.getConfiguration()); this.tableDescriptors = context.getTableDescriptors(); // HRS multiplies client retries by 10 globally for meta operations, but we do not want this. // We are resetting it here because we want default number of retries (35) rather than 10 times // that which makes very long retries for disabled tables etc. int defaultNumRetries = conf.getInt( HConstants.HBASE_CLIENT_RETRIES_NUMBER, HConstants.DEFAULT_HBASE_CLIENT_RETRIES_NUMBER); if (defaultNumRetries > 10) { int mult = conf.getInt("hbase.client.serverside.retries.multiplier", 10); defaultNumRetries = defaultNumRetries / mult; // reset if HRS has multiplied this already } conf.setInt("hbase.client.serverside.retries.multiplier", 1); int numRetries = conf.getInt(CLIENT_RETRIES_NUMBER, defaultNumRetries); conf.setInt(HConstants.HBASE_CLIENT_RETRIES_NUMBER, numRetries); this.numWriterThreads = this.conf.getInt("hbase.region.replica.replication.writer.threads", 3); controller = new PipelineController(); entryBuffers = new EntryBuffers( controller, this.conf.getInt("hbase.region.replica.replication.buffersize", 128 * 1024 * 1024)); // use the regular RPC timeout for replica replication RPC's this.operationTimeout = conf.getInt( HConstants.HBASE_CLIENT_OPERATION_TIMEOUT, HConstants.DEFAULT_HBASE_CLIENT_OPERATION_TIMEOUT); }
/** * Constructor to set maximum versions and use the specified configuration, table factory and pool * type. The HTablePool supports the {@link PoolType#Reusable} and {@link PoolType#ThreadLocal}. * If the pool type is null or not one of those two values, then it will default to {@link * PoolType#Reusable}. * * @param config configuration * @param maxSize maximum number of references to keep for each table * @param tableFactory table factory * @param poolType pool type which is one of {@link PoolType#Reusable} or {@link * PoolType#ThreadLocal} */ public HTablePool( final Configuration config, final int maxSize, final HTableInterfaceFactory tableFactory, PoolType poolType) { // Make a new configuration instance so I can safely cleanup when // done with the pool. this.config = config == null ? HBaseConfiguration.create() : config; this.maxSize = maxSize; this.tableFactory = tableFactory == null ? new HTableFactory() : tableFactory; if (poolType == null) { this.poolType = PoolType.Reusable; } else { switch (poolType) { case Reusable: case ThreadLocal: this.poolType = poolType; break; default: this.poolType = PoolType.Reusable; break; } } this.tables = new PoolMap<String, HTableInterface>(this.poolType, this.maxSize); }
public static Path[] getInputPaths(String rootPath) { try { Configuration conf = HBaseConfiguration.create(); Path root = new Path(rootPath); ArrayList<Path> paths = new ArrayList<Path>(); FileSystem fs = root.getFileSystem(conf); LinkedList<Path> list = new LinkedList<Path>(); list.push(root); if (!fs.exists(root)) { System.out.println("path not exists: " + root.toString()); return new Path[0]; } while (!list.isEmpty()) { Path path = list.pop(); if (fs.isFile(path)) { if (path.getName().matches("^.*part-r-\\d{5}.*$")) { paths.add(path); System.out.println("something is wrong with path" + path.toString()); } } else { FileStatus[] statuses = fs.listStatus(path); for (FileStatus status : statuses) { if (status.isDir()) { list.add(status.getPath()); } else if (status.getPath().getName().matches("^.*part-r-\\d{5}.*$")) { paths.add(status.getPath()); } } } } return paths.toArray(new Path[paths.size()]); } catch (IOException ignored) { return new Path[0]; } }
@Test public void testRegionObserverFlushTimeStacking() throws Exception { byte[] ROW = Bytes.toBytes("testRow"); byte[] TABLE = Bytes.toBytes(getClass().getName()); byte[] A = Bytes.toBytes("A"); byte[][] FAMILIES = new byte[][] {A}; Configuration conf = HBaseConfiguration.create(); HRegion region = initHRegion(TABLE, getClass().getName(), conf, FAMILIES); RegionCoprocessorHost h = region.getCoprocessorHost(); h.load(NoDataFromFlush.class, Coprocessor.PRIORITY_HIGHEST, conf); h.load(EmptyRegionObsever.class, Coprocessor.PRIORITY_USER, conf); // put a row and flush it to disk Put put = new Put(ROW); put.add(A, A, A); region.put(put); region.flushcache(); Get get = new Get(ROW); Result r = region.get(get); assertNull( "Got an unexpected number of rows - no data should be returned with the NoDataFromScan coprocessor. Found: " + r, r.listCells()); }
/** * @param args * @author Nagamallikarjuna * @throws IOException */ public static void main(String[] args) throws IOException { Configuration conf = HBaseConfiguration.create(); HTable table = new HTable(conf, "stocks"); File file = new File("/home/naga/bigdata/hadoop-1.0.3/daily"); BufferedReader br = new BufferedReader(new FileReader(file)); String line = br.readLine(); Put data = null; while (line != null) { String parts[] = line.trim().split("\\t"); if (parts.length == 9) { String key = parts[1] + ":" + parts[2]; data = new Put(key.getBytes()); data.add("cf".getBytes(), "exchange".getBytes(), parts[0].getBytes()); data.add("cf".getBytes(), "open".getBytes(), parts[3].getBytes()); data.add("cf".getBytes(), "high".getBytes(), parts[4].getBytes()); data.add("cf".getBytes(), "low".getBytes(), parts[5].getBytes()); data.add("cf".getBytes(), "close".getBytes(), parts[6].getBytes()); data.add("cf".getBytes(), "volume".getBytes(), parts[7].getBytes()); data.add("cf".getBytes(), "adj_close".getBytes(), parts[8].getBytes()); table.put(data); } line = br.readLine(); } br.close(); table.close(); }
public static void main(String[] args) throws Exception { if (args.length < 2) { throw new Exception("Table name not specified."); } Configuration conf = HBaseConfiguration.create(); HTable table = new HTable(conf, args[0]); String startKey = args[1]; TimeCounter executeTimer = new TimeCounter(); executeTimer.begin(); executeTimer.enter(); Expression exp = ExpressionFactory.eq( ExpressionFactory.toLong( ExpressionFactory.toString(ExpressionFactory.columnValue("family", "longStr2"))), ExpressionFactory.constant(Long.parseLong("99"))); ExpressionFilter expressionFilter = new ExpressionFilter(exp); Scan scan = new Scan(Bytes.toBytes(startKey), expressionFilter); int count = 0; ResultScanner scanner = table.getScanner(scan); Result r = scanner.next(); while (r != null) { count++; r = scanner.next(); } System.out.println("++ Scanning finished with count : " + count + " ++"); scanner.close(); executeTimer.leave(); executeTimer.end(); System.out.println("++ Time cost for scanning: " + executeTimer.getTimeString() + " ++"); }
/** * 类HBaseFactory.java的实现描述:TODO 类实现描述 * * @author zhaoheng Jul 28, 2015 4:53:52 PM */ public class HBaseFactory { protected static final Configuration CONF = HBaseConfiguration.create(); protected static final Map<String, CMap> CMPS = new HashMap<String, CMap>(); protected static final Map<String, HTable> TABLES = new HashMap<String, HTable>(); public static synchronized HTable getTable(Table tb) throws IOException { if (TABLES.get(tb.tbName) != null) { return TABLES.get(tb.tbName); } else { HTable table = new HTable(CONF, tb.tbName); table.setAutoFlush(true, false); TABLES.put(tb.tbName, table); return table; } } public static synchronized CMap getHBaseCMap(Table tb) throws IOException { if (CMPS.get(tb.tbName) != null) { return CMPS.get(tb.tbName); } else { CMap table = new HBaseCMap(tb); CMPS.put(tb.tbName, table); return table; } } }
/** * Remove the @Ignore to try out timeout and retry asettings * * @throws IOException */ @Ignore @Test public void testTimeoutAndRetries() throws IOException { Configuration localConfig = HBaseConfiguration.create(this.conf); // This override mocks up our exists/get call to throw a RegionServerStoppedException. localConfig.set("hbase.client.connection.impl", RpcTimeoutConnection.class.getName()); HTable table = new HTable(localConfig, TableName.META_TABLE_NAME); Throwable t = null; LOG.info("Start"); try { // An exists call turns into a get w/ a flag. table.exists(new Get(Bytes.toBytes("abc"))); } catch (SocketTimeoutException e) { // I expect this exception. LOG.info("Got expected exception", e); t = e; } catch (RetriesExhaustedException e) { // This is the old, unwanted behavior. If we get here FAIL!!! fail(); } finally { table.close(); } LOG.info("Stop"); assertTrue(t != null); }
/** * @param conn The HBase connection. * @param conf The HBase configuration * @param perRegionServerBufferQueueSize determines the max number of the buffered Put ops for * each region server before dropping the request. */ public HTableMultiplexer( Connection conn, Configuration conf, int perRegionServerBufferQueueSize) { this.conn = (ClusterConnection) conn; this.pool = HTable.getDefaultExecutor(conf); // how many times we could try in total, one more than retry number this.maxAttempts = conf.getInt( HConstants.HBASE_CLIENT_RETRIES_NUMBER, HConstants.DEFAULT_HBASE_CLIENT_RETRIES_NUMBER) + 1; this.perRegionServerBufferQueueSize = perRegionServerBufferQueueSize; this.maxKeyValueSize = HTable.getMaxKeyValueSize(conf); this.flushPeriod = conf.getLong(TABLE_MULTIPLEXER_FLUSH_PERIOD_MS, 100); int initThreads = conf.getInt(TABLE_MULTIPLEXER_INIT_THREADS, 10); this.executor = Executors.newScheduledThreadPool( initThreads, new ThreadFactoryBuilder() .setDaemon(true) .setNameFormat("HTableFlushWorker-%d") .build()); this.workerConf = HBaseConfiguration.create(conf); // We do not do the retry because we need to reassign puts to different queues if regions are // moved. this.workerConf.setInt(HConstants.HBASE_CLIENT_RETRIES_NUMBER, 0); }
public static void addColumnFamily(String keyspace, String columnFamily) throws Exception { Configuration conf = HBaseConfiguration.create(); HBaseAdmin admin = new HBaseAdmin(conf); HColumnDescriptor column = new HColumnDescriptor(columnFamily); // admin.deleteColumn(keyspace, columnFamily); admin.addColumn(keyspace, column); }
@Override public void configure(HTraceConfiguration conf) { this.conf = conf; this.hconf = HBaseConfiguration.create(); this.table = Bytes.toBytes(conf.get(TABLE_KEY, DEFAULT_TABLE)); this.cf = Bytes.toBytes(conf.get(COLUMNFAMILY_KEY, DEFAULT_COLUMNFAMILY)); this.maxSpanBatchSize = conf.getInt(MAX_SPAN_BATCH_SIZE_KEY, DEFAULT_MAX_SPAN_BATCH_SIZE); String quorum = conf.get(COLLECTOR_QUORUM_KEY, DEFAULT_COLLECTOR_QUORUM); hconf.set(HConstants.ZOOKEEPER_QUORUM, quorum); String znodeParent = conf.get(ZOOKEEPER_ZNODE_PARENT_KEY, DEFAULT_ZOOKEEPER_ZNODE_PARENT); hconf.set(HConstants.ZOOKEEPER_ZNODE_PARENT, znodeParent); int clientPort = conf.getInt(ZOOKEEPER_CLIENT_PORT_KEY, DEFAULT_ZOOKEEPER_CLIENT_PORT); hconf.setInt(HConstants.ZOOKEEPER_CLIENT_PORT, clientPort); // If there are already threads runnnig tear them down. if (this.service != null) { this.service.shutdownNow(); this.service = null; } int numThreads = conf.getInt(NUM_THREADS_KEY, DEFAULT_NUM_THREADS); this.service = Executors.newFixedThreadPool(numThreads, tf); for (int i = 0; i < numThreads; i++) { this.service.submit(new WriteSpanRunnable()); } }
@Override public Event intercept(Event event) { // TODO Auto-generated method stub Map<String, String> headers = event.getHeaders(); String Filename = headers.get("file"); String fileType = getFileType(new String(event.getBody())); Configuration conf = HBaseConfiguration.create(); HTable table = null; try { table = new HTable(conf, "fs"); } catch (IOException e) { // TODO Auto-generated catch block e.printStackTrace(); } Put put = new Put(Bytes.toBytes(fileType + "_" + Filename)); put.add(Bytes.toBytes("fn"), Bytes.toBytes("ST"), Bytes.toBytes("PICKED")); try { table.put(put); } catch (RetriesExhaustedWithDetailsException e) { // TODO Auto-generated catch block e.printStackTrace(); } catch (InterruptedIOException e) { // TODO Auto-generated catch block e.printStackTrace(); } try { table.close(); } catch (IOException e) { // TODO Auto-generated catch block e.printStackTrace(); } return event; }
public static void main(String args[]) { if (args.length == 0) { System.out.println("JavaHBaseDistributedScan {master} {tableName}"); } String master = args[0]; String tableName = args[1]; JavaSparkContext jsc = new JavaSparkContext(master, "JavaHBaseDistributedScan"); jsc.addJar("SparkHBase.jar"); Configuration conf = HBaseConfiguration.create(); conf.addResource(new Path("/etc/hbase/conf/core-site.xml")); conf.addResource(new Path("/etc/hbase/conf/hbase-site.xml")); JavaHBaseContext hbaseContext = new JavaHBaseContext(jsc, conf); Scan scan = new Scan(); scan.setCaching(100); JavaRDD<Tuple2<byte[], List<Tuple3<byte[], byte[], byte[]>>>> javaRdd = hbaseContext.hbaseRDD(tableName, scan); List<Tuple2<byte[], List<Tuple3<byte[], byte[], byte[]>>>> results = javaRdd.collect(); results.size(); }
public synchronized Configuration configuration() { if (conf == null) { conf = HBaseConfiguration.create(); } return conf; }
@Before public void setUp() { conf = HBaseConfiguration.create(); rpcServices = Mockito.mock(RSRpcServices.class); when(rpcServices.getConfiguration()).thenReturn(conf); qosFunction = new AnnotationReadingPriorityFunction(rpcServices, RSRpcServices.class); }
public Blog(String blogid) throws IOException { Configuration conf = HBaseConfiguration.create(); table = new HTable(conf, "blogs"); // 1. Get the row whose row key is blogid from above Get g = new Get(Bytes.toBytes(blogid)); Result r = table.get(g); // 2. Extract the rowkey, blog text (column "body") and blog title // (column "meta:title") key = r.getRow(); keyStr = Bytes.toString(key); blogText = Bytes.toString(r.getValue(Bytes.toBytes("body"), Bytes.toBytes(""))); blogTitle = Bytes.toString(r.getValue(Bytes.toBytes("meta"), Bytes.toBytes("title"))); Long reverseTimestamp = Long.parseLong(keyStr.substring(4)); Long epoch = Math.abs(reverseTimestamp - Long.MAX_VALUE); dateOfPost = new Date(epoch); // Get an iterator for the comments Scan s = new Scan(); s.addFamily(Bytes.toBytes("comment")); // Use a PrefixFilter PrefixFilter filter = new PrefixFilter(key); s.setFilter(filter); scanner = table.getScanner(s); resultIterator = scanner.iterator(); }
public static void main(String[] args) throws MasterNotRunningException, ZooKeeperConnectionException, IOException { Configuration conf = HBaseConfiguration.create(); conf.set( "hbase.zookeeper.quorum", "192.168.10.163:2181,192.168.10.164:2181,192.168.10.165:2181"); // admin用户 HBaseAdmin admin = new HBaseAdmin(conf); // table HTableDescriptor htd = new HTableDescriptor(TableName.valueOf("people")); // info列簇 HColumnDescriptor hcd_info = new HColumnDescriptor("info"); hcd_info.setMaxVersions(3); // data列簇 HColumnDescriptor hcd_data = new HColumnDescriptor("data"); // 将列簇添加到htable中 htd.addFamily(hcd_info); htd.addFamily(hcd_data); // 创建表 admin.createTable(htd); // 关闭连接 admin.close(); }