@Override public List<SpanBo> selectSpans(TransactionId transactionId) { if (transactionId == null) { throw new NullPointerException("transactionId must not be null"); } byte[] transactionIdRowKey = rowKeyDecoder.encodeRowKey(transactionId); Get get = new Get(transactionIdRowKey); get.addFamily(HBaseTables.TRACES_CF_SPAN); get.addFamily(HBaseTables.TRACES_CF_TERMINALSPAN); return template2.get(HBaseTables.TRACES, get, spanMapper); }
@SuppressWarnings("deprecation") @Override public DResult get(Get get, long startId) throws IOException { if (get.hasFamilies()) get.addFamily(DominoConst.INNER_FAMILY); get.setTimeRange(0, startId + 1); // [x, y) get.setMaxVersions(); Result preRead = region.get(get); List<KeyValue> status = preRead.getColumn(DominoConst.INNER_FAMILY, DominoConst.STATUS_COL); if (status == null || status.size() == 0) { Result ret = MVCC.handleResult(this, getTrxMetaTable(), preRead, startId, null); return new DResult(ret, null); } Integer lockId = region.getLock(null, get.getRow(), true); try { Result r = MVCC.handleResult(this, getTrxMetaTable(), region.get(get, lockId), startId, lockId); return new DResult(r, null); } catch (TransactionOutOfDateException oode) { return new DResult(null, oode.getMessage()); } catch (InvalidRowStatusException e) { return new DResult(null, e.getMessage()); } finally { region.releaseRowLock(lockId); } }
public List<Query> getQueries(final String creator) throws IOException { if (null == creator) { return null; } List<Query> queries = new ArrayList<Query>(); HTableInterface htable = null; try { htable = HBaseConnection.get(hbaseUrl).getTable(userTableName); Get get = new Get(Bytes.toBytes(creator)); get.addFamily(Bytes.toBytes(USER_QUERY_FAMILY)); Result result = htable.get(get); Query[] query = querySerializer.deserialize( result.getValue(Bytes.toBytes(USER_QUERY_FAMILY), Bytes.toBytes(USER_QUERY_COLUMN))); if (null != query) { queries.addAll(Arrays.asList(query)); } } finally { IOUtils.closeQuietly(htable); } return queries; }
public static int getDynamicTable(Configuration config) { /** Connection to the cluster. A single connection shared by all application threads. */ Connection connection = null; /** A lightweight handle to a specific table. Used from a single thread. */ Table table = null; try { connection = ConnectionFactory.createConnection(config); table = connection.getTable(TABLE_NAME1); Get get = new Get(Bytes.toBytes("cloudera")); get.addFamily(CF); get.setMaxVersions(Integer.MAX_VALUE); Result result = table.get(get); NavigableMap<byte[], NavigableMap<byte[], NavigableMap<Long, byte[]>>> map = result.getMap(); for (Entry<byte[], NavigableMap<byte[], NavigableMap<Long, byte[]>>> columnFamilyEntry : map.entrySet()) { NavigableMap<byte[], NavigableMap<Long, byte[]>> columnMap = columnFamilyEntry.getValue(); for (Entry<byte[], NavigableMap<Long, byte[]>> columnEntry : columnMap.entrySet()) { NavigableMap<Long, byte[]> cellMap = columnEntry.getValue(); for (Entry<Long, byte[]> cellEntry : cellMap.entrySet()) { System.out.println( String.format( "Key : %s, Value :%s", Bytes.toString(columnEntry.getKey()), Bytes.toString(cellEntry.getValue()))); } } } } catch (IOException e) { e.printStackTrace(); } return 0; }
@Before public void setUp() throws Exception { super.setUp(); row1 = Bytes.toBytes("row1"); row2 = Bytes.toBytes("row2"); row3 = Bytes.toBytes("row3"); fam1 = Bytes.toBytes("fam1"); fam2 = Bytes.toBytes("fam2"); col1 = Bytes.toBytes("col1"); col2 = Bytes.toBytes("col2"); col3 = Bytes.toBytes("col3"); col4 = Bytes.toBytes("col4"); col5 = Bytes.toBytes("col5"); data = Bytes.toBytes("data"); // Create Get get = new Get(row1); get.addFamily(fam1); get.addColumn(fam2, col2); get.addColumn(fam2, col4); get.addColumn(fam2, col5); this.scan = new Scan(get); rowComparator = CellComparator.COMPARATOR; }
/** * Gets the region info and assignment for the specified region. * * @param catalogTracker * @param regionName Region to lookup. * @return Location and HRegionInfo for <code>regionName</code> * @throws IOException */ public static Pair<HRegionInfo, ServerName> getRegion( CatalogTracker catalogTracker, byte[] regionName) throws IOException { Get get = new Get(regionName); get.addFamily(HConstants.CATALOG_FAMILY); Result r = get(getCatalogHTable(catalogTracker, regionName), get); return (r == null || r.isEmpty()) ? null : parseCatalogResult(r); }
public static void readTest(String tableStr, String row) { try { Configuration conf = HBaseConfiguration.create(); byte[] tableName = Bytes.toBytes(tableStr); HConnection hConnection = HConnectionManager.createConnection(conf); HTableInterface table = hConnection.getTable(tableName); byte[] rowkey = Bytes.toBytes(row); Get get = new Get(rowkey); get.addFamily(f0); Result result = table.get(get); NavigableMap<byte[], byte[]> m = result.getFamilyMap(f0); if (m == null || m.isEmpty()) { System.err.println("Empty." + m); return; } for (Map.Entry<byte[], byte[]> entry : m.entrySet()) { String qualifier = Bytes.toString(entry.getKey()); String value = Bytes.toString(entry.getValue()); System.out.println(qualifier + ":" + value); } table.close(); // very important } catch (IOException e) { e.printStackTrace(); } }
/** * Read a record from the database. Each field/value pair from the result will be stored in a * HashMap. * * @param table The name of the table * @param key The record key of the record to read. * @param fields The list of fields to read, or null for all of them * @param result A HashMap of field/value pairs for the result * @return Zero on success, a non-zero error code on error */ public Status read( String table, String key, Set<String> fields, HashMap<String, ByteIterator> result) { // if this is a "new" table, init HTable object. Else, use existing one if (!tableName.equals(table)) { currentTable = null; try { getHTable(table); tableName = table; } catch (IOException e) { System.err.println("Error accessing HBase table: " + e); return Status.ERROR; } } Result r = null; try { if (debug) { System.out.println("Doing read from HBase columnfamily " + columnFamily); System.out.println("Doing read for key: " + key); } Get g = new Get(Bytes.toBytes(key)); if (fields == null) { g.addFamily(columnFamilyBytes); } else { for (String field : fields) { g.addColumn(columnFamilyBytes, Bytes.toBytes(field)); } } r = currentTable.get(g); } catch (IOException e) { if (debug) { System.err.println("Error doing get: " + e); } return Status.ERROR; } catch (ConcurrentModificationException e) { // do nothing for now...need to understand HBase concurrency model better return Status.ERROR; } if (r.isEmpty()) { return Status.NOT_FOUND; } while (r.advance()) { final Cell c = r.current(); result.put( Bytes.toString(CellUtil.cloneQualifier(c)), new ByteArrayByteIterator(CellUtil.cloneValue(c))); if (debug) { System.out.println( "Result for field: " + Bytes.toString(CellUtil.cloneQualifier(c)) + " is: " + Bytes.toString(CellUtil.cloneValue(c))); } } return Status.OK; }
public static HBaseTuple getHBaseTuple(String row, String colFamily, String colName) throws IOException { Configuration conf = getConfiguration(); HTable table1 = new HTable(conf, "table1"); Get get = new Get(Bytes.toBytes(row)); get.addFamily(Bytes.toBytes(colFamily)); get.addColumn(Bytes.toBytes(colFamily), Bytes.toBytes(colName)); Result result = table1.get(get); return getHBaseTuple(result); }
public static void main(String[] args) throws IOException { Configuration conf = HBaseConfiguration.create(); HBaseHelper helper = HBaseHelper.getHelper(conf); helper.dropTable("testtable"); helper.createTable("testtable", "colfam1", "colfam2", "colfam3", "colfam4"); System.out.println("Adding rows to table..."); helper.fillTable("testtable", 1, 10, 2, "colfam1", "colfam2", "colfam3", "colfam4"); HTable table = new HTable(conf, "testtable"); // vv FamilyFilterExample Filter filter1 = new FamilyFilter( CompareFilter.CompareOp .LESS, // co FamilyFilterExample-1-Filter Create filter, while specifying the // comparison operator and comparator. new BinaryComparator(Bytes.toBytes("colfam3"))); Scan scan = new Scan(); scan.setFilter(filter1); ResultScanner scanner = table.getScanner( scan); // co FamilyFilterExample-2-Scan Scan over table while applying the filter. // ^^ FamilyFilterExample System.out.println("Scanning table... "); // vv FamilyFilterExample for (Result result : scanner) { System.out.println(result); } scanner.close(); Get get1 = new Get(Bytes.toBytes("row-5")); get1.setFilter(filter1); Result result1 = table.get(get1); // co FamilyFilterExample-3-Get Get a row while applying the same filter. System.out.println("Result of get(): " + result1); Filter filter2 = new FamilyFilter( CompareFilter.CompareOp.EQUAL, new BinaryComparator(Bytes.toBytes("colfam3"))); Get get2 = new Get( Bytes.toBytes( "row-5")); // co FamilyFilterExample-4-Mismatch Create a filter on one column family // while trying to retrieve another. get2.addFamily(Bytes.toBytes("colfam1")); get2.setFilter(filter2); Result result2 = table.get( get2); // co FamilyFilterExample-5-Get2 Get the same row while applying the new filter, // this will return "NONE". System.out.println("Result of get(): " + result2); // ^^ FamilyFilterExample }
private void addFieldsToGet(Get get, List<FieldType> fields) { if (fields != null && (!fields.isEmpty())) { for (FieldType field : fields) { get.addColumn(RecordCf.DATA.bytes, ((FieldTypeImpl) field).getQualifier()); } RecordDecoder.addSystemColumnsToGet(get); } else { // Retrieve everything get.addFamily(RecordCf.DATA.bytes); } }
public static void sxr(String file) throws Exception { Configuration conf = HBaseConfiguration.create(); HTablePool tablePool = new HTablePool(conf, 100); HTableInterface cardTable = tablePool.getTable("dis_sxbzxr_new_card"); HTableInterface nameTable = tablePool.getTable("dis_sxbzxr_new_name"); File f = new File(file); BufferedReader br = new BufferedReader(new FileReader(f)); String line; String[] subs; String name, card; long cnt = 0; long findCard = 0; long findName = 0; while ((line = br.readLine()) != null) { subs = line.split(","); name = subs[0]; card = subs[1]; ++cnt; Get get = new Get(Bytes.toBytes(card)); get.addFamily(Bytes.toBytes("f")); Result result = cardTable.get(get); if (!result.isEmpty()) { ++findCard; LOG.info("find card: " + card + ", line: " + cnt); } Get nGet = new Get(Bytes.toBytes(name)); nGet.addFamily(Bytes.toBytes("f")); Result nResult = nameTable.get(nGet); if (!nResult.isEmpty()) { ++findName; LOG.info("find name: " + name); } } br.close(); LOG.info("line: " + cnt + ", find card: " + findCard + ", find name: " + findName); cardTable.close(); }
/** * HBaseのsc_item_dataから、商品データを取得。 * * @param shopID 店舗ID * @param itemID 商品ID。 * @param families column families * @return 該当商品データ */ public ItemData get(int shopID, int itemID, String[] families) throws IOException { byte[] rowkey = encodeRowkey(shopID, itemID); if (rowkey == null || rowkey.length == 0) { return null; } Get g = new Get(rowkey); for (String family : families) { byte[] bFamily = Bytes.toBytes(family); g.addFamily(bFamily); } g.setMaxVersions(); Result r = table.get(g); return toDataStore(r); }
@Override public List<SqlMetaDataBo> getSqlMetaData(String agentId, long time, int hashCode) { if (agentId == null) { throw new NullPointerException("agentId must not be null"); } SqlMetaDataBo sqlMetaData = new SqlMetaDataBo(agentId, time, hashCode); byte[] sqlId = getDistributedKey(sqlMetaData.toRowKey()); Get get = new Get(sqlId); get.addFamily(HBaseTables.SQL_METADATA_CF_SQL); return hbaseOperations2.get(HBaseTables.SQL_METADATA, get, sqlMetaDataMapper); }
/** * Read a record from the database. Each field/value pair from the result will be stored in a * HashMap. * * @param table The name of the table * @param key The record key of the record to read. * @param fields The list of fields to read, or null for all of them * @param result A HashMap of field/value pairs for the result * @return Zero on success, a non-zero error code on error */ public int read( String table, String key, Set<String> fields, HashMap<String, ByteIterator> result) { // if this is a "new" table, init HTable object. Else, use existing one if (!_table.equals(table)) { _hTable = null; try { getHTable(table); _table = table; } catch (IOException e) { System.err.println("Error accessing HBase table: " + e); return ServerError; } } Result r = null; try { if (_debug) { System.out.println("Doing read from HBase columnfamily " + _columnFamily); System.out.println("Doing read for key: " + key); } Get g = new Get(Bytes.toBytes(key)); if (fields == null) { g.addFamily(_columnFamilyBytes); } else { for (String field : fields) { g.addColumn(_columnFamilyBytes, Bytes.toBytes(field)); } } r = _hTable.get(g); } catch (IOException e) { System.err.println("Error doing get: " + e); return ServerError; } catch (ConcurrentModificationException e) { // do nothing for now...need to understand HBase concurrency model better return ServerError; } for (KeyValue kv : r.raw()) { result.put(Bytes.toString(kv.getQualifier()), new ByteArrayByteIterator(kv.getValue())); if (_debug) { System.out.println( "Result for field: " + Bytes.toString(kv.getQualifier()) + " is: " + Bytes.toString(kv.getValue())); } } return Ok; }
@Override public Get getGet(byte[] row) { Get get = new Get(row); for (Entry<byte[], NavigableSet<byte[]>> familyMapEntry : familyMap.entrySet()) { byte[] columnFamily = familyMapEntry.getKey(); if (familyMapEntry.getValue() == null) { get.addFamily(columnFamily); } else { for (byte[] qualifier : familyMapEntry.getValue()) { get.addColumn(columnFamily, qualifier); } } } return get; }
/** * Instantiate with {@code FieldDefinitions}s and {@code DocumentExtractDefinition}s. * * @param fieldDefinitions define fields to be indexed * @param documentExtractDefinitions additional document extraction definitions */ public DefaultResultToSolrMapper( String indexerName, List<FieldDefinition> fieldDefinitions, List<DocumentExtractDefinition> documentExtractDefinitions) { extractors = Lists.newArrayList(); resultDocumentExtractors = Lists.newArrayList(); for (FieldDefinition fieldDefinition : fieldDefinitions) { ByteArrayExtractor byteArrayExtractor = ByteArrayExtractors.getExtractor( fieldDefinition.getValueExpression(), fieldDefinition.getValueSource()); ByteArrayValueMapper valueMapper = ByteArrayValueMappers.getMapper(fieldDefinition.getTypeName()); ConfigureUtil.configure(valueMapper, ConfigureUtil.mapToJson(fieldDefinition.getParams())); resultDocumentExtractors.add( new HBaseSolrDocumentExtractor( fieldDefinition.getName(), byteArrayExtractor, valueMapper)); extractors.add(byteArrayExtractor); } for (DocumentExtractDefinition extractDefinition : documentExtractDefinitions) { ByteArrayExtractor byteArrayExtractor = ByteArrayExtractors.getExtractor( extractDefinition.getValueExpression(), extractDefinition.getValueSource()); extractors.add(byteArrayExtractor); } Get get = newGet(); for (ByteArrayExtractor extractor : extractors) { byte[] columnFamily = extractor.getColumnFamily(); byte[] columnQualifier = extractor.getColumnQualifier(); if (columnFamily != null) { if (columnQualifier != null) { get.addColumn(columnFamily, columnQualifier); } else { get.addFamily(columnFamily); } } } familyMap = get.getFamilyMap(); mappingTimer = Metrics.newTimer( metricName(getClass(), "HBase Result to Solr mapping time", indexerName), TimeUnit.MILLISECONDS, TimeUnit.SECONDS); }
/** * Transactional version of {@link HTable#delete(Delete)} * * @param transactionState Identifier of the transaction * @see HTable#delete(Delete) * @throws IOException */ public void delete(TransactionState transactionState, Delete delete) throws IOException { final long startTimestamp = transactionState.getStartTimestamp(); boolean issueGet = false; final Put deleteP = new Put(delete.getRow(), startTimestamp); final Get deleteG = new Get(delete.getRow()); Map<byte[], List<KeyValue>> fmap = delete.getFamilyMap(); if (fmap.isEmpty()) { issueGet = true; } for (List<KeyValue> kvl : fmap.values()) { for (KeyValue kv : kvl) { switch (KeyValue.Type.codeToType(kv.getType())) { case DeleteColumn: deleteP.add(kv.getFamily(), kv.getQualifier(), startTimestamp, null); break; case DeleteFamily: deleteG.addFamily(kv.getFamily()); issueGet = true; break; case Delete: if (kv.getTimestamp() == HConstants.LATEST_TIMESTAMP) { deleteP.add(kv.getFamily(), kv.getQualifier(), startTimestamp, null); break; } else { throw new UnsupportedOperationException( "Cannot delete specific versions on Snapshot Isolation."); } } } } if (issueGet) { Result result = this.get(deleteG); for (Entry<byte[], NavigableMap<byte[], NavigableMap<Long, byte[]>>> entryF : result.getMap().entrySet()) { byte[] family = entryF.getKey(); for (Entry<byte[], NavigableMap<Long, byte[]>> entryQ : entryF.getValue().entrySet()) { byte[] qualifier = entryQ.getKey(); deleteP.add(family, qualifier, null); } } } transactionState.addRow( new RowKeyFamily(delete.getRow(), getTableName(), deleteP.getFamilyMap())); put(deleteP); }
/** * Transactional version of {@link HTable#get(Get)} * * @param transactionState Identifier of the transaction * @see HTable#get(Get) * @throws IOException */ public Result get(TransactionState transactionState, final Get get) throws IOException { final long readTimestamp = transactionState.getStartTimestamp(); final Get tsget = new Get(get.getRow()); TimeRange timeRange = get.getTimeRange(); long startTime = timeRange.getMin(); long endTime = Math.min(timeRange.getMax(), readTimestamp + 1); // int maxVersions = get.getMaxVersions(); tsget .setTimeRange(startTime, endTime) .setMaxVersions((int) (versionsAvg + CACHE_VERSIONS_OVERHEAD)); Map<byte[], NavigableSet<byte[]>> kvs = get.getFamilyMap(); for (Map.Entry<byte[], NavigableSet<byte[]>> entry : kvs.entrySet()) { byte[] family = entry.getKey(); NavigableSet<byte[]> qualifiers = entry.getValue(); if (qualifiers == null || qualifiers.isEmpty()) { tsget.addFamily(family); } else { for (byte[] qualifier : qualifiers) { tsget.addColumn(family, qualifier); } } } // Result result; // Result filteredResult; // do { // result = super.get(tsget); // filteredResult = filter(super.get(tsget), readTimestamp, maxVersions); // } while (!result.isEmpty() && filteredResult == null); getsPerformed++; Result result = filter( transactionState, super.get(tsget), readTimestamp, (int) (versionsAvg + CACHE_VERSIONS_OVERHEAD)); return result == null ? new Result() : result; // Scan scan = new Scan(get); // scan.setRetainDeletesInOutput(true); // ResultScanner rs = this.getScanner(transactionState, scan); // Result r = rs.next(); // if (r == null) { // r = new Result(); // } // return r; }
@SuppressWarnings("deprecation") @Override public void rollbackRow(byte[] row, long startId, Integer lockId) throws IOException { byte[] family = DominoConst.INNER_FAMILY; Get get = new Get(row); get.setTimeStamp(startId); get.addFamily(family); Result r = region.get(get, lockId); if (r == null || r.isEmpty()) return; byte[] colBytes = r.getValue(family, DominoConst.COLUMNS_COL); if (colBytes == null || colBytes.length == 0) return; Delete del = new Delete(row); Columns cols = new Columns(colBytes); for (Column col : cols.cols) { del.deleteColumn(col.family, col.qualifier, startId); } del.deleteColumn(family, DominoConst.COLUMNS_COL, startId); del.deleteColumn(family, DominoConst.STATUS_COL, startId); mutateRow(del, lockId); }
private List<List<SpanBo>> getSpans0( List<TransactionId> transactionIdList, List<byte[]> hBaseFamiliyList) { if (transactionIdList == null || transactionIdList.isEmpty()) { return Collections.emptyList(); } if (hBaseFamiliyList == null) { throw new NullPointerException("hBaseFamiliyList may not be null."); } final List<Get> getList = new ArrayList<>(transactionIdList.size()); for (TransactionId transactionId : transactionIdList) { byte[] transactionIdRowKey = rowKeyDecoder.encodeRowKey(transactionId); final Get get = new Get(transactionIdRowKey); for (byte[] hbaseFamily : hBaseFamiliyList) { get.addFamily(hbaseFamily); } getList.add(get); } return template2.get(HBaseTables.TRACES, getList, spanMapper); }
@SuppressWarnings("deprecation") @Override public void commitRow(byte[] row, long startId, long commitId, boolean isDelete, Integer lockId) throws IOException { Get get = new Get(row); get.setMaxVersions(); get.addFamily(DominoConst.INNER_FAMILY); Result r = region.get(get, lockId); if (!containsStatus(r, startId)) { // Other transaction may have committed this row of this version LOG.info( "Commit: No status found, returning: {}.{}", new String(this.getName()), new String(row)); return; } List<KeyValue> versions = r.getColumn(DominoConst.INNER_FAMILY, DominoConst.VERSION_COL); Put commit = new Put(row); commit.setWriteToWAL(true); boolean isFresh = true; if (versions.size() >= DominoConst.MAX_VERSION) { // We need to clean the earliest version. LOG.info( "Commit: rolling version window: {}.{}", new String(this.getName()), new String(row)); isFresh = addClearColumns(commit, versions, r, row, isDelete, commitId, startId, lockId); } KeyValue clearStatusKV = new KeyValue( row, DominoConst.INNER_FAMILY, DominoConst.STATUS_COL, startId, KeyValue.Type.Delete); commit.add(clearStatusKV); byte[] value = DominoConst.versionValue(startId, isDelete); if (isFresh) { KeyValue commitKV = new KeyValue(row, DominoConst.INNER_FAMILY, DominoConst.VERSION_COL, commitId, value); commit.add(commitKV); } // commitNumericModifications(row, startId, lockId, commit); mutateRow(commit, lockId); }
@Override public Map<String, Map<String, String>> batch(List<GetItem> getItems) { Map<String, Map<String, String>> result = new ConcurrentHashMap<>(); List<Get> gets = new ArrayList<>(getItems.size()); for (GetItem getItem : getItems) { Get g = new Get(getItem.getRowkey()); g.addFamily(getItem.getFamily()); gets.add(g); } GetGroupTask task = new GetGroupTask(gets, result); Future<Boolean> future = AsyncThreadPoolFactory.ASYNC_HBASE_THREAD_POOL.submit(task); Boolean r = Boolean.FALSE; try { r = future.get(500, TimeUnit.MILLISECONDS); } catch (TimeoutException ignored) { } catch (InterruptedException | ExecutionException e) { e.printStackTrace(); } if (!r) { future.cancel(Boolean.FALSE); } return result; }
@Override public GetBuilder addFamily(byte[] family) { get.addFamily(family); return this; }