/* (non-Javadoc)
* @see com.hazelcast.core.MapLoader#loadAllKeys()
*/
@Override
public Set<String> loadAllKeys() {
Set<String> keySet = null;
if (allowLoadAll) {
keySet = new HashSet<String>();
HTableInterface hti = null;
try {
hti = pool.getTable(tableName);
Scan s = new Scan();
s.addColumn(family, qualifier);
ResultScanner rs = hti.getScanner(s);
Result r = null;
while ((r = rs.next()) != null) {
String k = new String(r.getRow());
keySet.add(k);
}
} catch (IOException e) {
LOG.error("IOException while loading all keys", e);
} finally {
if (hti != null) {
pool.putTable(hti);
}
}
}
return keySet;
}
public static void deleteTest(String tableStr) {
try {
Configuration conf = HBaseConfiguration.create();
byte[] tableName = Bytes.toBytes(tableStr);
HConnection hConnection = HConnectionManager.createConnection(conf);
HTableInterface table = hConnection.getTable(tableName);
byte[] startRow = Bytes.toBytes("rowKey_1");
byte[] stopRow = Bytes.toBytes("rowKey_3");
byte[] family = f0;
Scan scan = new Scan();
scan.addFamily(family);
scan.setMaxVersions(1);
// scan.setStartRow(startRow);
// scan.setStopRow(stopRow);
ResultScanner scanner = table.getScanner(scan);
Result result = scanner.next();
List<Delete> delete = new ArrayList<Delete>();
while (result != null) {
Delete del = new Delete(result.getRow());
delete.add(del);
result = scanner.next();
}
table.delete(delete);
System.out.println("delete done");
table.close(); // very important
} catch (IOException e) {
e.printStackTrace();
}
}
@Override
protected ArrayList<String> listResourcesImpl(String resPath) throws IOException {
assert resPath.startsWith("/");
String lookForPrefix = resPath.endsWith("/") ? resPath : resPath + "/";
byte[] startRow = Bytes.toBytes(lookForPrefix);
byte[] endRow = Bytes.toBytes(lookForPrefix);
endRow[endRow.length - 1]++;
ArrayList<String> result = new ArrayList<String>();
HTableInterface table = getConnection().getTable(getAllInOneTableName());
Scan scan = new Scan(startRow, endRow);
scan.setFilter(new KeyOnlyFilter());
try {
ResultScanner scanner = table.getScanner(scan);
for (Result r : scanner) {
String path = Bytes.toString(r.getRow());
assert path.startsWith(lookForPrefix);
int cut = path.indexOf('/', lookForPrefix.length());
String child = cut < 0 ? path : path.substring(0, cut);
if (result.contains(child) == false) result.add(child);
}
} finally {
IOUtils.closeQuietly(table);
}
// return null to indicate not a folder
return result.isEmpty() ? null : result;
}
@Override
protected List<RawResource> getAllResources(String rangeStart, String rangeEnd)
throws IOException {
byte[] startRow = Bytes.toBytes(rangeStart);
byte[] endRow = plusZero(Bytes.toBytes(rangeEnd));
Scan scan = new Scan(startRow, endRow);
scan.addColumn(B_FAMILY, B_COLUMN_TS);
scan.addColumn(B_FAMILY, B_COLUMN);
HTableInterface table = getConnection().getTable(getAllInOneTableName());
List<RawResource> result = Lists.newArrayList();
try {
ResultScanner scanner = table.getScanner(scan);
for (Result r : scanner) {
result.add(new RawResource(getInputStream(Bytes.toString(r.getRow()), r), getTimestamp(r)));
}
} catch (IOException e) {
for (RawResource rawResource : result) {
IOUtils.closeQuietly(rawResource.resource);
}
throw e;
} finally {
IOUtils.closeQuietly(table);
}
return result;
}
public <V> List<V> getObjects(String tableName, V object, HbaseMapper<V> mapper)
throws Exception {
HTableInterface htable = dataSource.getConnection(tableName);
Scan scan = mapper.mapScan(object);
log.info("scan is -[" + scan + "]");
ResultScanner resultScanner = htable.getScanner(scan);
List<V> result = trasfer(resultScanner, mapper);
htable.close();
log.info("get " + result.size() + " objects from " + tableName + "");
return result;
}
public static void main(String[] args) throws IOException {
Configuration conf = HBaseConfiguration.create();
HTableInterface literaturesTable = new HTable(conf, "pb_literatures");
Scan s = new Scan();
s.addFamily(Bytes.toBytes("info"));
ResultScanner rs = literaturesTable.getScanner(s);
for (Result r : rs) {
byte[] b = r.getValue(Bytes.toBytes("info"), Bytes.toBytes("title"));
System.out.println(Bytes.toString(b));
}
}
public List<RowLogMessage> next(String subscription, Long minimalTimestamp, boolean problematic)
throws RowLogException {
byte[] rowPrefix;
byte[] subscriptionBytes = Bytes.toBytes(subscription);
if (problematic) {
rowPrefix = PROBLEMATIC_MARKER;
rowPrefix = Bytes.add(rowPrefix, subscriptionBytes);
} else {
rowPrefix = subscriptionBytes;
}
byte[] startRow = rowPrefix;
if (minimalTimestamp != null) startRow = Bytes.add(startRow, Bytes.toBytes(minimalTimestamp));
try {
List<RowLogMessage> rowLogMessages = new ArrayList<RowLogMessage>();
Scan scan = new Scan(startRow);
if (minimalTimestamp != null) scan.setTimeRange(minimalTimestamp, Long.MAX_VALUE);
scan.addColumn(MESSAGES_CF, MESSAGE_COLUMN);
ResultScanner scanner = table.getScanner(scan);
boolean keepScanning = problematic;
do {
Result[] results = scanner.next(batchSize);
if (results.length == 0) {
keepScanning = false;
}
for (Result next : results) {
byte[] rowKey = next.getRow();
if (!Bytes.startsWith(rowKey, rowPrefix)) {
keepScanning = false;
break; // There were no messages for this subscription
}
if (problematic) {
rowKey = Bytes.tail(rowKey, rowKey.length - PROBLEMATIC_MARKER.length);
}
byte[] value = next.getValue(MESSAGES_CF, MESSAGE_COLUMN);
byte[] messageId = Bytes.tail(rowKey, rowKey.length - subscriptionBytes.length);
rowLogMessages.add(decodeMessage(messageId, value));
}
} while (keepScanning);
// The scanner is not closed in a finally block, since when we get an IOException from
// HBase, it is likely that closing the scanner will give problems too. Not closing
// the scanner is not fatal since HBase will expire it after a while.
Closer.close(scanner);
return rowLogMessages;
} catch (IOException e) {
throw new RowLogException("Failed to fetch next message from RowLogShard", e);
}
}
public <V> List<V> list(
String tableName, byte[] startKey, byte[] stopKey, Filter filter, HbaseMapper<V> mapper)
throws Exception {
HTableInterface htable = dataSource.getConnection(tableName);
Scan scan = new Scan();
scan.setStartRow(startKey);
scan.setStopRow(stopKey);
scan.setFilter(filter);
log.info("scan is -[" + scan + "]");
ResultScanner resultScanner = htable.getScanner(scan);
List<V> result = trasfer(resultScanner, mapper);
htable.close();
log.info("list " + result.size() + " objects from " + tableName);
return result;
}
/*
* 遍历查询hbase表
*
* @tableName 表名
*/
public static void getResultScann(String tableName) throws IOException {
Scan scan = new Scan();
ResultScanner rs = null;
HTableInterface table = conn.getTable(tableName);
try {
// String split = StringUtils.S001;
// QualifierFilter ff = new QualifierFilter(CompareOp.EQUAL, new BinaryComparator(
// Bytes.toBytes("A")));
// scan.setFilter(ff);
rs = table.getScanner(scan);
int count = 0;
for (Result r : rs) {
count++;
for (KeyValue kv : r.list()) {
System.out.println("row:" + Bytes.toString(kv.getRow()));
// System.out.println("family:" + Bytes.toString(kv.getFamily()));
// System.out.println("qualifier:" + Bytes.toString(kv.getQualifier()));
System.out.println("value:" + Bytes.toString(kv.getValue()));
//
// System.out.println("timestamp:" + kv.getTimestamp());
// StringBuilder sb = new StringBuilder();
// sb.append(Bytes.toString(r.getRow()));
// sb.append(split);
// sb.append(Bytes.toString(kv.getValue()));
// EntBaseinfo baseInfo = new EntBaseinfo();
// baseInfo.parseFromString(sb.toString());
// System.out.println(baseInfo.getENTNAME());
// if(baseInfo.getNAME()!=null&&baseInfo.getNAME().isEmpty()){
// System.out.println(baseInfo.getENTNAME());
// }
//
//
// if(baseInfo.getDOM()!=null&&baseInfo.getNAME().isEmpty()){
// System.out.println(baseInfo.getENTNAME());
// }
}
if (count > 1000) {
return;
}
}
} finally {
rs.close();
}
}
public void checkHbase() throws IOException {
List<RequiredEmail> temp = new LinkedList<RequiredEmail>(requiredEmails);
HTableInterface table = hbase.getTestingTable();
Scan scan =
createScan(
hbase.getFamily(), new String[] {"Subject", "Sender", "Bcc", "Cc", "To", "Body"});
ResultScanner scanner = table.getScanner(scan);
int countInHBase = 0;
List<Result> extraResults = new LinkedList<Result>();
for (Result result = scanner.next(); result != null; result = scanner.next()) {
countInHBase++;
if (!removeResult(temp, result)) {
extraResults.add(result);
}
}
assertEmpty(temp);
assertNoExtraResults(extraResults);
}
private void loadIPs() {
dns = new HashMap(100000000); // ���貢��
unknownHosts = new HashMap(1000000);
querying = new HashMap(100000);
try {
int statsCommit = 500000;
HConnection connection = HConnectionManager.createConnection(HBaseConfiguration.create());
HTableInterface fetchFailTable = connection.getTable("fetchFail");
Scan scan = new Scan();
scan.setCaching(statsCommit);
List<Filter> filters = new ArrayList<Filter>();
Filter filter = new ColumnPrefixFilter(Bytes.toBytes("ip"));
filters.add(filter);
FilterList filterList = new FilterList(filters);
scan.setFilter(filterList);
ResultScanner rs = fetchFailTable.getScanner(scan);
long cnt = 0;
for (Result r : rs) {
NavigableMap<byte[], byte[]> map = r.getFamilyMap(Bytes.toBytes("cf"));
String ip = Bytes.toString(map.get(Bytes.toBytes("ip")));
String host = Bytes.toString(r.getRow()).split("��")[0];
if (host != null && ip != null) {
dns.put(host, ip);
}
if (++cnt % statsCommit == 0) {
LOG.info("loadIPs url=" + Bytes.toString(r.getRow()) + " cnt=" + cnt);
}
}
rs.close();
fetchFailTable.close();
LOG.info("load hostip cache=" + dns.size());
connection.close();
} catch (Exception e) {
e.printStackTrace();
} finally {
//
}
}
@Test
public void testEmptyRowId() throws IOException {
deleteTable(TABLE);
HTablePool pool = new HTablePool();
HTableInterface table = pool.getTable(TABLE);
byte[] EMPTY = new byte[0];
Put put = new Put(EMPTY);
put.add(COLUMN_FAMILY, $("test").getBytes(), $("test").getBytes());
table.put(put);
Scan scan = new Scan();
boolean found = false;
for (Result r : table.getScanner(scan)) {
assertFalse(found);
found = true;
assertEquals(0, EMPTY.length);
}
assertTrue(found);
deleteTable(TABLE);
}
public static void scanTest(String tableStr) {
try {
Configuration conf = HBaseConfiguration.create();
byte[] tableName = Bytes.toBytes(tableStr);
HConnection hConnection = HConnectionManager.createConnection(conf);
HTableInterface table = hConnection.getTable(tableName);
byte[] startRow = Bytes.toBytes("rowKey_0");
byte[] stopRow = Bytes.toBytes("rowKey_6");
byte[] family = f0;
Scan scan = new Scan();
scan.addFamily(family);
scan.setMaxVersions(1);
// scan.setStartRow(startRow);
// scan.setStopRow(stopRow);
int count = 0;
ResultScanner scanner = table.getScanner(scan);
Result result = scanner.next();
while (result != null) {
String rowKey = Bytes.toString(result.getRow());
NavigableMap<byte[], byte[]> m = result.getFamilyMap(family);
if (m == null || m.isEmpty()) {
System.err.println("Empty." + m);
return;
}
for (Map.Entry<byte[], byte[]> entry : m.entrySet()) {
String qualifier = Bytes.toString(entry.getKey());
String value = Bytes.toString(entry.getValue());
System.out.println(rowKey + ":" + qualifier + ":" + value);
}
result = scanner.next();
count++;
System.out.println("-----------------------------");
}
table.close(); // very important
System.out.println("count:" + count);
} catch (IOException e) {
e.printStackTrace();
}
}
private static void assertTenantIds(
Expression e, HTableInterface htable, Filter filter, String[] tenantIds) throws IOException {
ImmutableBytesWritable ptr = new ImmutableBytesWritable();
Scan scan = new Scan();
scan.setFilter(filter);
ResultScanner scanner = htable.getScanner(scan);
Result result = null;
ResultTuple tuple = new ResultTuple();
List<String> actualTenantIds = Lists.newArrayListWithExpectedSize(tenantIds.length);
List<String> expectedTenantIds = Arrays.asList(tenantIds);
while ((result = scanner.next()) != null) {
tuple.setResult(result);
e.evaluate(tuple, ptr);
String tenantId = (String) PVarchar.INSTANCE.toObject(ptr);
actualTenantIds.add(tenantId == null ? "" : tenantId);
}
// Need to sort because of salting
Collections.sort(actualTenantIds);
assertEquals(expectedTenantIds, actualTenantIds);
}
/** Delte all the data in table. */
private void deleteData() throws Exception {
try {
// full scan.
Scan scan = new Scan();
ResultScanner resultScanner = table.getScanner(scan);
List<byte[]> rows = new LinkedList<byte[]>();
for (Result result = resultScanner.next(); result != null; result = resultScanner.next()) {
rows.add(result.getRow());
}
close(resultScanner);
for (byte[] row : rows) {
table.delete(new Delete(row));
log.info("delete " + Bytes.toString(row));
}
} catch (IOException e) {
throw new RuntimeException(e);
}
}
private Result getByScan(String path, byte[] family, byte[] column) throws IOException {
byte[] startRow = Bytes.toBytes(path);
byte[] endRow = plusZero(startRow);
Scan scan = new Scan(startRow, endRow);
if (family == null || column == null) {
scan.setFilter(new KeyOnlyFilter());
} else {
scan.addColumn(family, column);
}
HTableInterface table = getConnection().getTable(getAllInOneTableName());
try {
ResultScanner scanner = table.getScanner(scan);
Result result = null;
for (Result r : scanner) {
result = r;
}
return result == null || result.isEmpty() ? null : result;
} finally {
IOUtils.closeQuietly(table);
}
}
private static void deleteTable(byte[] tableName) {
HTablePool pool = new HTablePool();
HTableInterface table = null;
try {
table = pool.getTable(tableName);
Scan scan = new Scan();
ResultScanner scanner = table.getScanner(scan);
for (Result r : scanner) {
Delete delete = new Delete(r.getRow());
NavigableMap<byte[], NavigableMap<byte[], byte[]>> map = r.getNoVersionMap();
for (Map.Entry<byte[], NavigableMap<byte[], byte[]>> family : map.entrySet()) {
delete.deleteFamily(family.getKey());
}
table.delete(delete);
}
Delete delete = new Delete(new byte[0]);
table.delete(delete);
} catch (Exception e) {
// ignore
} finally {
if (table != null) pool.putTable(table);
}
}
private PageBean getDataMapTemp(SearchParam searchParam, Integer currentPage, Integer pageSize)
throws IOException {
String tableName = searchParam.getTableName();
String startRow = searchParam.getStartKey();
String stopRow = searchParam.getEndKey();
List<Map<String, String>> mapList = null;
mapList = new LinkedList<Map<String, String>>();
ResultScanner scanner = null;
// 为分页创建的封装类对象，下面有给出具体属性 TBData tbData = null;
PageBean tbData = new PageBean();
Scan scan = null;
HbaseWhereEngine whereEngine = null;
HTableInterface table = HbaseTool.getTable(tableName); // 获取筛选对象
if ("ca_summary_optimize".equals(tableName) && compoundFieldMap.isEmpty()) {
compoundFieldMap = CompoundFieldConfigService.getParentComField(tableName);
}
try {
if (StringUtils.isNotEmpty(searchParam.getWhere())) {
whereEngine =
new HbaseWhereEngine(tableName, searchParam.getFamily(), searchParam.getWhere());
}
String[] selectArray = null;
if (StringUtils.isNotEmpty(searchParam.getSelect())) {
selectArray = searchParam.getSelect().split(",");
}
byte[] cf = Bytes.toBytes(searchParam.getFamily());
// if keys is not null,then we can sure which records to be selected!
if (StringUtils.isNotEmpty(searchParam.getKeys())) {
List<Get> getKeysList = new ArrayList<Get>();
for (String key : searchParam.getKeys().split(",")) {
Get get = new Get(Bytes.toBytes(key));
getKeysList.add(get);
if (selectArray != null) {
for (String field : selectArray) {
String temp[] = processField(field, searchParam.getFamily());
if ("ca_summary_optimize".equals(tableName)
&& compoundFieldMap.containsKey(temp[1])) {
get.addColumn(Bytes.toBytes(temp[0]), Bytes.toBytes(compoundFieldMap.get(temp[1])));
} else {
get.addColumn(Bytes.toBytes(temp[0]), Bytes.toBytes(temp[1]));
}
}
}
if (selectArray != null && whereEngine != null) {
Set<String> varSet = whereEngine.getVarSet();
for (String var : varSet) {
get.addColumn(cf, Bytes.toBytes(var));
}
}
}
Result[] resultsFromKeys = table.get(getKeysList);
for (Result rr : resultsFromKeys) {
if (!rr.isEmpty()) {
if (whereEngine != null && !whereEngine.meetCondition(rr)) {
continue;
}
Map<String, String> map = new TreeMap<String, String>();
map.put("_id", Bytes.toString(rr.getRow()));
for (String field : selectArray) {
String value = HbaseWhereEngine.getColumnValue(tableName, cf, rr, field);
if (!field.equals("id")) {
map.put(field, value);
}
}
mapList.add(map);
}
}
pageSize = mapList.size();
tbData.setCurrentPage(currentPage);
tbData.setLength(pageSize);
tbData.setTotalRecords(mapList.size());
} else { // if keys is null,we select some records between startKey and end key or top
// pageSize !
// 获取最大返回结果数量
if (pageSize == null || pageSize == 0L) pageSize = 100;
if (currentPage == null || currentPage == 0) currentPage = 1;
// 计算起始页和结束页
Integer firstPage = (currentPage - 1) * pageSize;
Integer endPage = firstPage + pageSize;
scan = getScan(startRow, stopRow);
// 给筛选对象放入过滤器(true标识分页,具体方法在下面)
scan.setFilter(packageFilters(searchParam, true));
// 缓存1000条数据
scan.setCaching(1000);
scan.setCacheBlocks(false);
if (selectArray != null) {
for (String field : selectArray) {
String temp[] = processField(field, searchParam.getFamily());
if ("ca_summary_optimize".equals(tableName) && compoundFieldMap.containsKey(temp[1])) {
scan.addColumn(Bytes.toBytes(temp[0]), Bytes.toBytes(compoundFieldMap.get(temp[1])));
} else {
scan.addColumn(Bytes.toBytes(temp[0]), Bytes.toBytes(temp[1]));
}
}
}
if (selectArray != null && whereEngine != null) {
Set<String> varSet = whereEngine.getVarSet();
for (String var : varSet) {
scan.addColumn(cf, Bytes.toBytes(var));
}
}
scanner = table.getScanner(scan);
int i = 0;
List<byte[]> rowList = new LinkedList<byte[]>();
// 遍历扫描器对象， 并将需要查询出来的数据row key取出
for (Result result : scanner) {
String row = toStr(result.getRow());
if (i >= firstPage && i < endPage) { // filter firstPage
rowList.add(getBytes(row));
}
if (i >= endPage) {
break;
}
i++;
}
// 获取取出的row key的GET对象
List<Get> getList = getList(rowList, cf, selectArray, tableName);
Result[] results = table.get(getList);
for (Result result : results) {
if (whereEngine != null && !whereEngine.meetCondition(result)) {
continue;
}
Map<byte[], byte[]> fmap = packFamilyMap(tableName, result, cf, selectArray);
Map<String, String> rmap = packRowMap(fmap);
rmap.put("_id", toStr(result.getRow()));
mapList.add(rmap);
}
// 封装分页对象
tbData.setCurrentPage(currentPage);
tbData.setLength(pageSize); // (pageSize);
tbData.setTotalRecords(i);
}
tbData.setResults(mapList);
} catch (Exception e) {
} finally {
if (table != null) {
table.close();
}
closeScanner(scanner);
}
compoundFieldMap.clear();
return tbData;
}
private ResultScanner createHBaseResultScanner(RecordScan scan)
throws RepositoryException, InterruptedException {
Scan hbaseScan = new Scan();
hbaseScan.setMaxVersions(1);
if (scan.getRawStartRecordId() != null) {
hbaseScan.setStartRow(scan.getRawStartRecordId());
} else if (scan.getStartRecordId() != null) {
hbaseScan.setStartRow(scan.getStartRecordId().toBytes());
}
if (scan.getRawStopRecordId() != null) {
hbaseScan.setStopRow(scan.getRawStopRecordId());
} else if (scan.getStopRecordId() != null) {
hbaseScan.setStopRow(scan.getStopRecordId().toBytes());
}
// Filters
FilterList filterList = new FilterList(FilterList.Operator.MUST_PASS_ALL);
// filter out deleted records
filterList.addFilter(REAL_RECORDS_FILTER);
// add user's filter
if (scan.getRecordFilter() != null) {
Filter filter = filterFactory.createHBaseFilter(scan.getRecordFilter(), this, filterFactory);
filterList.addFilter(filter);
}
hbaseScan.setFilter(filterList);
hbaseScan.setCaching(scan.getCaching());
hbaseScan.setCacheBlocks(scan.getCacheBlocks());
ReturnFields returnFields = scan.getReturnFields();
if (returnFields != null && returnFields.getType() != ReturnFields.Type.ALL) {
RecordDecoder.addSystemColumnsToScan(hbaseScan);
switch (returnFields.getType()) {
case ENUM:
for (QName field : returnFields.getFields()) {
FieldTypeImpl fieldType = (FieldTypeImpl) typeManager.getFieldTypeByName(field);
hbaseScan.addColumn(RecordCf.DATA.bytes, fieldType.getQualifier());
}
break;
case NONE:
// nothing to add
break;
default:
throw new RuntimeException("Unrecognized ReturnFields type: " + returnFields.getType());
}
} else {
hbaseScan.addFamily(RecordCf.DATA.bytes);
}
ResultScanner hbaseScanner;
try {
hbaseScanner = recordTable.getScanner(hbaseScan);
} catch (IOException e) {
throw new RecordException("Error creating scanner", e);
}
return hbaseScanner;
}