public boolean fetchReadAllFieldLine(List<Line> lines, LineSender sender) throws IOException {
if (null == this.rs) {
throw new IllegalStateException("HBase Client try to fetch data failed .");
}
for (Result result = rs.next(); result != null; result = rs.next()) {
Get get = new Get(result.getRow());
for (int i = 0; i < this.families.length; i++) {
get.addColumn(this.families[i].getBytes(), this.columns[i].getBytes());
}
gets.add(get);
if (gets.size() > this.BUFFER_LINE) {
Result[] getResults = this.htable.get(gets);
for (Result resu : getResults) {
if (null != resu) {
Line line = sender.createLine();
for (int i = 0; i < this.families.length; i++) {
byte[] value = resu.getValue(this.families[i].getBytes(), this.columns[i].getBytes());
if (null == value) {
line.addField(null);
} else {
line.addField(new String(value, encode));
}
}
line.addField(new String(resu.getRow(), encode));
}
}
return true;
}
}
return false;
}
private void testSimpleScanInternal(
long origKeyPrefix,
Scan scan,
int numValues,
int startWithValue,
int seekIntervalMinValue,
int seekIntervalMaxValue)
throws IOException {
int valuesCountInSeekInterval =
writeTestData(
origKeyPrefix, numValues, startWithValue, seekIntervalMinValue, seekIntervalMaxValue);
// TODO: add some filters to the scan for better testing
ResultScanner distributedScanner = DistributedScanner.create(hTable, scan, keyDistributor);
Result previous = null;
int countMatched = 0;
for (Result current : distributedScanner) {
countMatched++;
if (previous != null) {
byte[] currentRowOrigKey = keyDistributor.getOriginalKey(current.getRow());
byte[] previousRowOrigKey = keyDistributor.getOriginalKey(previous.getRow());
Assert.assertTrue(Bytes.compareTo(currentRowOrigKey, previousRowOrigKey) >= 0);
int currentValue = Bytes.toInt(current.getValue(CF, QUAL));
Assert.assertTrue(currentValue >= seekIntervalMinValue);
Assert.assertTrue(currentValue <= seekIntervalMaxValue);
}
previous = current;
}
Assert.assertEquals(valuesCountInSeekInterval, countMatched);
}
private HashMap<String, String[][]> getObjectList(
ResultScanner resultScan, byte[][] family, List<byte[]> rowKeyList) {
String tableFamilies; // = new table;
String data[][];
String rowKey;
HashMap<String, String[][]> tableDataList = new HashMap<String, String[][]>();
long k = 0;
for (Result rows : resultScan) {
System.out.println("Rows passed : " + (++k));
for (int i = 0; i < family.length; i++) {
tableFamilies = Bytes.toString(family[i]);
rowKey = Bytes.toString(rows.getRow());
rowKeyList.add(rows.getRow());
if (tblMngr.getDataFiller(rows, tableFamilies)) {
data = tblMngr.getResultMap();
tableFamilies = tableFamilies + "," + rowKey;
tableDataList.put(tableFamilies, data);
}
}
}
return tableDataList;
}
/**
* Looks at every value of the mapreduce output and verifies that indeed the values have been
* reversed.
*
* @param table Table to scan.
* @throws IOException
* @throws NullPointerException if we failed to find a cell value
*/
private void verifyAttempt(final Table table) throws IOException, NullPointerException {
Scan scan = new Scan();
scan.addFamily(INPUT_FAMILY);
scan.addFamily(OUTPUT_FAMILY);
ResultScanner scanner = table.getScanner(scan);
try {
Iterator<Result> itr = scanner.iterator();
assertTrue(itr.hasNext());
while (itr.hasNext()) {
Result r = itr.next();
if (LOG.isDebugEnabled()) {
if (r.size() > 2) {
throw new IOException("Too many results, expected 2 got " + r.size());
}
}
byte[] firstValue = null;
byte[] secondValue = null;
int count = 0;
for (Cell kv : r.listCells()) {
if (count == 0) {
firstValue = CellUtil.cloneValue(kv);
} else if (count == 1) {
secondValue = CellUtil.cloneValue(kv);
} else if (count == 2) {
break;
}
count++;
}
String first = "";
if (firstValue == null) {
throw new NullPointerException(Bytes.toString(r.getRow()) + ": first value is null");
}
first = Bytes.toString(firstValue);
String second = "";
if (secondValue == null) {
throw new NullPointerException(Bytes.toString(r.getRow()) + ": second value is null");
}
byte[] secondReversed = new byte[secondValue.length];
for (int i = 0, j = secondValue.length - 1; j >= 0; j--, i++) {
secondReversed[i] = secondValue[j];
}
second = Bytes.toString(secondReversed);
if (first.compareTo(second) != 0) {
if (LOG.isDebugEnabled()) {
LOG.debug(
"second key is not the reverse of first. row="
+ Bytes.toStringBinary(r.getRow())
+ ", first value="
+ first
+ ", second value="
+ second);
}
fail();
}
}
} finally {
scanner.close();
}
}
public static void main(String[] args) throws Exception {
conf.set("hbase.zookeeper.quorum", "hadoop271.itversity.com");
conf.set("hbase.zookeeper.property.clientPort", "2181");
Connection connection = ConnectionFactory.createConnection(conf);
Table table = connection.getTable(TableName.valueOf("demo"));
Scan scan1 = new Scan();
ResultScanner scanner1 = table.getScanner(scan1);
for (Result res : scanner1) {
System.out.println(Bytes.toString(res.getRow()));
System.out.println(Bytes.toString(res.getValue("cf1".getBytes(), "column1".getBytes())));
System.out.println(Bytes.toString(res.getValue("cf1".getBytes(), "column2".getBytes())));
}
scanner1.close();
Put put = new Put("3".getBytes());
put.addColumn("cf1".getBytes(), "column1".getBytes(), "value1".getBytes());
put.addColumn("cf1".getBytes(), "column2".getBytes(), "value2".getBytes());
table.put(put);
Get get = new Get("3".getBytes());
Result getResult = table.get(get);
System.out.println("Printing colunns for rowkey 3");
System.out.println(Bytes.toString(getResult.getValue("cf1".getBytes(), "column1".getBytes())));
System.out.println(Bytes.toString(getResult.getValue("cf1".getBytes(), "column2".getBytes())));
scanner1 = table.getScanner(scan1);
System.out.println("Before Delete");
for (Result res : scanner1) {
System.out.println(Bytes.toString(res.getRow()));
System.out.println(Bytes.toString(res.getValue("cf1".getBytes(), "column1".getBytes())));
System.out.println(Bytes.toString(res.getValue("cf1".getBytes(), "column2".getBytes())));
}
scanner1.close();
Delete del = new Delete("3".getBytes());
table.delete(del);
System.out.println("After Delete");
scanner1 = table.getScanner(scan1);
for (Result res : scanner1) {
System.out.println(Bytes.toString(res.getRow()));
System.out.println(Bytes.toString(res.getValue("cf1".getBytes(), "column1".getBytes())));
System.out.println(Bytes.toString(res.getValue("cf1".getBytes(), "column2".getBytes())));
}
scanner1.close();
table.close();
connection.close();
}
@Override
protected SizeResult createResult(Result result) {
if (result.getRow() != null) {
SizeResult size = new SizeResult(result.getRow());
size.setProbability(result.getValue(Bytes.toBytes("bp"), Bytes.toBytes("prob")));
return size;
}
return null;
}
// Get all events with exit at last location
public static Map<Integer, VesselEvent> getAllEventsStartBeforeEndAfterBeforeLocation(
Table VTEvent_Table, String IMO_str, VesselLocation location) throws IOException {
Scan getAllEventsWithExistAtLastLocation = new Scan();
getAllEventsWithExistAtLastLocation
.setStartRow(
Bytes.toBytes(
IMO_str + LpadNum(Long.MAX_VALUE - location.recordtime, 19) + "0000000000"))
.setStopRow(Bytes.toBytes(IMO_str + LpadNum(Long.MAX_VALUE, 19) + "9999999999"))
.addColumn(details, exittime);
getAllEventsWithExistAtLastLocation.setCaching(100);
Filter ExistTimeValuefilter =
new ValueFilter(
CompareFilter.CompareOp.GREATER_OR_EQUAL,
new BinaryComparator(
Bytes.toBytes(new DateTime(location.recordtime).toString(rawformatter))));
getAllEventsWithExistAtLastLocation.setFilter(ExistTimeValuefilter);
ResultScanner Result_event = VTEvent_Table.getScanner(getAllEventsWithExistAtLastLocation);
Map<Integer, VesselEvent> events = new HashMap<Integer, VesselEvent>();
for (Result res : Result_event) {
Get get = new Get(res.getRow());
get.addColumn(details, entrytime);
get.addColumn(details, entrycoordinates);
Result result = VTEvent_Table.get(get);
String rowkey = Bytes.toString(result.getRow());
String polygonid = rowkey.substring(26);
VesselEvent VE = new VesselEvent();
VE.exittime = location.recordtime;
VE.exitcoordinates = location.coordinates;
VE.destination = location.destination;
VE.polygonid = Integer.parseInt(polygonid);
for (Cell cell : result.rawCells()) {
String Qualifier = Bytes.toString(CellUtil.cloneQualifier(cell));
String Value = Bytes.toString(CellUtil.cloneValue(cell));
if (Qualifier.equals("entertime")) {
VE.entrytime = DateTime.parse(Value, rawformatter).getMillis();
} else if (Qualifier.equals("entercoordinates")) {
VE.entrycoordinates = Value;
}
}
events.put(VE.polygonid, VE);
}
Result_event.close();
return events;
}
public void doAnAction() throws Exception {
Scan s = new Scan();
for (byte[] family : targetFamilies) {
s.addFamily(family);
}
ResultScanner scanner = table.getScanner(s);
for (Result res : scanner) {
byte[] lastRow = null, lastFam = null, lastQual = null;
byte[] gotValue = null;
for (byte[] family : targetFamilies) {
byte qualifier[] = QUAL;
byte thisValue[] = res.getValue(family, qualifier);
if (gotValue != null && thisValue != null && !Bytes.equals(gotValue, thisValue)) {
StringBuilder msg = new StringBuilder();
msg.append("Failed on scan ")
.append(numScans)
.append(" after scanning ")
.append(numRowsScanned)
.append(" rows!\n");
msg.append(
"Current was "
+ Bytes.toString(res.getRow())
+ "/"
+ Bytes.toString(family)
+ ":"
+ Bytes.toString(qualifier)
+ " = "
+ Bytes.toString(thisValue)
+ "\n");
msg.append(
"Previous was "
+ Bytes.toString(lastRow)
+ "/"
+ Bytes.toString(lastFam)
+ ":"
+ Bytes.toString(lastQual)
+ " = "
+ Bytes.toString(gotValue));
throw new RuntimeException(msg.toString());
}
lastFam = family;
lastQual = qualifier;
lastRow = res.getRow();
gotValue = thisValue;
}
numRowsScanned.getAndIncrement();
}
numScans.getAndIncrement();
}
public Blog(String blogid) throws IOException {
Configuration conf = HBaseConfiguration.create();
table = new HTable(conf, "blogs");
// 1. Get the row whose row key is blogid from above
Get g = new Get(Bytes.toBytes(blogid));
Result r = table.get(g);
// 2. Extract the rowkey, blog text (column "body") and blog title
// (column "meta:title")
key = r.getRow();
keyStr = Bytes.toString(key);
blogText = Bytes.toString(r.getValue(Bytes.toBytes("body"), Bytes.toBytes("")));
blogTitle = Bytes.toString(r.getValue(Bytes.toBytes("meta"), Bytes.toBytes("title")));
Long reverseTimestamp = Long.parseLong(keyStr.substring(4));
Long epoch = Math.abs(reverseTimestamp - Long.MAX_VALUE);
dateOfPost = new Date(epoch);
// Get an iterator for the comments
Scan s = new Scan();
s.addFamily(Bytes.toBytes("comment"));
// Use a PrefixFilter
PrefixFilter filter = new PrefixFilter(key);
s.setFilter(filter);
scanner = table.getScanner(s);
resultIterator = scanner.iterator();
}
public static List<Delete> GetDeleteEventsBetween(
Table VTEvent_Table, String imo_str, long first_timestamp, long last_timestamp)
throws IOException {
// scan
// 'cdb_vessel:vessel_event',{FILTER=>"(PrefixFilter('0000003162')"}
Scan GetEventsBetween = new Scan();
GetEventsBetween.setStartRow(
Bytes.toBytes(imo_str + LpadNum(Long.MAX_VALUE - last_timestamp, 19) + "0000000000"))
.setStopRow(
Bytes.toBytes(
imo_str + LpadNum(Long.MAX_VALUE - first_timestamp + 1, 19) + "9999999999"))
.addColumn(details, exittime);
GetEventsBetween.setCaching(100);
Filter ExistTimeValuefilter =
new ValueFilter(
CompareFilter.CompareOp.LESS_OR_EQUAL,
new BinaryComparator(
Bytes.toBytes(new DateTime(last_timestamp).toString(rawformatter))));
GetEventsBetween.setFilter(ExistTimeValuefilter);
ResultScanner Result_ExistingEvents = VTEvent_Table.getScanner(GetEventsBetween);
List<Delete> deletes = new ArrayList<Delete>();
for (Result res : Result_ExistingEvents) {
deletes.add(new Delete(res.getRow()));
}
Result_ExistingEvents.close();
return deletes;
}
/**
* 遍历多行
*
* @param tableName 表名
* @param start_rowkey 开始行键
* @param stop_rowkey 结束行键
* @return 行列表
*/
public ArrayList<HbaseRow> scanRows(String tableName, String start_rowkey, String stop_rowkey) {
ResultScanner rowstmp = null;
ArrayList<HbaseRow> rows = null;
try {
Scan scan = new Scan();
scan.setStartRow(Bytes.toBytes(start_rowkey));
scan.setStopRow(Bytes.toBytes(stop_rowkey));
HTable table = new HTable(conf, Bytes.toBytes(tableName));
rowstmp = table.getScanner(scan);
rows = new ArrayList<>();
for (Result rowtmp : rowstmp) {
HbaseRow row = new HbaseRow();
row.rowkey = Bytes.toString(rowtmp.getRow());
for (Cell cell : rowtmp.listCells()) {
HbaseColumn col = new HbaseColumn(cell);
row.cols.add(col);
}
rows.add(row);
}
} catch (Exception e) {
logger.error("scanRows failed", e);
} finally {
rowstmp.close();
}
return rows;
}
@TimeDepend
@Test
public void testScan_ts_same() throws Exception {
recreateTable();
Date ts = parse("2000-01-01", "yyyy-MM-dd");
Put put = new Put(rowKey_ForTest);
put.add(ColumnFamilyNameBytes, QName1, ts.getTime(), Bytes.toBytes("a"));
table.put(put);
Set<String> resultRowKeys = new HashSet<String>();
Scan scan = new Scan(rowKey_ForTest, rowKey_ForTest);
scan.setTimeRange(ts.getTime(), ts.getTime());
ResultScanner resultScanner = table.getScanner(scan);
for (Result result = resultScanner.next(); result != null; result = resultScanner.next()) {
resultRowKeys.add(Bytes.toString(result.getRow()));
}
close(resultScanner);
Assert.assertTrue(resultRowKeys.size() == 0);
recreateTable();
}
public static List<String> getBooksbyPrice(String min, String max) throws IOException {
List list = new ArrayList<String>();
Scan scan = new Scan();
scan.setMaxVersions();
FilterList filterList = new FilterList();
Filter maxFilter =
new SingleColumnValueFilter(
Bytes.toBytes("statics"),
Bytes.toBytes("price"),
CompareOp.GREATER_OR_EQUAL,
Bytes.toBytes(min));
Filter minFilter =
new SingleColumnValueFilter(
Bytes.toBytes("statics"),
Bytes.toBytes("price"),
CompareOp.LESS_OR_EQUAL,
Bytes.toBytes(max));
filterList.addFilter(maxFilter);
filterList.addFilter(minFilter);
scan.setFilter(filterList);
ResultScanner rs = table.getScanner(scan);
for (Result r : rs) {
String str = "title: " + Bytes.toString(r.getRow());
for (KeyValue kv : r.raw()) {
str = str + " " + Bytes.toString(kv.getQualifier()) + ": ";
str += Bytes.toString(kv.getValue());
}
System.out.println(str);
list.add(str);
}
return list;
}
public static List<String> getFreeBooks() throws IOException {
List list = new ArrayList<String>();
Scan scan = new Scan();
scan.setMaxVersions();
Filter filter =
new SingleColumnValueFilter(
Bytes.toBytes("statics"),
Bytes.toBytes("price"),
CompareOp.EQUAL,
Bytes.toBytes("0.00"));
scan.setFilter(filter);
ResultScanner rs = table.getScanner(scan);
for (Result r : rs) {
String str = "title: " + Bytes.toString(r.getRow());
for (KeyValue kv : r.raw()) {
str = str + " " + Bytes.toString(kv.getQualifier()) + ": ";
str += Bytes.toString(kv.getValue());
}
/*
* for (KeyValue kv : r.raw()) { System.out.println(String.format(
* "row:%s, family:%s, qualifier:%s, qualifiervalue:%s, timestamp:%s."
* , Bytes.toString(kv.getRow()), Bytes.toString(kv.getFamily()),
* Bytes.toString(kv.getQualifier()), Bytes.toString(kv.getValue()),
* kv.getTimestamp())); }
*/
System.out.println(str);
list.add(str);
}
return list;
}
@Override
public void emitTuples() {
try {
Scan scan = nextScan();
if (scan == null) return;
ResultScanner resultScanner = store.getTable().getScanner(scan);
while (true) {
Result result = resultScanner.next();
if (result == null) break;
String readRow = Bytes.toString(result.getRow());
if (readRow.equals(lastReadRow)) continue;
Object instance = pojoType.newInstance();
rowSetter.set(instance, readRow);
List<Cell> cells = result.listCells();
for (Cell cell : cells) {
String columnName = Bytes.toString(CellUtil.cloneQualifier(cell));
byte[] value = CellUtil.cloneValue(cell);
fieldValueGenerator.setColumnValue(instance, columnName, value, valueConverter);
}
outputPort.emit(instance);
lastReadRow = readRow;
}
} catch (Exception e) {
throw new RuntimeException(e.getMessage());
}
}
@Override
protected ArrayList<String> listResourcesImpl(String resPath) throws IOException {
assert resPath.startsWith("/");
String lookForPrefix = resPath.endsWith("/") ? resPath : resPath + "/";
byte[] startRow = Bytes.toBytes(lookForPrefix);
byte[] endRow = Bytes.toBytes(lookForPrefix);
endRow[endRow.length - 1]++;
ArrayList<String> result = new ArrayList<String>();
HTableInterface table = getConnection().getTable(getAllInOneTableName());
Scan scan = new Scan(startRow, endRow);
scan.setFilter(new KeyOnlyFilter());
try {
ResultScanner scanner = table.getScanner(scan);
for (Result r : scanner) {
String path = Bytes.toString(r.getRow());
assert path.startsWith(lookForPrefix);
int cut = path.indexOf('/', lookForPrefix.length());
String child = cut < 0 ? path : path.substring(0, cut);
if (result.contains(child) == false) result.add(child);
}
} finally {
IOUtils.closeQuietly(table);
}
// return null to indicate not a folder
return result.isEmpty() ? null : result;
}
@Override
protected List<RawResource> getAllResources(String rangeStart, String rangeEnd)
throws IOException {
byte[] startRow = Bytes.toBytes(rangeStart);
byte[] endRow = plusZero(Bytes.toBytes(rangeEnd));
Scan scan = new Scan(startRow, endRow);
scan.addColumn(B_FAMILY, B_COLUMN_TS);
scan.addColumn(B_FAMILY, B_COLUMN);
HTableInterface table = getConnection().getTable(getAllInOneTableName());
List<RawResource> result = Lists.newArrayList();
try {
ResultScanner scanner = table.getScanner(scan);
for (Result r : scanner) {
result.add(new RawResource(getInputStream(Bytes.toString(r.getRow()), r), getTimestamp(r)));
}
} catch (IOException e) {
for (RawResource rawResource : result) {
IOUtils.closeQuietly(rawResource.resource);
}
throw e;
} finally {
IOUtils.closeQuietly(table);
}
return result;
}
/**
* This utility method creates a list of Thrift TRowResult "struct" based on an Hbase RowResult
* object. The empty list is returned if the input is null.
*
* @param in Hbase RowResult object
* @param sortColumns This boolean dictates if row data is returned in a sorted order sortColumns
* = True will set TRowResult's sortedColumns member which is an ArrayList of TColumn struct
* sortColumns = False will set TRowResult's columns member which is a map of columnName and
* TCell struct
* @return Thrift TRowResult array
*/
public static List<TRowResult> rowResultFromHBase(Result[] in, boolean sortColumns) {
List<TRowResult> results = new ArrayList<TRowResult>();
for (Result result_ : in) {
if (result_ == null || result_.isEmpty()) {
continue;
}
TRowResult result = new TRowResult();
result.row = ByteBuffer.wrap(result_.getRow());
if (sortColumns) {
result.sortedColumns = new ArrayList<TColumn>();
for (Cell kv : result_.rawCells()) {
result.sortedColumns.add(
new TColumn(
ByteBuffer.wrap(
KeyValue.makeColumn(CellUtil.cloneFamily(kv), CellUtil.cloneQualifier(kv))),
new TCell(ByteBuffer.wrap(CellUtil.cloneValue(kv)), kv.getTimestamp())));
}
} else {
result.columns = new TreeMap<ByteBuffer, TCell>();
for (Cell kv : result_.rawCells()) {
result.columns.put(
ByteBuffer.wrap(
KeyValue.makeColumn(CellUtil.cloneFamily(kv), CellUtil.cloneQualifier(kv))),
new TCell(ByteBuffer.wrap(CellUtil.cloneValue(kv)), kv.getTimestamp()));
}
}
results.add(result);
}
return results;
}
/* (non-Javadoc)
* @see com.hazelcast.core.MapLoader#loadAllKeys()
*/
@Override
public Set<String> loadAllKeys() {
Set<String> keySet = null;
if (allowLoadAll) {
keySet = new HashSet<String>();
HTableInterface hti = null;
try {
hti = pool.getTable(tableName);
Scan s = new Scan();
s.addColumn(family, qualifier);
ResultScanner rs = hti.getScanner(s);
Result r = null;
while ((r = rs.next()) != null) {
String k = new String(r.getRow());
keySet.add(k);
}
} catch (IOException e) {
LOG.error("IOException while loading all keys", e);
} finally {
if (hti != null) {
pool.putTable(hti);
}
}
}
return keySet;
}
/**
* rowFilter的使用
*
* @param tableName
* @param reg
* @throws Exception
*/
public void getRowFilter(String tableName, String reg) throws Exception {
Connection conn = null;
HTable table = null;
try {
conn = ConnectionFactory.createConnection(conf);
table = (HTable) conn.getTable(TableName.valueOf(tableName));
Scan scan = new Scan();
// Filter
RowFilter rowFilter = new RowFilter(CompareOp.NOT_EQUAL, new RegexStringComparator(reg));
scan.setFilter(rowFilter);
ResultScanner scanner = table.getScanner(scan);
for (Result result : scanner) {
System.out.println(new String(result.getRow()));
}
} catch (Exception e) {
e.printStackTrace();
} finally {
if (null != table) {
try {
table.close();
} catch (IOException e) {
logger.error("HTable close exception, errMsg:{}", e.getMessage());
}
}
if (null != conn) {
try {
conn.close();
} catch (Exception e) {
logger.error("Connection close exception, errMsg:{}", e.getMessage());
}
}
}
}
public static void deleteTest(String tableStr) {
try {
Configuration conf = HBaseConfiguration.create();
byte[] tableName = Bytes.toBytes(tableStr);
HConnection hConnection = HConnectionManager.createConnection(conf);
HTableInterface table = hConnection.getTable(tableName);
byte[] startRow = Bytes.toBytes("rowKey_1");
byte[] stopRow = Bytes.toBytes("rowKey_3");
byte[] family = f0;
Scan scan = new Scan();
scan.addFamily(family);
scan.setMaxVersions(1);
// scan.setStartRow(startRow);
// scan.setStopRow(stopRow);
ResultScanner scanner = table.getScanner(scan);
Result result = scanner.next();
List<Delete> delete = new ArrayList<Delete>();
while (result != null) {
Delete del = new Delete(result.getRow());
delete.add(del);
result = scanner.next();
}
table.delete(delete);
System.out.println("delete done");
table.close(); // very important
} catch (IOException e) {
e.printStackTrace();
}
}
@Override
public GCResult createResult(Result result) {
if (result.getRow() != null) {
GCResult gcResult = new GCResult(result.getRow());
gcResult.setChromosome(result.getValue(Bytes.toBytes("chr"), Bytes.toBytes("name")));
byte[] family = Bytes.toBytes("gc");
gcResult.setMin(result.getValue(family, Bytes.toBytes("min")));
gcResult.setMax(result.getValue(family, Bytes.toBytes("max")));
gcResult.setTotalFragments(result.getValue(Bytes.toBytes("frag"), Bytes.toBytes("total")));
return gcResult;
}
return null;
}
public void map(ImmutableBytesWritable row, Result value, Context context)
throws InterruptedException, IOException {
/* BERLIN SPARQL BENHCMARK QUERY 11
----------------------------------------
SELECT ?property ?hasValue ?isValueOf
WHERE {
[TP-01] { %OfferXYZ% ?property ?hasValue }
UNION
[TP-02] { ?isValueOf ?property %OfferXYZ% }
}
---------------------------------------
*/
// TP-01
if (isPartOfFirstUnion(value)) {
List<KeyValue> entireRowAsList = value.list();
KeyValue[] kvsAsArray = new KeyValue[entireRowAsList.size()];
for (int i = 0; i < entireRowAsList.size(); i++) {
kvsAsArray[i] = entireRowAsList.get(i);
}
context.write(
new CompositeKeyWritable(new String(value.getRow()), 1),
new KeyValueArrayWritable(kvsAsArray));
return;
}
// TP-02
else {
List<KeyValue> entireRowAsList = value.list();
List<KeyValue> kvsToTransmit = new LinkedList<KeyValue>();
// Check all cells and see if the OFFER is part of the value
for (KeyValue kv : entireRowAsList) {
if (new String(kv.getValue()).equals(OfferXYZ)) {
kvsToTransmit.add(kv);
}
}
KeyValue[] kvsAsArray = new KeyValue[kvsToTransmit.size()];
for (int i = 0; i < kvsToTransmit.size(); i++) {
kvsAsArray[i] = kvsToTransmit.get(i);
}
if (kvsAsArray.length > 0) {
context.write(
new CompositeKeyWritable(new String(value.getRow()), 2),
new KeyValueArrayWritable(kvsAsArray));
} else {
return;
}
}
}
/**
* Get the field out of the row without checking whether parsing is needed. This is called by both
* getField and getFieldsAsList.
*
* @param fieldID The id of the field starting from 0.
* @return The value of the field
*/
private Object uncheckedGetField(int fieldID) {
LazyObjectBase[] fields = getFields();
boolean[] fieldsInited = getFieldInited();
if (!fieldsInited[fieldID]) {
fieldsInited[fieldID] = true;
ColumnMapping colMap = columnsMapping[fieldID];
if (!colMap.hbaseRowKey && !colMap.hbaseTimestamp && colMap.qualifierName == null) {
// it is a column family
// primitive type for Map<Key, Value> can be stored in binary format. Pass in the
// qualifier prefix to cherry pick the qualifiers that match the prefix instead of picking
// up everything
((LazyHBaseCellMap) fields[fieldID])
.init(
result,
colMap.familyNameBytes,
colMap.binaryStorage,
colMap.qualifierPrefixBytes,
colMap.isDoPrefixCut());
return fields[fieldID].getObject();
}
if (colMap.hbaseTimestamp) {
// Get the latest timestamp of all the cells as the row timestamp
long timestamp = result.rawCells()[0].getTimestamp(); // from hbase-0.96.0
for (int i = 1; i < result.rawCells().length; i++) {
timestamp = Math.max(timestamp, result.rawCells()[i].getTimestamp());
}
LazyObjectBase lz = fields[fieldID];
if (lz instanceof LazyTimestamp) {
((LazyTimestamp) lz).getWritableObject().setTime(timestamp);
} else {
((LazyLong) lz).getWritableObject().set(timestamp);
}
return lz.getObject();
}
byte[] bytes;
if (colMap.hbaseRowKey) {
bytes = result.getRow();
} else {
// it is a column i.e. a column-family with column-qualifier
bytes = result.getValue(colMap.familyNameBytes, colMap.qualifierNameBytes);
}
if (bytes == null || isNull(oi.getNullSequence(), bytes, 0, bytes.length)) {
fields[fieldID].setNull();
} else {
ByteArrayRef ref = new ByteArrayRef();
ref.setData(bytes);
fields[fieldID].init(ref, 0, bytes.length);
}
}
return fields[fieldID].getObject();
}
private void loadIPs() {
dns = new HashMap(100000000); // ���貢��
unknownHosts = new HashMap(1000000);
querying = new HashMap(100000);
try {
int statsCommit = 500000;
HConnection connection = HConnectionManager.createConnection(HBaseConfiguration.create());
HTableInterface fetchFailTable = connection.getTable("fetchFail");
Scan scan = new Scan();
scan.setCaching(statsCommit);
List<Filter> filters = new ArrayList<Filter>();
Filter filter = new ColumnPrefixFilter(Bytes.toBytes("ip"));
filters.add(filter);
FilterList filterList = new FilterList(filters);
scan.setFilter(filterList);
ResultScanner rs = fetchFailTable.getScanner(scan);
long cnt = 0;
for (Result r : rs) {
NavigableMap<byte[], byte[]> map = r.getFamilyMap(Bytes.toBytes("cf"));
String ip = Bytes.toString(map.get(Bytes.toBytes("ip")));
String host = Bytes.toString(r.getRow()).split("��")[0];
if (host != null && ip != null) {
dns.put(host, ip);
}
if (++cnt % statsCommit == 0) {
LOG.info("loadIPs url=" + Bytes.toString(r.getRow()) + " cnt=" + cnt);
}
}
rs.close();
fetchFailTable.close();
LOG.info("load hostip cache=" + dns.size());
connection.close();
} catch (Exception e) {
e.printStackTrace();
} finally {
//
}
}
private void printLocations(Result r) {
RegionLocations rl = null;
if (r == null) {
LOG.info("FAILED FOR null Result");
return;
}
LOG.info("FAILED FOR " + resultToString(r) + " Stale " + r.isStale());
if (r.getRow() == null) {
return;
}
try {
rl = ((ClusterConnection) connection).locateRegion(tableName, r.getRow(), true, true);
} catch (IOException e) {
LOG.warn("Couldn't get locations for row " + Bytes.toString(r.getRow()));
}
HRegionLocation locations[] = rl.getRegionLocations();
for (HRegionLocation h : locations) {
LOG.info("LOCATION " + h);
}
}
@Override
public Map<String, Set<String>> claimQueues(String regionserver) {
Map<String, Set<String>> queues = new HashMap<>();
if (isThisOurRegionServer(regionserver)) {
return queues;
}
ResultScanner queuesToClaim = null;
try {
queuesToClaim = getAllQueuesScanner(regionserver);
for (Result queue : queuesToClaim) {
if (attemptToClaimQueue(queue, regionserver)) {
String rowKey = Bytes.toString(queue.getRow());
ReplicationQueueInfo replicationQueueInfo = new ReplicationQueueInfo(rowKey);
if (replicationState.peerExists(replicationQueueInfo.getPeerId())) {
Set<String> sortedLogs = new HashSet<String>();
List<String> logs = getLogsInQueue(queue.getRow());
for (String log : logs) {
sortedLogs.add(log);
}
queues.put(rowKey, sortedLogs);
LOG.info(serverName + " has claimed queue " + rowKey + " from " + regionserver);
} else {
// Delete orphaned queues
removeQueue(Bytes.toString(queue.getRow()));
LOG.info(
serverName + " has deleted abandoned queue " + rowKey + " from " + regionserver);
}
}
}
} catch (IOException | KeeperException e) {
String errMsg = "Failed claiming queues for regionserver=" + regionserver;
abortable.abort(errMsg, e);
queues.clear();
} finally {
if (queuesToClaim != null) {
queuesToClaim.close();
}
}
return queues;
}
private MetaRecord getMetaRecord(Result r) {
Value[] data = new Value[4];
// id
// data[0] = ValueInt.get(Bytes.toInt(r.getValue(FAMILY, ID)));
data[0] = ValueInt.get(Bytes.toInt(r.getRow()));
// head 未使用
// data[1] = null;
// type
data[2] = ValueInt.get(Bytes.toInt(r.getValue(FAMILY, OBJECT_TYPE)));
// sql
data[3] = ValueString.get(Bytes.toString(r.getValue(FAMILY, SQL)));
return new MetaRecord(new SimpleRow(data));
}
protected void wipeOutMeta() throws IOException {
// Mess it up by blowing up meta.
Admin admin = TEST_UTIL.getHBaseAdmin();
Scan s = new Scan();
Table meta = TEST_UTIL.getConnection().getTable(TableName.META_TABLE_NAME);
ResultScanner scanner = meta.getScanner(s);
List<Delete> dels = new ArrayList<Delete>();
for (Result r : scanner) {
HRegionInfo info = HRegionInfo.getHRegionInfo(r);
if (info != null
&& !info.getTable()
.getNamespaceAsString()
.equals(NamespaceDescriptor.SYSTEM_NAMESPACE_NAME_STR)) {
Delete d = new Delete(r.getRow());
dels.add(d);
admin.unassign(r.getRow(), true);
}
}
meta.delete(dels);
scanner.close();
meta.close();
}
/*
* (non-Javadoc)
*
* @see com.hazelcast.core.MapLoader#loadAll(java.util.Collection)
*/
@Override
public Map<String, String> loadAll(Collection<String> keys) {
if (allowLoadAll) {
List<Get> gets = new ArrayList<Get>(keys.size());
for (String k : keys) {
Get g = new Get(Bytes.toBytes(k));
gets.add(g);
}
HTableInterface table = null;
Map<String, String> kvMap = new HashMap<String, String>();
try {
table = pool.getTable(tableName);
table.get(gets);
Result[] result = table.get(gets);
for (Result r : result) {
byte[] value = r.getValue(family, qualifier);
if (value != null) {
if (outputFormatType == StoreFormatType.SMILE) {
kvMap.put(
new String(r.getRow()),
jsonSmileConverter.convertFromSmile(r.getValue(family, qualifier)));
} else {
kvMap.put(new String(r.getRow()), new String(r.getValue(family, qualifier)));
}
}
}
} catch (IOException e) {
LOG.error("IOException while getting values", e);
} finally {
if (pool != null && table != null) {
pool.putTable(table);
}
}
}
return null;
}
