public List<String> getStationsNearPoint_Schema2(double lat, double lon) throws IOException {
Scan scan = new Scan();
scan.addFamily(BixiConstant.SCHEMA2_CLUSTER_FAMILY_NAME.getBytes());
InternalScanner scanner =
((RegionCoprocessorEnvironment) getEnvironment()).getRegion().getScanner(scan);
boolean hasMoreResult = false;
List<KeyValue> res = new ArrayList<KeyValue>();
List<String> result = new ArrayList<String>();
try {
do {
hasMoreResult = scanner.next(res);
for (KeyValue kv : res) {
String clusterId = Bytes.toString(kv.getRow());
String[] parts = clusterId.split(":");
double cLat = Double.parseDouble(parts[0]);
double cLon = Double.parseDouble(parts[1]);
double dx = Double.parseDouble(parts[2]);
double dy = Double.parseDouble(parts[3]);
double distx = lat - cLat;
double disty = lon - cLon;
if (distx >= 0 && distx <= dx && disty >= 0 && disty <= dy) {
// get stations in cluster
result.add(Bytes.toString(kv.getQualifier()));
}
}
res.clear();
} while (hasMoreResult);
} finally {
scanner.close();
}
return result;
}
public void verifyInvocationResults(Integer[] selectQualifiers, Integer[] expectedQualifiers)
throws Exception {
Get get = new Get(ROW_BYTES);
for (int i = 0; i < selectQualifiers.length; i++) {
get.addColumn(FAMILY_NAME_BYTES, Bytes.toBytes(QUALIFIER_PREFIX + selectQualifiers[i]));
}
get.setFilter(new InvocationRecordFilter());
List<KeyValue> expectedValues = new ArrayList<KeyValue>();
for (int i = 0; i < expectedQualifiers.length; i++) {
expectedValues.add(
new KeyValue(
ROW_BYTES,
FAMILY_NAME_BYTES,
Bytes.toBytes(QUALIFIER_PREFIX + expectedQualifiers[i]),
expectedQualifiers[i],
Bytes.toBytes(VALUE_PREFIX + expectedQualifiers[i])));
}
Scan scan = new Scan(get);
List<Cell> actualValues = new ArrayList<Cell>();
List<Cell> temp = new ArrayList<Cell>();
InternalScanner scanner = this.region.getScanner(scan);
while (scanner.next(temp)) {
actualValues.addAll(temp);
temp.clear();
}
actualValues.addAll(temp);
Assert.assertTrue(
"Actual values " + actualValues + " differ from the expected values:" + expectedValues,
expectedValues.equals(actualValues));
}
@Override
public Map<String, Integer> giveAvailableBikes(
long milliseconds, List<String> stationIds, Scan scan) throws IOException {
// scan has set the time stamp accordingly, i.e., the start and end row of
// the scan.
for (String qualifier : stationIds) {
log.debug("adding qualifier: " + qualifier);
scan.addColumn(colFamily, qualifier.getBytes());
}
InternalScanner scanner =
((RegionCoprocessorEnvironment) getEnvironment()).getRegion().getScanner(scan);
List<KeyValue> res = new ArrayList<KeyValue>();
Map<String, Integer> result = new HashMap<String, Integer>();
boolean hasMoreResult = false;
try {
do {
hasMoreResult = scanner.next(res);
for (KeyValue kv : res) {
// log.debug("got a kv: " + kv);
int availBikes = getFreeBikes(kv);
String id = Bytes.toString(kv.getQualifier());
// log.debug("result to be added is: " + availBikes + " id: " + id);
result.put(id, availBikes);
}
res.clear();
} while (hasMoreResult);
} finally {
scanner.close();
}
return result;
}
@Override
public Map<String, TotalNum> getTotalUsage_Schema2(Scan scan) throws IOException {
// System.err.println("scanning");
scan.addFamily(colFamilyStat);
InternalScanner scanner =
((RegionCoprocessorEnvironment) getEnvironment()).getRegion().getScanner(scan);
List<KeyValue> res = new ArrayList<KeyValue>();
Map<String, TotalNum> result = new HashMap<String, TotalNum>();
boolean hasMoreResult = false;
try {
do {
hasMoreResult = scanner.next(res);
for (KeyValue kv : res) {
String stationId = Bytes.toString(kv.getRow()).split("-")[1];
String value = new String(kv.getValue());
Long usage = Long.parseLong(value.split(";")[1]);
if (result.containsKey(stationId)) {
TotalNum tn = result.get(stationId);
tn.add(usage);
result.put(stationId, tn);
} else {
TotalNum tn = new TotalNum();
tn.add(usage);
result.put(stationId, tn);
}
}
res.clear();
} while (hasMoreResult);
} finally {
scanner.close();
}
return result;
}
@Override
public Map<String, Integer> getAvailableBikesFromAPoint_Schema2(Scan scan) throws IOException {
scan.addFamily(colFamilyStat);
InternalScanner scanner =
((RegionCoprocessorEnvironment) getEnvironment()).getRegion().getScanner(scan);
Map<String, Integer> result = new HashMap<String, Integer>();
boolean hasMoreResult = false;
List<KeyValue> res = new ArrayList<KeyValue>();
try {
do {
hasMoreResult = scanner.next(res);
for (KeyValue kv : res) {
String stationId = Bytes.toString(kv.getRow()).split("-")[1];
String value = new String(kv.getValue());
Integer free = Integer.parseInt(value.split(";")[0]);
result.put(stationId, free);
/*if(result.containsKey(stationId)){
result.put(stationId, free + result.get(stationId));
}else{
result.put(stationId, free);
}*/
}
res.clear();
} while (hasMoreResult);
} finally {
scanner.close();
}
return result;
}
/**
* Performs the compaction.
*
* @param scanner Where to read from.
* @param writer Where to write to.
* @param smallestReadPoint Smallest read point.
* @param cleanSeqId When true, remove seqId(used to be mvcc) value which is <= smallestReadPoint
* @return Whether compaction ended; false if it was interrupted for some reason.
*/
protected boolean performCompaction(
InternalScanner scanner, CellSink writer, long smallestReadPoint, boolean cleanSeqId)
throws IOException {
int bytesWritten = 0;
// Since scanner.next() can return 'false' but still be delivering data,
// we have to use a do/while loop.
List<Cell> kvs = new ArrayList<Cell>();
int closeCheckInterval = HStore.getCloseCheckInterval();
long lastMillis;
if (LOG.isDebugEnabled()) {
lastMillis = System.currentTimeMillis();
} else {
lastMillis = 0;
}
boolean hasMore;
do {
hasMore = scanner.next(kvs, compactionKVMax);
// output to writer:
for (Cell c : kvs) {
KeyValue kv = KeyValueUtil.ensureKeyValue(c);
if (cleanSeqId && kv.getSequenceId() <= smallestReadPoint) {
kv.setSequenceId(0);
}
writer.append(kv);
++progress.currentCompactedKVs;
progress.totalCompactedSize += kv.getLength();
// check periodically to see if a system stop is requested
if (closeCheckInterval > 0) {
bytesWritten += kv.getLength();
if (bytesWritten > closeCheckInterval) {
// Log the progress of long running compactions every minute if
// logging at DEBUG level
if (LOG.isDebugEnabled()) {
long now = System.currentTimeMillis();
if ((now - lastMillis) >= 60 * 1000) {
LOG.debug(
"Compaction progress: "
+ progress
+ String.format(
", rate=%.2f kB/sec",
(bytesWritten / 1024.0) / ((now - lastMillis) / 1000.0)));
lastMillis = now;
}
}
bytesWritten = 0;
if (!store.areWritesEnabled()) {
progress.cancel();
return false;
}
}
}
}
kvs.clear();
} while (hasMore);
progress.complete();
return true;
}
public static void doScan(HRegion region, Scan scan, List<Cell> result) throws IOException {
InternalScanner scanner = null;
try {
scan.setIsolationLevel(IsolationLevel.READ_UNCOMMITTED);
scanner = region.getScanner(scan);
result.clear();
scanner.next(result);
} finally {
if (scanner != null) scanner.close();
}
}
public Result get(final long transactionId, final Get get) throws IOException {
Scan scan = new Scan(get);
List<KeyValue> results = new ArrayList<KeyValue>();
InternalScanner scanner = null;
try {
scanner = getScanner(transactionId, scan);
scanner.next(results);
} finally {
if (scanner != null) {
scanner.close();
}
}
return new Result(results);
}
OfflineMerger(Configuration conf, FileSystem fs) throws IOException {
super(conf, fs, HConstants.META_TABLE_NAME);
Path rootTableDir =
HTableDescriptor.getTableDir(
fs.makeQualified(new Path(conf.get(HConstants.HBASE_DIR))),
HConstants.ROOT_TABLE_NAME);
// Scan root region to find all the meta regions
root =
HRegion.newHRegion(
rootTableDir,
hlog,
fs,
conf,
HRegionInfo.ROOT_REGIONINFO,
HTableDescriptor.ROOT_TABLEDESC,
null);
root.initialize();
Scan scan = new Scan();
scan.addColumn(HConstants.CATALOG_FAMILY, HConstants.REGIONINFO_QUALIFIER);
InternalScanner rootScanner = root.getScanner(scan);
try {
List<KeyValue> results = new ArrayList<KeyValue>();
boolean hasMore;
do {
hasMore = rootScanner.next(results);
for (KeyValue kv : results) {
HRegionInfo info = Writables.getHRegionInfoOrNull(kv.getValue());
if (info != null) {
metaRegions.add(info);
}
}
} while (hasMore);
} finally {
rootScanner.close();
try {
root.close();
} catch (IOException e) {
LOG.error(e);
}
}
}
@Override
public Map<String, TotalNum> giveTotalUsage(List<String> stationIds, Scan scan)
throws IOException {
for (String qualifier : stationIds) {
log.debug("adding qualifier: " + qualifier);
String colName = Integer.toString(Integer.parseInt(qualifier));
scan.addColumn(colFamily, colName.getBytes());
}
InternalScanner scanner =
((RegionCoprocessorEnvironment) getEnvironment()).getRegion().getScanner(scan);
List<KeyValue> res = new ArrayList<KeyValue>();
Map<String, TotalNum> result = new HashMap<String, TotalNum>();
boolean hasMoreResult = false;
try {
do {
hasMoreResult = scanner.next(res);
for (KeyValue kv : res) {
// log.debug("got a kv: " + kv);
long emptyDocks = getEmptyDocks(kv);
String id = Bytes.toString(kv.getQualifier());
TotalNum tn;
if (result.containsKey(id)) {
tn = result.get(id);
} else {
tn = new TotalNum();
}
tn.add(emptyDocks);
// emptyDocks = emptyDocks + (prevVal != null ? prevVal.intValue() : 0);
// log.debug("result to be added is: " + emptyDocks + " id: " + id);
result.put(id, tn);
}
res.clear();
} while (hasMoreResult);
} finally {
scanner.close();
}
return result;
}
/**
* Asynchronous HBase scan read as RAW qualifier
*
* @param scan
* @param listener
* @throws Exception
*/
protected InternalReadReport asyncStreamRead(
EntityDefinition ed, Scan scan, QualifierCreationListener listener) throws IOException {
// _init();
long counter = 0;
long startTimestamp = 0;
long stopTimestamp = 0;
InternalScanner scanner = this.getCurrentRegion().getScanner(scan);
List<Cell> results = new ArrayList<Cell>();
try {
boolean hasMoreRows; // false by default
do {
hasMoreRows = scanner.next(results);
Map<String, byte[]> kvMap = new HashMap<String, byte[]>();
if (!results.isEmpty()) {
counter++;
byte[] row = results.get(0).getRow();
// if(ed.isTimeSeries()){
long timestamp = RowkeyBuilder.getTimestamp(row, ed);
// Min
if (startTimestamp == 0 || startTimestamp > timestamp) {
startTimestamp = timestamp;
}
// Max
if (stopTimestamp == 0 || stopTimestamp < timestamp) {
stopTimestamp = timestamp;
}
// }
for (Cell kv : results) {
String qualifierName = Bytes.toString(kv.getQualifier());
Qualifier qualifier = null;
if (!ed.isTag(qualifierName)) {
qualifier = ed.getQualifierNameMap().get(qualifierName);
if (qualifier == null) {
LOG.error("qualifier for field " + qualifierName + " not exist");
throw new IOException(
new NullPointerException("qualifier for field " + qualifierName + " is null"));
}
qualifierName = qualifier.getDisplayName();
}
if (kv.getValue() != null) kvMap.put(qualifierName, kv.getValue());
}
// LOG.info("DEBUG: timestamp="+timestamp+",
// keys=["+StringUtils.join(kvMap.keySet(),",")+"]");
if (!kvMap.isEmpty()) listener.qualifierCreated(kvMap);
results.clear();
} else {
if (LOG.isDebugEnabled()) LOG.warn("Empty batch of KeyValue");
}
} while (hasMoreRows);
} catch (IOException ex) {
LOG.error(ex.getMessage(), ex);
throw ex;
} finally {
if (scanner != null) {
scanner.close();
}
}
return new InternalReadReport(counter, startTimestamp, stopTimestamp);
}
/**
* Asynchronous HBase scan read as entity
*
* @param scan
* @throws java.io.IOException
*/
protected InternalReadReport asyncStreamRead(
EntityDefinition ed, Scan scan, EntityCreationListener listener) throws IOException {
// _init();
long counter = 0;
long startTimestamp = 0;
long stopTimestamp = 0;
InternalScanner scanner = this.getCurrentRegion().getScanner(scan);
List<Cell> results = new ArrayList<Cell>();
try {
boolean hasMoreRows;
GenericMetricShadowEntity singleMetricEntity = null;
do {
hasMoreRows = scanner.next(results);
Map<String, byte[]> kvMap = new HashMap<String, byte[]>();
if (!results.isEmpty()) {
counter++;
byte[] row = results.get(0).getRow();
long timestamp = RowkeyBuilder.getTimestamp(row, ed);
// Min
if (startTimestamp == 0 || startTimestamp > timestamp) {
startTimestamp = timestamp;
}
// Max
if (stopTimestamp == 0 || stopTimestamp < timestamp) {
stopTimestamp = timestamp;
}
for (Cell kv : results) {
String qualifierName = Bytes.toString(kv.getQualifier());
// Qualifier qualifier = null;
// if(!ed.isTag(qualifierName)){
// qualifier = ed.getQualifierNameMap().get(qualifierName);
// if(qualifier == null){
// LOG.error("qualifier for " + qualifierName + " not exist");
// throw new NullPointerException("qualifier for field "+qualifierName+" not
// exist");
// }
// }
if (kv.getValue() != null) kvMap.put(qualifierName, kv.getValue());
}
// LOG.info("DEBUG: timestamp="+timestamp+",
// keys=["+StringUtils.join(kvMap.keySet(),",")+"]");
InternalLog internalLog = HBaseInternalLogHelper.buildObject(ed, row, timestamp, kvMap);
if (internalLog != null) {
TaggedLogAPIEntity logAPIEntity = null;
try {
logAPIEntity = HBaseInternalLogHelper.buildEntity(internalLog, ed);
if (logAPIEntity instanceof GenericMetricEntity) {
if (singleMetricEntity == null)
singleMetricEntity = new GenericMetricShadowEntity();
GenericMetricEntity e = (GenericMetricEntity) logAPIEntity;
if (e.getValue() != null) {
int count = e.getValue().length;
@SuppressWarnings("unused")
Class<?> cls = ed.getMetricDefinition().getSingleTimestampEntityClass();
for (int i = 0; i < count; i++) {
long ts =
logAPIEntity.getTimestamp() + i * ed.getMetricDefinition().getInterval();
// exclude those entity which is not within the time range in search condition.
// [start, end)
singleMetricEntity.setTimestamp(ts);
singleMetricEntity.setTags(e.getTags());
singleMetricEntity.setValue(e.getValue()[i]);
// Min
if (startTimestamp == 0 || startTimestamp > ts) startTimestamp = ts;
// Max
if (stopTimestamp == 0 || stopTimestamp < ts) stopTimestamp = ts;
listener.entityCreated(singleMetricEntity);
}
}
} else {
// LOG.info("DEBUG: rowKey="+logAPIEntity.getEncodedRowkey());
listener.entityCreated(logAPIEntity);
}
} catch (Exception e) {
if (internalLog != null) {
LOG.error(
"Got exception to handle " + internalLog.toString() + ": " + e.getMessage(), e);
}
throw new IOException(e);
}
} else {
LOG.error(
"Got null to parse internal log for row: " + row.length + " with fields: " + kvMap);
}
results.clear();
} else {
if (LOG.isDebugEnabled()) LOG.warn("Empty batch of KeyValue");
}
} while (hasMoreRows);
} catch (IOException ex) {
LOG.error(ex.getMessage(), ex);
throw ex;
} finally {
if (scanner != null) {
scanner.close();
}
}
return new InternalReadReport(counter, startTimestamp, stopTimestamp);
}
@SuppressWarnings("unchecked")
public <T extends Number, S extends Number> Map<BytesWritable, GroupByStatsValues<T, S>> getStats(
Scan scan, List<byte[][]> groupByTuples, byte[][] statsTuple, ColumnInterpreter<T, S> statsCI)
throws IOException {
long bt = System.currentTimeMillis();
long et;
long nt = 0, ntemp;
long octime = 0;
long nrecords = 0;
double ntmillis, ocmillis;
log.debug("getStats: start.");
byte[] sfamily = statsTuple[0];
byte[] squalifier = statsTuple[1];
List<KeyValue> results = new ArrayList<KeyValue>();
Map<Writable, Writable> facets = new MapWritable();
ntemp = System.nanoTime();
InternalScanner scanner =
((RegionCoprocessorEnvironment) getEnvironment()).getRegion().getScanner(scan);
octime += System.nanoTime() - ntemp;
try {
KeyValue statsFieldKV;
boolean hasMoreRows = false;
do {
ntemp = System.nanoTime();
hasMoreRows = scanner.next(results);
nt += System.nanoTime() - ntemp;
++nrecords;
ByteArrayOutputStream gbkey = new ByteArrayOutputStream(128);
BytesWritable gbkeyt;
int countOfKVs = 0;
statsFieldKV = null;
for (byte[][] gbtuple : groupByTuples) {
byte[] gbfamily = gbtuple[0];
byte[] gbqualifier = gbtuple[1];
for (KeyValue kv : results) {
if (Bytes.equals(kv.getFamily(), gbfamily)
&& Bytes.equals(kv.getQualifier(), gbqualifier)) {
if (countOfKVs > 0) {
gbkey.write(GROUP_BY_SEPARATOR);
}
gbkey.write(kv.getValue());
++countOfKVs;
}
if (Bytes.equals(kv.getFamily(), sfamily)
&& Bytes.equals(kv.getQualifier(), squalifier)) {
statsFieldKV = kv;
}
}
}
gbkeyt = new BytesWritable(gbkey.toByteArray());
if (countOfKVs == groupByTuples.size()) {
if (statsFieldKV != null) {
T statsFieldValue = statsCI.getValue(sfamily, squalifier, statsFieldKV);
if (facets.containsKey(gbkeyt)) {
GroupByStatsValues<T, S> svs = (GroupByStatsValues<T, S>) facets.get(gbkeyt);
svs.accumulate(statsFieldValue);
} else {
GroupByStatsValues<T, S> svs = new GroupByStatsValues<T, S>(statsCI);
svs.accumulate(statsFieldValue);
facets.put(gbkeyt, svs);
}
} else {
if (facets.containsKey(gbkeyt)) {
GroupByStatsValues<T, S> svs = (GroupByStatsValues<T, S>) facets.get(gbkeyt);
svs.addMissing(1);
} else {
GroupByStatsValues<T, S> svs = new GroupByStatsValues<T, S>(statsCI);
svs.addMissing(1);
facets.put(gbkeyt, svs);
}
}
}
results.clear();
} while (hasMoreRows);
} finally {
ntemp = System.nanoTime();
scanner.close();
octime += System.nanoTime() - ntemp;
}
Map<? extends Writable, ? extends Writable> derefmap = facets;
Map<BytesWritable, GroupByStatsValues<T, S>> retmap =
(Map<BytesWritable, GroupByStatsValues<T, S>>) derefmap;
et = System.currentTimeMillis();
ntmillis = nt / 1000000.0;
ocmillis = octime / 1000000.0;
log.debug("getStats(): InternalScanner next took " + ntmillis + " ms.");
log.debug("getStats(): InternalScanner open/close tool " + ocmillis + " ms.");
log.debug("getStats(): aggregations took " + ((et - bt) - ntmillis) + " ms.");
log.debug("getStats(): number of values aggregated is " + nrecords);
log.debug(
"getStats(): constructed GroupByImplementation HashMap of size " + retmap.size() + ".");
log.debug("getStats(): finished at " + (et - bt) + " ms.");
return retmap;
}
@Override
public boolean next(List<Cell> result, ScannerContext scannerContext) throws IOException {
boolean ret = delegate.next(result, scannerContext);
updateStat(result);
return ret;
}