/** {@inheritDoc} */
@Override
public void write(HFileKeyValue entry, NullWritable unused) throws IOException {
final KeyValue kv = entry.getKeyValue();
kv.updateLatestStamp(mLatestTimestampBytes);
final long recordLength = kv.getLength();
if (mCurrentHFileSize + recordLength >= mMaxFileSizeBytes) {
// We can't fit this record in the current HFile without exceeding the max file size.
if (Arrays.equals(mCurrentRow, kv.getRow())) {
// But we're still adding data for a single row, so we can't close this HFile yet.
LOG.debug("Reached max HFile size, but waiting to finish this row before closing.");
} else {
// Close it and open a new one.
closeWriter(mWriter);
mWriter = openNewWriter();
}
}
mWriter.append(kv);
mTimeRangeTracker.includeTimestamp(kv);
mCurrentHFileSize += recordLength;
// Remember the row so we know when we are transitioning.
mCurrentRow = kv.getRow();
}
/**
* Transactional version of {@link HTable#put(Put)}
*
* @param transactionState Identifier of the transaction
* @see HTable#put(Put)
* @throws IOException
*/
public void put(TransactionState transactionState, Put put)
throws IOException, IllegalArgumentException {
final long startTimestamp = transactionState.getStartTimestamp();
// byte[] startTSBytes = Bytes.toBytes(startTimestamp);
// create put with correct ts
final Put tsput = new Put(put.getRow(), startTimestamp);
Map<byte[], List<KeyValue>> kvs = put.getFamilyMap();
for (List<KeyValue> kvl : kvs.values()) {
for (KeyValue kv : kvl) {
// int tsOffset = kv.getTimestampOffset();
// System.arraycopy(startTSBytes, 0, kv.getBuffer(), tsOffset,
// Bytes.SIZEOF_LONG);
tsput.add(
new KeyValue(
kv.getRow(), kv.getFamily(), kv.getQualifier(), startTimestamp, kv.getValue()));
}
}
// should add the table as well
transactionState.addRow(new RowKeyFamily(put.getRow(), getTableName(), put.getFamilyMap()));
put(tsput);
// super.getConnection().getRegionServerWithRetries(
// new ServerCallable<Boolean>(super.getConnection(), super.getTableName(),
// put.getRow()) {
// public Boolean call() throws IOException {
// server.put(location.getRegionInfo().getRegionName(), tsput);
// return true;
// }
// });
}
@Override
public Map<String, TotalNum> getTotalUsage_Schema2(Scan scan) throws IOException {
// System.err.println("scanning");
scan.addFamily(colFamilyStat);
InternalScanner scanner =
((RegionCoprocessorEnvironment) getEnvironment()).getRegion().getScanner(scan);
List<KeyValue> res = new ArrayList<KeyValue>();
Map<String, TotalNum> result = new HashMap<String, TotalNum>();
boolean hasMoreResult = false;
try {
do {
hasMoreResult = scanner.next(res);
for (KeyValue kv : res) {
String stationId = Bytes.toString(kv.getRow()).split("-")[1];
String value = new String(kv.getValue());
Long usage = Long.parseLong(value.split(";")[1]);
if (result.containsKey(stationId)) {
TotalNum tn = result.get(stationId);
tn.add(usage);
result.put(stationId, tn);
} else {
TotalNum tn = new TotalNum();
tn.add(usage);
result.put(stationId, tn);
}
}
res.clear();
} while (hasMoreResult);
} finally {
scanner.close();
}
return result;
}
public List<String> getStationsNearPoint_Schema2(double lat, double lon) throws IOException {
Scan scan = new Scan();
scan.addFamily(BixiConstant.SCHEMA2_CLUSTER_FAMILY_NAME.getBytes());
InternalScanner scanner =
((RegionCoprocessorEnvironment) getEnvironment()).getRegion().getScanner(scan);
boolean hasMoreResult = false;
List<KeyValue> res = new ArrayList<KeyValue>();
List<String> result = new ArrayList<String>();
try {
do {
hasMoreResult = scanner.next(res);
for (KeyValue kv : res) {
String clusterId = Bytes.toString(kv.getRow());
String[] parts = clusterId.split(":");
double cLat = Double.parseDouble(parts[0]);
double cLon = Double.parseDouble(parts[1]);
double dx = Double.parseDouble(parts[2]);
double dy = Double.parseDouble(parts[3]);
double distx = lat - cLat;
double disty = lon - cLon;
if (distx >= 0 && distx <= dx && disty >= 0 && disty <= dy) {
// get stations in cluster
result.add(Bytes.toString(kv.getQualifier()));
}
}
res.clear();
} while (hasMoreResult);
} finally {
scanner.close();
}
return result;
}
@Override
public Map<String, Integer> getAvailableBikesFromAPoint_Schema2(Scan scan) throws IOException {
scan.addFamily(colFamilyStat);
InternalScanner scanner =
((RegionCoprocessorEnvironment) getEnvironment()).getRegion().getScanner(scan);
Map<String, Integer> result = new HashMap<String, Integer>();
boolean hasMoreResult = false;
List<KeyValue> res = new ArrayList<KeyValue>();
try {
do {
hasMoreResult = scanner.next(res);
for (KeyValue kv : res) {
String stationId = Bytes.toString(kv.getRow()).split("-")[1];
String value = new String(kv.getValue());
Integer free = Integer.parseInt(value.split(";")[0]);
result.put(stationId, free);
/*if(result.containsKey(stationId)){
result.put(stationId, free + result.get(stationId));
}else{
result.put(stationId, free);
}*/
}
res.clear();
} while (hasMoreResult);
} finally {
scanner.close();
}
return result;
}
@GET
@Produces(MIMETYPE_BINARY)
public Response getBinary(final @Context UriInfo uriInfo) {
if (LOG.isDebugEnabled()) {
LOG.debug("GET " + uriInfo.getAbsolutePath() + " as " + MIMETYPE_BINARY);
}
servlet.getMetrics().incrementRequests(1);
try {
KeyValue value = generator.next();
if (value == null) {
LOG.info("generator exhausted");
return Response.noContent().build();
}
ResponseBuilder response = Response.ok(value.getValue());
response.cacheControl(cacheControl);
response.header("X-Row", Base64.encodeBytes(value.getRow()));
response.header(
"X-Column",
Base64.encodeBytes(KeyValue.makeColumn(value.getFamily(), value.getQualifier())));
response.header("X-Timestamp", value.getTimestamp());
return response.build();
} catch (IllegalStateException e) {
ScannerResource.delete(id);
throw new WebApplicationException(Response.Status.GONE);
}
}
public static HBaseTuple getHBaseTuple(KeyValue kv) {
HBaseTuple tuple = new HBaseTuple();
tuple.setRow(new String(kv.getRow()));
tuple.setColFamily(new String(kv.getFamily()));
tuple.setColName(new String(kv.getQualifier()));
tuple.setColValue(new String(kv.getValue()));
return tuple;
}
/**
* Sets the row this instance holds in RAM using a row from a scanner.
*
* @param row The compacted HBase row to set.
* @throws IllegalStateException if this method was already called.
*/
void setRow(final KeyValue row) {
if (this.key != null) {
throw new IllegalStateException("setRow was already called on " + this);
}
this.key = row.getRow();
this.qualifiers = row.getQualifier();
this.values = row.getValue();
}
private static void verifyData(
KeyValue kv, String expectedRow, String expectedCol, long expectedVersion) {
assertEquals("RowCheck", expectedRow, Bytes.toString(kv.getRow()));
assertEquals("ColumnCheck", expectedCol, Bytes.toString(kv.getQualifier()));
assertEquals("TSCheck", expectedVersion, kv.getTimestamp());
assertEquals(
"ValueCheck",
Bytes.toString(genValue(expectedRow, expectedCol, expectedVersion)),
Bytes.toString(kv.getValue()));
}
public boolean isDeleted(KeyValue kv) {
Long timestamp;
timestamp = deletedRows.get(new ByteArray(kv.getRow()));
if (timestamp != null && kv.getTimestamp() < timestamp) return true;
timestamp = deletedFamilies.get(new ByteArray(kv.getFamily()));
if (timestamp != null && kv.getTimestamp() < timestamp) return true;
timestamp = deletedColumns.get(new ByteArray(Bytes.add(kv.getFamily(), kv.getQualifier())));
if (timestamp != null && kv.getTimestamp() < timestamp) return true;
return false;
}
protected void doReconstructionLog(
final Path oldCoreLogFile,
final long minSeqId,
final long maxSeqId,
final Progressable reporter)
throws UnsupportedEncodingException, IOException {
Path trxPath = new Path(oldCoreLogFile.getParent(), THLog.HREGION_OLD_THLOGFILE_NAME);
// We can ignore doing anything with the Trx Log table, it is
// not-transactional.
if (super.getTableDesc().getNameAsString().equals(HBaseBackedTransactionLogger.TABLE_NAME)) {
return;
}
THLogRecoveryManager recoveryManager = new THLogRecoveryManager(this);
Map<Long, WALEdit> commitedTransactionsById =
recoveryManager.getCommitsFromLog(trxPath, minSeqId, reporter);
if (commitedTransactionsById != null && commitedTransactionsById.size() > 0) {
LOG.debug("found " + commitedTransactionsById.size() + " COMMITED transactions to recover.");
for (Entry<Long, WALEdit> entry : commitedTransactionsById.entrySet()) {
LOG.debug(
"Writing " + entry.getValue().size() + " updates for transaction " + entry.getKey());
WALEdit b = entry.getValue();
for (KeyValue kv : b.getKeyValues()) {
// FIXME need to convert these into puts and deletes. Not sure this is
// the write way.
// Could probably combine multiple KV's into single put/delete.
// Also timestamps?
if (kv.getType() == KeyValue.Type.Put.getCode()) {
Put put = new Put();
put.add(kv);
super.put(put);
} else if (kv.isDelete()) {
Delete del = new Delete(kv.getRow());
if (kv.isDeleteFamily()) {
del.deleteFamily(kv.getFamily());
} else if (kv.isDeleteType()) {
del.deleteColumn(kv.getFamily(), kv.getQualifier());
}
}
}
}
LOG.debug("Flushing cache"); // We must trigger a cache flush,
// otherwise we will would ignore the log on subsequent failure
if (!super.flushcache()) {
LOG.warn("Did not flush cache");
}
}
}
/**
* Maps the data.
*
* @param row The current table row key.
* @param values The columns.
* @param context The current context.
* @throws IOException When something is broken with the data.
* @see org.apache.hadoop.mapreduce.Mapper#map(KEYIN, VALUEIN,
* org.apache.hadoop.mapreduce.Mapper.Context)
*/
@Override
public void map(ImmutableBytesWritable row, Result values, Context context) throws IOException {
String currentFamilyName = null;
String currentQualifierName = null;
String currentRowKey = null;
Configuration config = context.getConfiguration();
String separator = config.get("ReportSeparator", ":");
try {
if (values != null) {
context.getCounter(Counters.ROWS).increment(1);
context.write(new Text("Total ROWS"), new IntWritable(1));
}
for (KeyValue value : values.list()) {
currentRowKey = Bytes.toStringBinary(value.getRow());
String thisRowFamilyName = Bytes.toStringBinary(value.getFamily());
if (thisRowFamilyName != null && !thisRowFamilyName.equals(currentFamilyName)) {
currentFamilyName = thisRowFamilyName;
context.getCounter("CF", thisRowFamilyName).increment(1);
context.write(new Text("Total Families Across all Rows"), new IntWritable(1));
context.write(new Text(thisRowFamilyName), new IntWritable(1));
}
String thisRowQualifierName =
thisRowFamilyName + separator + Bytes.toStringBinary(value.getQualifier());
if (thisRowQualifierName != null && !thisRowQualifierName.equals(currentQualifierName)) {
currentQualifierName = thisRowQualifierName;
context.getCounter("CFQL", thisRowQualifierName).increment(1);
context.write(new Text("Total Qualifiers across all Rows"), new IntWritable(1));
context.write(new Text(thisRowQualifierName), new IntWritable(1));
// Intialize versions
context
.getCounter("QL_VERSIONS", currentRowKey + separator + thisRowQualifierName)
.increment(1);
context.write(
new Text(currentRowKey + separator + thisRowQualifierName + "_Versions"),
new IntWritable(1));
} else {
// Increment versions
currentQualifierName = thisRowQualifierName;
context
.getCounter("QL_VERSIONS", currentRowKey + separator + thisRowQualifierName)
.increment(1);
context.write(
new Text(currentRowKey + separator + thisRowQualifierName + "_Versions"),
new IntWritable(1));
}
}
} catch (InterruptedException e) {
e.printStackTrace();
}
}
/**
* Test that {@link HFileOutputFormat} RecordWriter amends timestamps if passed a keyvalue whose
* timestamp is {@link HConstants#LATEST_TIMESTAMP}.
*
* @see <a href="https://issues.apache.org/jira/browse/HBASE-2615">HBASE-2615</a>
*/
@Test
public void test_LATEST_TIMESTAMP_isReplaced() throws Exception {
Configuration conf = new Configuration(this.util.getConfiguration());
RecordWriter<ImmutableBytesWritable, KeyValue> writer = null;
TaskAttemptContext context = null;
Path dir = util.getDataTestDir("test_LATEST_TIMESTAMP_isReplaced");
try {
Job job = new Job(conf);
FileOutputFormat.setOutputPath(job, dir);
context = createTestTaskAttemptContext(job);
HFileOutputFormat hof = new HFileOutputFormat();
writer = hof.getRecordWriter(context);
final byte[] b = Bytes.toBytes("b");
// Test 1. Pass a KV that has a ts of LATEST_TIMESTAMP. It should be
// changed by call to write. Check all in kv is same but ts.
KeyValue kv = new KeyValue(b, b, b);
KeyValue original = kv.clone();
writer.write(new ImmutableBytesWritable(), kv);
assertFalse(original.equals(kv));
assertTrue(Bytes.equals(original.getRow(), kv.getRow()));
assertTrue(CellUtil.matchingColumn(original, kv.getFamily(), kv.getQualifier()));
assertNotSame(original.getTimestamp(), kv.getTimestamp());
assertNotSame(HConstants.LATEST_TIMESTAMP, kv.getTimestamp());
// Test 2. Now test passing a kv that has explicit ts. It should not be
// changed by call to record write.
kv = new KeyValue(b, b, b, kv.getTimestamp() - 1, b);
original = kv.clone();
writer.write(new ImmutableBytesWritable(), kv);
assertTrue(original.equals(kv));
} finally {
if (writer != null && context != null) writer.close(context);
dir.getFileSystem(conf).delete(dir, true);
}
}
@Override
public void prePut(
final ObserverContext<RegionCoprocessorEnvironment> e,
final Put put,
final WALEdit edit,
final Durability durability)
throws IOException {
byte[] attribute = put.getAttribute("visibility");
byte[] cf = null;
List<Cell> updatedCells = new ArrayList<Cell>();
if (attribute != null) {
for (List<? extends Cell> edits : put.getFamilyCellMap().values()) {
for (Cell cell : edits) {
KeyValue kv = KeyValueUtil.ensureKeyValue(cell);
if (cf == null) {
cf = kv.getFamily();
}
Tag tag = new Tag(TAG_TYPE, attribute);
List<Tag> tagList = new ArrayList<Tag>();
tagList.add(tag);
KeyValue newKV =
new KeyValue(
kv.getRow(),
0,
kv.getRowLength(),
kv.getFamily(),
0,
kv.getFamilyLength(),
kv.getQualifier(),
0,
kv.getQualifierLength(),
kv.getTimestamp(),
KeyValue.Type.codeToType(kv.getType()),
kv.getValue(),
0,
kv.getValueLength(),
tagList);
((List<Cell>) updatedCells).add(newKV);
}
}
put.getFamilyCellMap().remove(cf);
// Update the family map
put.getFamilyCellMap().put(cf, updatedCells);
}
}
/*
* 遍历查询hbase表
*
* @tableName 表名
*/
public static void getResultScann(String tableName) throws IOException {
Scan scan = new Scan();
ResultScanner rs = null;
HTableInterface table = conn.getTable(tableName);
try {
// String split = StringUtils.S001;
// QualifierFilter ff = new QualifierFilter(CompareOp.EQUAL, new BinaryComparator(
// Bytes.toBytes("A")));
// scan.setFilter(ff);
rs = table.getScanner(scan);
int count = 0;
for (Result r : rs) {
count++;
for (KeyValue kv : r.list()) {
System.out.println("row:" + Bytes.toString(kv.getRow()));
// System.out.println("family:" + Bytes.toString(kv.getFamily()));
// System.out.println("qualifier:" + Bytes.toString(kv.getQualifier()));
System.out.println("value:" + Bytes.toString(kv.getValue()));
//
// System.out.println("timestamp:" + kv.getTimestamp());
// StringBuilder sb = new StringBuilder();
// sb.append(Bytes.toString(r.getRow()));
// sb.append(split);
// sb.append(Bytes.toString(kv.getValue()));
// EntBaseinfo baseInfo = new EntBaseinfo();
// baseInfo.parseFromString(sb.toString());
// System.out.println(baseInfo.getENTNAME());
// if(baseInfo.getNAME()!=null&&baseInfo.getNAME().isEmpty()){
// System.out.println(baseInfo.getENTNAME());
// }
//
//
// if(baseInfo.getDOM()!=null&&baseInfo.getNAME().isEmpty()){
// System.out.println(baseInfo.getENTNAME());
// }
}
if (count > 1000) {
return;
}
}
} finally {
rs.close();
}
}
/*
* 遍历查询hbase表
*
* @tableName 表名
*/
public static void getResultScann(String tableName) throws IOException {
Scan scan = new Scan();
ResultScanner rs = null;
HTable table = new HTable(conf, Bytes.toBytes(tableName));
try {
rs = table.getScanner(scan);
for (Result r : rs) {
for (KeyValue kv : r.list()) {
System.out.println("row:" + Bytes.toString(kv.getRow()));
System.out.println("family:" + Bytes.toString(kv.getFamily()));
System.out.println("qualifier:" + Bytes.toString(kv.getQualifier()));
System.out.println("value:" + Bytes.toString(kv.getValue()));
System.out.println("timestamp:" + kv.getTimestamp());
System.out.println("-------------------------------------------");
}
}
} finally {
rs.close();
}
}
/**
* Merges another HBase row into this one. When two continuous rows in HBase have data points that
* are close enough together that they could be stored into the same row, it makes sense to merge
* them into the same {@link RowSeq} instance in memory in order to save RAM.
*
* @param row The compacted HBase row to merge into this instance.
* @throws IllegalStateException if {@link #setRow} wasn't called first.
* @throws IllegalArgumentException if the data points in the argument aren't close enough to
* those in this instance time-wise to be all merged together.
*/
void addRow(final KeyValue row) {
if (this.key == null) {
throw new IllegalStateException("setRow was never called on " + this);
}
final byte[] key = row.getRow();
final long base_time = Bytes.getUnsignedInt(key, UniqueIds.metrics().width());
final int time_adj = (int) (base_time - baseTime());
if (time_adj <= 0) {
// Corner case: if the time difference is 0 and the key is the same, it
// means we've already added this row, possibly parts of it. This
// doesn't normally happen but can happen if the scanner we're using
// timed out (its lease expired for whatever reason), in which case
// asynchbase will transparently re-open the scanner and start scanning
// from the row key we were on at the time the timeout happened. In
// that case, the easiest thing to do is to discard everything we know
// about this row and start over, since we're going to get the full row
// again anyway.
if (time_adj != 0 || !Bytes.equals(this.key, key)) {
throw new IllegalDataException(
"Attempt to add a row with a base_time="
+ base_time
+ " <= baseTime()="
+ baseTime()
+ "; Row added="
+ row
+ ", this="
+ this);
}
this.key = null; // To keep setRow happy.
this.qualifiers = null; // Throw away our previous work.
this.values = null; // free();
setRow(row);
return;
}
final byte[] qual = row.getQualifier();
final int len = qual.length;
int last_delta = Bytes.getUnsignedShort(qualifiers, qualifiers.length - 2);
last_delta >>= Const.FLAG_BITS;
final int old_qual_len = qualifiers.length;
final byte[] newquals = new byte[old_qual_len + len];
System.arraycopy(qualifiers, 0, newquals, 0, old_qual_len);
// Adjust the delta in all the qualifiers.
for (int i = 0; i < len; i += 2) {
short qualifier = Bytes.getShort(qual, i);
final int time_delta = time_adj + ((qualifier & 0xFFFF) >>> Const.FLAG_BITS);
if (!canTimeDeltaFit(time_delta)) {
throw new IllegalDataException(
"time_delta at index "
+ i
+ " is too large: "
+ time_delta
+ " (qualifier=0x"
+ Integer.toHexString(qualifier & 0xFFFF)
+ " baseTime()="
+ baseTime()
+ ", base_time="
+ base_time
+ ", time_adj="
+ time_adj
+ ") for "
+ row
+ " to be added to "
+ this);
}
if (last_delta >= time_delta) {
LOG.error(
"new timestamp = "
+ (baseTime() + time_delta)
+ " (index="
+ i
+ ") is < previous="
+ (baseTime() + last_delta)
+ " in addRow with row="
+ row
+ " in this="
+ this);
return; // Ignore this row, it came out of order.
}
qualifier = (short) ((time_delta << Const.FLAG_BITS) | (qualifier & Const.FLAGS_MASK));
Bytes.setShort(newquals, qualifier, old_qual_len + i);
}
this.qualifiers = newquals;
final byte[] val = row.getValue();
// If both the current `values' and the new `val' are single values, then
// we neither of them has a meta data byte so we need to add one to be
// consistent with what we expect from compacted values. Otherwise, we
// need to subtract 1 from the value length.
final int old_val_len = values.length - (old_qual_len == 2 ? 0 : 1);
final byte[] newvals =
new byte
[old_val_len
+ val.length
// Only add a meta-data byte if the new values don't have it.
+ (len == 2 ? 1 : 0)];
System.arraycopy(values, 0, newvals, 0, old_val_len);
System.arraycopy(val, 0, newvals, old_val_len, val.length);
assert newvals[newvals.length - 1] == 0
: "Incorrect meta data byte after merge of "
+ row
+ " resulting qualifiers="
+ Arrays.toString(qualifiers)
+ ", values="
+ Arrays.toString(newvals)
+ ", old values="
+ Arrays.toString(values);
this.values = newvals;
}
private Result filter(
TransactionState state, Result result, long startTimestamp, int localVersions)
throws IOException {
if (result == null) {
return null;
}
List<KeyValue> kvs = result.list();
if (kvs == null) {
return result;
}
Map<ByteArray, Map<ByteArray, Integer>> occurrences =
new HashMap<TransactionalTable.ByteArray, Map<ByteArray, Integer>>();
Map<ByteArray, Map<ByteArray, Long>> minTimestamp =
new HashMap<TransactionalTable.ByteArray, Map<ByteArray, Long>>();
List<KeyValue> nonDeletes = new ArrayList<KeyValue>();
List<KeyValue> filtered = new ArrayList<KeyValue>();
Map<ByteArray, Set<ByteArray>> read = new HashMap<ByteArray, Set<ByteArray>>();
DeleteTracker tracker = new DeleteTracker();
for (KeyValue kv : kvs) {
ByteArray family = new ByteArray(kv.getFamily());
ByteArray qualifier = new ByteArray(kv.getQualifier());
Set<ByteArray> readQualifiers = read.get(family);
if (readQualifiers == null) {
readQualifiers = new HashSet<TransactionalTable.ByteArray>();
read.put(family, readQualifiers);
} else if (readQualifiers.contains(qualifier)) continue;
// RowKey rk = new RowKey(kv.getRow(), getTableName());
if (state.tsoclient.validRead(kv.getTimestamp(), startTimestamp)) {
if (!tracker.addDeleted(kv)) nonDeletes.add(kv);
{
// Read valid value
readQualifiers.add(qualifier);
// statistics
// elementsGotten++;
Map<ByteArray, Integer> occurrencesCols = occurrences.get(family);
Integer times = null;
if (occurrencesCols != null) {
times = occurrencesCols.get(qualifier);
}
if (times != null) {
// elementsRead += times;
versionsAvg = times > versionsAvg ? times : alpha * versionsAvg + (1 - alpha) * times;
// extraVersionsAvg = times > extraVersionsAvg ? times : alpha *
// extraVersionsAvg + (1 - alpha) * times;
} else {
// elementsRead++;
versionsAvg = alpha * versionsAvg + (1 - alpha);
// extraVersionsAvg = alpha * extraVersionsAvg + (1 - alpha);
}
}
} else {
Map<ByteArray, Integer> occurrencesCols = occurrences.get(family);
Map<ByteArray, Long> minTimestampCols = minTimestamp.get(family);
if (occurrencesCols == null) {
occurrencesCols = new HashMap<TransactionalTable.ByteArray, Integer>();
minTimestampCols = new HashMap<TransactionalTable.ByteArray, Long>();
occurrences.put(family, occurrencesCols);
minTimestamp.put(family, minTimestampCols);
}
Integer times = occurrencesCols.get(qualifier);
Long timestamp = minTimestampCols.get(qualifier);
if (times == null) {
times = 0;
timestamp = kv.getTimestamp();
}
times++;
timestamp = Math.min(timestamp, kv.getTimestamp());
if (times == localVersions) {
// We need to fetch more versions
Get get = new Get(kv.getRow());
get.addColumn(kv.getFamily(), kv.getQualifier());
get.setMaxVersions(localVersions);
Result r;
GOTRESULT:
do {
extraGetsPerformed++;
get.setTimeRange(0, timestamp);
r = this.get(get);
List<KeyValue> list = r.list();
if (list == null) break;
for (KeyValue t : list) {
times++;
timestamp = Math.min(timestamp, t.getTimestamp());
// rk = new RowKey(kv.getRow(), getTableName());
if (state.tsoclient.validRead(t.getTimestamp(), startTimestamp)) {
if (!tracker.addDeleted(t)) nonDeletes.add(t);
readQualifiers.add(qualifier);
elementsGotten++;
elementsRead += times;
versionsAvg =
times > versionsAvg ? times : alpha * versionsAvg + (1 - alpha) * times;
extraVersionsAvg =
times > extraVersionsAvg
? times
: alpha * extraVersionsAvg + (1 - alpha) * times;
break GOTRESULT;
}
}
} while (r.size() == localVersions);
} else {
occurrencesCols.put(qualifier, times);
minTimestampCols.put(qualifier, timestamp);
}
}
}
for (KeyValue kv : nonDeletes) {
if (!tracker.isDeleted(kv)) {
filtered.add(kv);
}
}
// cacheVersions = (int) versionsAvg;
if (filtered.isEmpty()) {
return null;
}
return new Result(filtered);
}
/**
* Sets the row this instance holds in RAM using a row from a scanner.
*
* @param row The compacted HBase row to set.
* @throws IllegalStateException if this method was already called.
*/
public void clearAndSetRow(final KeyValue row) {
this.key = row.getRow();
this.qualifiers = row.getQualifier();
this.values = row.getValue();
}
public boolean runRandomReadWorkload() throws IOException {
if (inputFileNames.size() != 1) {
throw new IOException("Need exactly one input file for random reads: " + inputFileNames);
}
Path inputPath = new Path(inputFileNames.get(0));
// Make sure we are using caching.
StoreFile storeFile = openStoreFile(inputPath, true);
StoreFile.Reader reader = storeFile.createReader();
LOG.info("First key: " + Bytes.toStringBinary(reader.getFirstKey()));
LOG.info("Last key: " + Bytes.toStringBinary(reader.getLastKey()));
KeyValue firstKV = KeyValue.createKeyValueFromKey(reader.getFirstKey());
firstRow = firstKV.getRow();
KeyValue lastKV = KeyValue.createKeyValueFromKey(reader.getLastKey());
lastRow = lastKV.getRow();
byte[] family = firstKV.getFamily();
if (!Bytes.equals(family, lastKV.getFamily())) {
LOG.error(
"First and last key have different families: "
+ Bytes.toStringBinary(family)
+ " and "
+ Bytes.toStringBinary(lastKV.getFamily()));
return false;
}
if (Bytes.equals(firstRow, lastRow)) {
LOG.error(
"First and last row are the same, cannot run read workload: "
+ "firstRow="
+ Bytes.toStringBinary(firstRow)
+ ", "
+ "lastRow="
+ Bytes.toStringBinary(lastRow));
return false;
}
ExecutorService exec = Executors.newFixedThreadPool(numReadThreads + 1);
int numCompleted = 0;
int numFailed = 0;
try {
ExecutorCompletionService<Boolean> ecs = new ExecutorCompletionService<Boolean>(exec);
endTime = System.currentTimeMillis() + 1000 * durationSec;
boolean pread = true;
for (int i = 0; i < numReadThreads; ++i) ecs.submit(new RandomReader(i, reader, pread));
ecs.submit(new StatisticsPrinter());
Future<Boolean> result;
while (true) {
try {
result = ecs.poll(endTime + 1000 - System.currentTimeMillis(), TimeUnit.MILLISECONDS);
if (result == null) break;
try {
if (result.get()) {
++numCompleted;
} else {
++numFailed;
}
} catch (ExecutionException e) {
LOG.error("Worker thread failure", e.getCause());
++numFailed;
}
} catch (InterruptedException ex) {
LOG.error("Interrupted after " + numCompleted + " workers completed");
Thread.currentThread().interrupt();
continue;
}
}
} finally {
storeFile.closeReader(true);
exec.shutdown();
BlockCache c = cacheConf.getBlockCache();
if (c != null) {
c.shutdown();
}
}
LOG.info("Worker threads completed: " + numCompleted);
LOG.info("Worker threads failed: " + numFailed);
return true;
}
@GET
@Produces({MIMETYPE_XML, MIMETYPE_JSON, MIMETYPE_PROTOBUF})
public Response get(
final @Context UriInfo uriInfo,
@QueryParam("n") int maxRows,
final @QueryParam("c") int maxValues) {
if (LOG.isDebugEnabled()) {
LOG.debug("GET " + uriInfo.getAbsolutePath());
}
servlet.getMetrics().incrementRequests(1);
CellSetModel model = new CellSetModel();
RowModel rowModel = null;
byte[] rowKey = null;
int limit = batch;
if (maxValues > 0) {
limit = maxValues;
}
int count = limit;
do {
KeyValue value = null;
try {
value = generator.next();
} catch (IllegalStateException e) {
ScannerResource.delete(id);
throw new WebApplicationException(Response.Status.GONE);
}
if (value == null) {
LOG.info("generator exhausted");
// respond with 204 (No Content) if an empty cell set would be
// returned
if (count == limit) {
return Response.noContent().build();
}
break;
}
if (rowKey == null) {
rowKey = value.getRow();
rowModel = new RowModel(rowKey);
}
if (!Bytes.equals(value.getRow(), rowKey)) {
// if maxRows was given as a query param, stop if we would exceed the
// specified number of rows
if (maxRows > 0) {
if (--maxRows == 0) {
generator.putBack(value);
break;
}
}
model.addRow(rowModel);
rowKey = value.getRow();
rowModel = new RowModel(rowKey);
}
rowModel.addCell(
new CellModel(
value.getFamily(), value.getQualifier(), value.getTimestamp(), value.getValue()));
} while (--count > 0);
model.addRow(rowModel);
ResponseBuilder response = Response.ok(model);
response.cacheControl(cacheControl);
return response.build();
}
