/** Clean up the environment */
protected void shutdown() {
if (state == Coprocessor.State.ACTIVE) {
state = Coprocessor.State.STOPPING;
Thread currentThread = Thread.currentThread();
ClassLoader hostClassLoader = currentThread.getContextClassLoader();
try {
currentThread.setContextClassLoader(this.getClassLoader());
impl.stop(this);
state = Coprocessor.State.STOPPED;
} catch (IOException ioe) {
LOG.error("Error stopping coprocessor " + impl.getClass().getName(), ioe);
} finally {
currentThread.setContextClassLoader(hostClassLoader);
}
} else {
LOG.warn(
"Not stopping coprocessor "
+ impl.getClass().getName()
+ " because not active (state="
+ state.toString()
+ ")");
}
// clean up any table references
for (HTableInterface table : openTables) {
try {
((HTableWrapper) table).internalClose();
} catch (IOException e) {
// nothing can be done here
LOG.warn("Failed to close " + Bytes.toStringBinary(table.getTableName()), e);
}
}
}
private void testAlgorithm(
byte[] encodedData, ByteBuffer unencodedDataBuf, DataBlockEncoder encoder)
throws IOException {
// decode
ByteArrayInputStream bais =
new ByteArrayInputStream(
encodedData, ENCODED_DATA_OFFSET, encodedData.length - ENCODED_DATA_OFFSET);
DataInputStream dis = new DataInputStream(bais);
ByteBuffer actualDataset;
HFileContext meta =
new HFileContextBuilder()
.withHBaseCheckSum(false)
.withIncludesMvcc(includesMemstoreTS)
.withIncludesTags(includesTags)
.withCompression(Compression.Algorithm.NONE)
.build();
actualDataset = encoder.decodeKeyValues(dis, encoder.newDataBlockDecodingContext(meta));
actualDataset.rewind();
// this is because in case of prefix tree the decoded stream will not have
// the
// mvcc in it.
assertEquals(
"Encoding -> decoding gives different results for " + encoder,
Bytes.toStringBinary(unencodedDataBuf),
Bytes.toStringBinary(actualDataset));
}
@Override
public String toString() {
return "rowkey="
+ Bytes.toStringBinary(rowkey)
+ ", expected seek next using hint: "
+ Bytes.toStringBinary(hint);
}
/**
* Advanced use only. Add an existing delete marker to this Delete object.
*
* @param kv An existing KeyValue of type "delete".
* @return this for invocation chaining
* @throws IOException
*/
@SuppressWarnings("unchecked")
public Delete addDeleteMarker(Cell kv) throws IOException {
// TODO: Deprecate and rename 'add' so it matches how we add KVs to Puts.
if (!CellUtil.isDelete(kv)) {
throw new IOException(
"The recently added KeyValue is not of type "
+ "delete. Rowkey: "
+ Bytes.toStringBinary(this.row));
}
if (Bytes.compareTo(
this.row, 0, row.length, kv.getRowArray(), kv.getRowOffset(), kv.getRowLength())
!= 0) {
throw new WrongRowIOException(
"The row in "
+ kv.toString()
+ " doesn't match the original one "
+ Bytes.toStringBinary(this.row));
}
byte[] family = CellUtil.cloneFamily(kv);
List<Cell> list = familyMap.get(family);
if (list == null) {
list = new ArrayList<Cell>();
}
list.add(kv);
familyMap.put(family, list);
return this;
}
public static void main(String[] args) {
Integer id = Integer.parseInt(args[0]);
System.out.println(id);
byte[] k = new byte[5];
k[0] = (byte) 1;
byte[] col = getColIdbyte(id);
for (int i = 0; i < 4; i++) {
k[i + 1] = col[i];
}
System.out.println(Bytes.toStringBinary(k));
Get get = new Get(k);
// get.addColumn(Bytes.toBytes("A:i"));
HTable table;
try {
table = new HTable(hconf, "new");
Result result = table.get(get);
KeyValue[] g = result.raw();
System.out.println("dfgsdfgsdfgsdfgsdfffffffffffffffffffffffffffffffffff");
System.out.println("dfgsdfgsdfgsdfgsdfffffffffffffffffffffffffffffffffff");
System.out.println("dfgsdfgsdfgsdfgsdfffffffffffffffffffffffffffffffffff");
System.out.println("dfgsdfgsdfgsdfgsdfffffffffffffffffffffffffffffffffff");
System.out.println("dfgsdfgsdfgsdfgsdfffffffffffffffffffffffffffffffffff");
System.out.println("dfgsdfgsdfgsdfgsdfffffffffffffffffffffffffffffffffff");
for (int i = 0; i < g.length; i++) {
System.out.println(Bytes.toStringBinary(g[i].getValue()) + " kkkkkkkkkkkkkkkkkkkkkkk");
}
} catch (IOException e) {
// TODO Auto-generated catch block
e.printStackTrace();
}
}
@Override
public String toString() {
return this.getClass().getSimpleName()
+ " "
+ (this.minColumnInclusive ? "[" : "(")
+ Bytes.toStringBinary(this.minColumn)
+ ", "
+ Bytes.toStringBinary(this.maxColumn)
+ (this.maxColumnInclusive ? "]" : ")");
}
/**
* Reads a magic record of the length {@link #MAGIC_LENGTH} from the given byte buffer and expects
* it to match this block type.
*/
public void readAndCheck(ByteBuffer in) throws IOException {
byte[] buf = new byte[MAGIC_LENGTH];
in.get(buf);
if (Bytes.compareTo(buf, magic) != 0) {
throw new IOException(
"Invalid magic: expected "
+ Bytes.toStringBinary(magic)
+ ", got "
+ Bytes.toStringBinary(buf));
}
}
/** @return Region name as a String for use in logging, etc. */
public String getRegionNameAsString() {
if (hasEncodedName(this.regionName)) {
// new format region names already have their encoded name.
return Bytes.toStringBinary(this.regionName);
}
// old format. regionNameStr doesn't have the region name.
//
//
return Bytes.toStringBinary(this.regionName) + "." + this.getEncodedName();
}
/**
* Maps the data.
*
* @param row The current table row key.
* @param values The columns.
* @param context The current context.
* @throws IOException When something is broken with the data.
* @see org.apache.hadoop.mapreduce.Mapper#map(KEYIN, VALUEIN,
* org.apache.hadoop.mapreduce.Mapper.Context)
*/
@Override
public void map(ImmutableBytesWritable row, Result values, Context context) throws IOException {
String currentFamilyName = null;
String currentQualifierName = null;
String currentRowKey = null;
Configuration config = context.getConfiguration();
String separator = config.get("ReportSeparator", ":");
try {
if (values != null) {
context.getCounter(Counters.ROWS).increment(1);
context.write(new Text("Total ROWS"), new IntWritable(1));
}
for (KeyValue value : values.list()) {
currentRowKey = Bytes.toStringBinary(value.getRow());
String thisRowFamilyName = Bytes.toStringBinary(value.getFamily());
if (thisRowFamilyName != null && !thisRowFamilyName.equals(currentFamilyName)) {
currentFamilyName = thisRowFamilyName;
context.getCounter("CF", thisRowFamilyName).increment(1);
context.write(new Text("Total Families Across all Rows"), new IntWritable(1));
context.write(new Text(thisRowFamilyName), new IntWritable(1));
}
String thisRowQualifierName =
thisRowFamilyName + separator + Bytes.toStringBinary(value.getQualifier());
if (thisRowQualifierName != null && !thisRowQualifierName.equals(currentQualifierName)) {
currentQualifierName = thisRowQualifierName;
context.getCounter("CFQL", thisRowQualifierName).increment(1);
context.write(new Text("Total Qualifiers across all Rows"), new IntWritable(1));
context.write(new Text(thisRowQualifierName), new IntWritable(1));
// Intialize versions
context
.getCounter("QL_VERSIONS", currentRowKey + separator + thisRowQualifierName)
.increment(1);
context.write(
new Text(currentRowKey + separator + thisRowQualifierName + "_Versions"),
new IntWritable(1));
} else {
// Increment versions
currentQualifierName = thisRowQualifierName;
context
.getCounter("QL_VERSIONS", currentRowKey + separator + thisRowQualifierName)
.increment(1);
context.write(
new Text(currentRowKey + separator + thisRowQualifierName + "_Versions"),
new IntWritable(1));
}
}
} catch (InterruptedException e) {
e.printStackTrace();
}
}
public static BlockType parse(byte[] buf, int offset, int length) throws IOException {
if (length != MAGIC_LENGTH) {
throw new IOException(
"Magic record of invalid length: " + Bytes.toStringBinary(buf, offset, length));
}
for (BlockType blockType : values())
if (Bytes.compareTo(blockType.magic, 0, MAGIC_LENGTH, buf, offset, MAGIC_LENGTH) == 0)
return blockType;
throw new IOException(
"Invalid HFile block magic: " + Bytes.toStringBinary(buf, offset, MAGIC_LENGTH));
}
/**
* Returns true if the given inclusive range of rows is fully contained by this region. For
* example, if the region is foo,a,g and this is passed ["b","c"] or ["a","c"] it will return
* true, but if this is passed ["b","z"] it will return false.
*
* @throws IllegalArgumentException if the range passed is invalid (ie end < start)
*/
public boolean containsRange(byte[] rangeStartKey, byte[] rangeEndKey) {
if (Bytes.compareTo(rangeStartKey, rangeEndKey) > 0) {
throw new IllegalArgumentException(
"Invalid range: "
+ Bytes.toStringBinary(rangeStartKey)
+ " > "
+ Bytes.toStringBinary(rangeEndKey));
}
boolean firstKeyInRange = Bytes.compareTo(rangeStartKey, startKey) >= 0;
boolean lastKeyInRange =
Bytes.compareTo(rangeEndKey, endKey) < 0
|| Bytes.equals(endKey, HConstants.EMPTY_BYTE_ARRAY);
return firstKeyInRange && lastKeyInRange;
}
/**
* Deletes daughters references in offlined split parent.
*
* @param catalogTracker
* @param parent Parent row we're to remove daughter reference from
* @throws NotAllMetaRegionsOnlineException
* @throws IOException
*/
public static void deleteDaughtersReferencesInParent(
CatalogTracker catalogTracker, final HRegionInfo parent)
throws NotAllMetaRegionsOnlineException, IOException {
Delete delete = new Delete(parent.getRegionName());
delete.deleteColumns(HConstants.CATALOG_FAMILY, HConstants.SPLITA_QUALIFIER);
delete.deleteColumns(HConstants.CATALOG_FAMILY, HConstants.SPLITB_QUALIFIER);
deleteFromMetaTable(catalogTracker, delete);
LOG.info(
"Deleted daughters references, qualifier="
+ Bytes.toStringBinary(HConstants.SPLITA_QUALIFIER)
+ " and qualifier="
+ Bytes.toStringBinary(HConstants.SPLITB_QUALIFIER)
+ ", from parent "
+ parent.getRegionNameAsString());
}
/**
* Produces a string map for this key. Useful for programmatic use and manipulation of the data
* stored in an WALKey, for example, printing as JSON.
*
* @return a Map containing data from this key
*/
public Map<String, Object> toStringMap() {
Map<String, Object> stringMap = new HashMap<String, Object>();
stringMap.put("table", tablename);
stringMap.put("region", Bytes.toStringBinary(encodedRegionName));
stringMap.put("sequence", logSeqNum);
return stringMap;
}
// @Override
// protected void map(ImmutableBytesWritable key, Text value, Context context) throws
// IOException, InterruptedException {
// Text combinedKeyValue = new Text();
// //the structure is key###value
// combinedKeyValue.set(Bytes.toString(key.get()) + "###" + value.toString());
// context.write(one, combinedKeyValue);
// }
@Override
protected void map(ImmutableBytesWritable key, Result columns, Context context)
throws IOException, InterruptedException {
Text combinedKeyValue = new Text();
// the structure is key###value
String value = null;
try {
for (KeyValue kv : columns.list()) {
byte[] gmmData = kv.getValue();
String gmmString = Bytes.toStringBinary(gmmData);
// /* just for checking that gmm is correctly constructed
MixtureModel m = null;
m = (MixtureModel) ObjectAndByte.byteArrayToObject(Bytes.toBytesBinary(gmmString));
System.out.println("m.size:" + m.size);
// */
combinedKeyValue.set(Bytes.toString(key.get()) + "###" + gmmString);
context.write(one, combinedKeyValue);
// context.write(key, new Text(gmmString));
}
} catch (Exception e) {
e.printStackTrace();
}
}
/**
* Write out a SequenceFile that can be read by TotalOrderPartitioner that contains the split
* points in startKeys.
*
* <p>This method was copied from HFileOutputFormat in hbase-0.90.1-cdh3u0. I had to copy it
* because it's private.
*
* @param conf The job configuration.
* @param partitionsPath output path for SequenceFile.
* @param startKeys the region start keys to use as the partitions.
* @throws IOException If there is an error.
*/
private static void writePartitionFile(
Configuration conf, Path partitionsPath, List<HFileKeyValue> startKeys) throws IOException {
if (startKeys.isEmpty()) {
throw new IllegalArgumentException("No regions passed");
}
// We're generating a list of split points, and we don't ever
// have keys < the first region (which has an empty start key)
// so we need to remove it. Otherwise we would end up with an
// empty reducer with index 0.
TreeSet<HFileKeyValue> sorted = new TreeSet<HFileKeyValue>();
sorted.addAll(startKeys);
HFileKeyValue first = sorted.first();
if (0 != first.getRowKey().length) {
throw new IllegalArgumentException(
"First region of table should have empty start row key. Instead has: "
+ Bytes.toStringBinary(first.getRowKey()));
}
sorted.remove(first);
// Write the actual file
final SequenceFile.Writer writer =
KijiMRPlatformBridge.get()
.newSeqFileWriter(conf, partitionsPath, HFileKeyValue.class, NullWritable.class);
try {
for (HFileKeyValue startKey : sorted) {
writer.append(startKey, NullWritable.get());
}
} finally {
writer.close();
}
}
@Override
public boolean prepare() throws IOException {
if (!this.parent.isSplittable()) return false;
// Split key can be null if this region is unsplittable; i.e. has refs.
if (this.splitrow == null) return false;
HRegionInfo hri = this.parent.getRegionInfo();
parent.prepareToSplit();
// Check splitrow.
byte[] startKey = hri.getStartKey();
byte[] endKey = hri.getEndKey();
if (Bytes.equals(startKey, splitrow) || !this.parent.getRegionInfo().containsRow(splitrow)) {
LOG.info(
"Split row is not inside region key range or is equal to "
+ "startkey: "
+ Bytes.toStringBinary(this.splitrow));
return false;
}
long rid = getDaughterRegionIdTimestamp(hri);
this.hri_a = new HRegionInfo(hri.getTable(), startKey, this.splitrow, false, rid);
this.hri_b = new HRegionInfo(hri.getTable(), this.splitrow, endKey, false, rid);
transition(SplitTransactionPhase.PREPARED);
return true;
}
/**
* Get the HRegionInfo from cache, if not there, from the hbase:meta table. Be careful. Does RPC.
* Do not hold a lock or synchronize when you call this method.
*
* @param regionName
* @return HRegionInfo for the region
*/
@SuppressWarnings("deprecation")
protected HRegionInfo getRegionInfo(final byte[] regionName) {
String encodedName = HRegionInfo.encodeRegionName(regionName);
RegionState regionState = getRegionState(encodedName);
if (regionState != null) {
return regionState.getRegion();
}
try {
Pair<HRegionInfo, ServerName> p =
MetaTableAccessor.getRegion(server.getConnection(), regionName);
HRegionInfo hri = p == null ? null : p.getFirst();
if (hri != null) {
createRegionState(hri);
}
return hri;
} catch (IOException e) {
server.abort(
"Aborting because error occoured while reading "
+ Bytes.toStringBinary(regionName)
+ " from hbase:meta",
e);
return null;
}
}
/**
* Looks at every value of the mapreduce output and verifies that indeed the values have been
* reversed.
*
* @param table Table to scan.
* @throws IOException
* @throws NullPointerException if we failed to find a cell value
*/
private void verifyAttempt(final Table table) throws IOException, NullPointerException {
Scan scan = new Scan();
scan.addFamily(INPUT_FAMILY);
scan.addFamily(OUTPUT_FAMILY);
ResultScanner scanner = table.getScanner(scan);
try {
Iterator<Result> itr = scanner.iterator();
assertTrue(itr.hasNext());
while (itr.hasNext()) {
Result r = itr.next();
if (LOG.isDebugEnabled()) {
if (r.size() > 2) {
throw new IOException("Too many results, expected 2 got " + r.size());
}
}
byte[] firstValue = null;
byte[] secondValue = null;
int count = 0;
for (Cell kv : r.listCells()) {
if (count == 0) {
firstValue = CellUtil.cloneValue(kv);
} else if (count == 1) {
secondValue = CellUtil.cloneValue(kv);
} else if (count == 2) {
break;
}
count++;
}
String first = "";
if (firstValue == null) {
throw new NullPointerException(Bytes.toString(r.getRow()) + ": first value is null");
}
first = Bytes.toString(firstValue);
String second = "";
if (secondValue == null) {
throw new NullPointerException(Bytes.toString(r.getRow()) + ": second value is null");
}
byte[] secondReversed = new byte[secondValue.length];
for (int i = 0, j = secondValue.length - 1; j >= 0; j--, i++) {
secondReversed[i] = secondValue[j];
}
second = Bytes.toString(secondReversed);
if (first.compareTo(second) != 0) {
if (LOG.isDebugEnabled()) {
LOG.debug(
"second key is not the reverse of first. row="
+ Bytes.toStringBinary(r.getRow())
+ ", first value="
+ first
+ ", second value="
+ second);
}
fail();
}
}
} finally {
scanner.close();
}
}
private String createNonSequential(String path, byte[] data, List<ACL> acl, CreateMode createMode)
throws KeeperException, InterruptedException {
RetryCounter retryCounter = retryCounterFactory.create();
boolean isRetry = false; // False for first attempt, true for all retries.
while (true) {
try {
return zk.create(path, data, acl, createMode);
} catch (KeeperException e) {
switch (e.code()) {
case NODEEXISTS:
if (isRetry) {
// If the connection was lost, there is still a possibility that
// we have successfully created the node at our previous attempt,
// so we read the node and compare.
byte[] currentData = zk.getData(path, false, null);
if (currentData != null && Bytes.compareTo(currentData, data) == 0) {
// We successfully created a non-sequential node
return path;
}
LOG.error(
"Node "
+ path
+ " already exists with "
+ Bytes.toStringBinary(currentData)
+ ", could not write "
+ Bytes.toStringBinary(data));
throw e;
}
LOG.info("Node " + path + " already exists and this is not a " + "retry");
throw e;
case CONNECTIONLOSS:
case SESSIONEXPIRED:
case OPERATIONTIMEOUT:
retryOrThrow(retryCounter, e, "create");
break;
default:
throw e;
}
}
retryCounter.sleepUntilNextRetry();
retryCounter.useRetry();
isRetry = true;
}
}
static String stringifyKvs(Collection<Cell> kvs) {
StringBuilder out = new StringBuilder();
out.append("[");
if (kvs != null) {
for (Cell kv : kvs) {
byte[] col = CellUtil.cloneQualifier(kv);
byte[] val = CellUtil.cloneValue(kv);
if (Bytes.equals(col, COUNTER)) {
out.append(Bytes.toStringBinary(col) + ":" + Bytes.toInt(val) + " ");
} else {
out.append(Bytes.toStringBinary(col) + ":" + Bytes.toStringBinary(val) + " ");
}
}
}
out.append("]");
return out.toString();
}
/** @see java.lang.Object#toString() */
@Override
public String toString() {
return "{ENCODED => "
+ getEncodedName()
+ ", "
+ HConstants.NAME
+ " => '"
+ Bytes.toStringBinary(this.regionName)
+ "', STARTKEY => '"
+ Bytes.toStringBinary(this.startKey)
+ "', ENDKEY => '"
+ Bytes.toStringBinary(this.endKey)
+ "'"
+ (isOffline() ? ", OFFLINE => true" : "")
+ (isSplit() ? ", SPLIT => true" : "")
+ ((replicaId > 0) ? ", REPLICA_ID => " + replicaId : "")
+ "}";
}
/**
* Create a new ClientScanner for the specified table Note that the passed {@link Scan}'s start
* row maybe changed changed.
*
* @param conf The {@link Configuration} to use.
* @param scan {@link Scan} to use in this scanner
* @param tableName The table that we wish to scan
* @param connection Connection identifying the cluster
* @throws IOException
*/
public ClientScanner(
final Configuration conf,
final Scan scan,
final TableName tableName,
ClusterConnection connection,
RpcRetryingCallerFactory rpcFactory,
RpcControllerFactory controllerFactory,
ExecutorService pool,
int primaryOperationTimeout)
throws IOException {
if (LOG.isTraceEnabled()) {
LOG.trace(
"Scan table=" + tableName + ", startRow=" + Bytes.toStringBinary(scan.getStartRow()));
}
this.scan = scan;
this.tableName = tableName;
this.lastNext = System.currentTimeMillis();
this.connection = connection;
this.pool = pool;
this.primaryOperationTimeout = primaryOperationTimeout;
this.retries =
conf.getInt(
HConstants.HBASE_CLIENT_RETRIES_NUMBER, HConstants.DEFAULT_HBASE_CLIENT_RETRIES_NUMBER);
if (scan.getMaxResultSize() > 0) {
this.maxScannerResultSize = scan.getMaxResultSize();
} else {
this.maxScannerResultSize =
conf.getLong(
HConstants.HBASE_CLIENT_SCANNER_MAX_RESULT_SIZE_KEY,
HConstants.DEFAULT_HBASE_CLIENT_SCANNER_MAX_RESULT_SIZE);
}
this.scannerTimeout =
HBaseConfiguration.getInt(
conf,
HConstants.HBASE_CLIENT_SCANNER_TIMEOUT_PERIOD,
HConstants.HBASE_REGIONSERVER_LEASE_PERIOD_KEY,
HConstants.DEFAULT_HBASE_CLIENT_SCANNER_TIMEOUT_PERIOD);
// check if application wants to collect scan metrics
initScanMetrics(scan);
// Use the caching from the Scan. If not set, use the default cache setting for this table.
if (this.scan.getCaching() > 0) {
this.caching = this.scan.getCaching();
} else {
this.caching =
conf.getInt(
HConstants.HBASE_CLIENT_SCANNER_CACHING,
HConstants.DEFAULT_HBASE_CLIENT_SCANNER_CACHING);
}
this.caller = rpcFactory.<Result[]>newCaller();
this.rpcControllerFactory = controllerFactory;
this.conf = conf;
initializeScannerInConstruction();
}
/** Clean up the current block */
private void finishBlock() throws IOException {
if (!fsBlockWriter.isWriting() || fsBlockWriter.blockSizeWritten() == 0) return;
long startTimeNs = System.nanoTime();
// Update the first data block offset for scanning.
if (firstDataBlockOffset == -1) {
firstDataBlockOffset = outputStream.getPos();
}
// Update the last data block offset
lastDataBlockOffset = outputStream.getPos();
fsBlockWriter.writeHeaderAndData(outputStream);
int onDiskSize = fsBlockWriter.getOnDiskSizeWithHeader();
// Generate a shorter faked key into index block. For example, consider a block boundary
// between the keys "the quick brown fox" and "the who test text". We can use "the r" as the
// key for the index block entry since it is > all entries in the previous block and <= all
// entries in subsequent blocks.
if (comparator instanceof KeyComparator) {
byte[] fakeKey =
((KeyComparator) comparator).getShortMidpointKey(lastKeyOfPreviousBlock, firstKeyInBlock);
if (comparator.compare(fakeKey, firstKeyInBlock) > 0) {
throw new IOException(
"Unexpected getShortMidpointKey result, fakeKey:"
+ Bytes.toStringBinary(fakeKey)
+ ", firstKeyInBlock:"
+ Bytes.toStringBinary(firstKeyInBlock));
}
if (lastKeyOfPreviousBlock != null
&& comparator.compare(lastKeyOfPreviousBlock, fakeKey) >= 0) {
throw new IOException(
"Unexpected getShortMidpointKey result, lastKeyOfPreviousBlock:"
+ Bytes.toStringBinary(lastKeyOfPreviousBlock)
+ ", fakeKey:"
+ Bytes.toStringBinary(fakeKey));
}
dataBlockIndexWriter.addEntry(fakeKey, lastDataBlockOffset, onDiskSize);
} else {
dataBlockIndexWriter.addEntry(firstKeyInBlock, lastDataBlockOffset, onDiskSize);
}
totalUncompressedBytes += fsBlockWriter.getUncompressedSizeWithHeader();
HFile.offerWriteLatency(System.nanoTime() - startTimeNs);
if (cacheConf.shouldCacheDataOnWrite()) {
doCacheOnWrite(lastDataBlockOffset);
}
}
@Override
public Boolean call() throws Exception {
Thread.currentThread().setName("reader " + readerId);
Random rand = new Random();
StoreFileScanner scanner = reader.getStoreFileScanner(true, pread);
while (System.currentTimeMillis() < endTime) {
byte[] row = createRandomRow(rand, firstRow, lastRow);
KeyValue kvToSeek = new KeyValue(row, family, createRandomQualifier(rand));
if (rand.nextDouble() < 0.0001) {
LOG.info("kvToSeek=" + kvToSeek);
}
boolean seekResult;
try {
seekResult = scanner.seek(kvToSeek);
} catch (IOException ex) {
throw new IOException("Seek failed for key " + kvToSeek + ", pread=" + pread, ex);
}
numSeeks.incrementAndGet();
if (!seekResult) {
error("Seek returned false for row " + Bytes.toStringBinary(row));
return false;
}
for (int i = 0; i < rand.nextInt(10) + 1; ++i) {
KeyValue kv = scanner.next();
numKV.incrementAndGet();
if (i == 0 && kv == null) {
error(
"scanner.next() returned null at the first iteration for "
+ "row "
+ Bytes.toStringBinary(row));
return false;
}
if (kv == null) break;
String keyHashStr = MD5Hash.getMD5AsHex(kv.getKey());
keysRead.add(keyHashStr);
totalBytes.addAndGet(kv.getLength());
}
}
return true;
}
/*
* Gets a scanner for the next region. If this.currentRegion != null, then
* we will move to the endrow of this.currentRegion. Else we will get
* scanner at the scan.getStartRow(). We will go no further, just tidy
* up outstanding scanners, if <code>currentRegion != null</code> and
* <code>done</code> is true.
* @param nbRows
* @param done Server-side says we're done scanning.
*/
protected boolean nextScanner(int nbRows, final boolean done) throws IOException {
// Close the previous scanner if it's open
if (this.callable != null) {
this.callable.setClose();
call(scan, callable, caller, scannerTimeout);
this.callable = null;
}
// Where to start the next scanner
byte[] localStartKey;
// if we're at end of table, close and return false to stop iterating
if (this.currentRegion != null) {
byte[] endKey = this.currentRegion.getEndKey();
if (endKey == null
|| Bytes.equals(endKey, HConstants.EMPTY_BYTE_ARRAY)
|| checkScanStopRow(endKey)
|| done) {
close();
if (LOG.isTraceEnabled()) {
LOG.trace("Finished " + this.currentRegion);
}
return false;
}
localStartKey = endKey;
if (LOG.isTraceEnabled()) {
LOG.trace("Finished " + this.currentRegion);
}
} else {
localStartKey = this.scan.getStartRow();
}
if (LOG.isDebugEnabled() && this.currentRegion != null) {
// Only worth logging if NOT first region in scan.
LOG.debug(
"Advancing internal scanner to startKey at '"
+ Bytes.toStringBinary(localStartKey)
+ "'");
}
try {
callable = getScannerCallable(localStartKey, nbRows);
// Open a scanner on the region server starting at the
// beginning of the region
call(scan, callable, caller, scannerTimeout);
this.currentRegion = callable.getHRegionInfo();
if (this.scanMetrics != null) {
this.scanMetrics.countOfRegions.incrementAndGet();
}
} catch (IOException e) {
close();
throw e;
}
return true;
}
@Test
public void testNextOnSample() throws IOException {
List<KeyValue> sampleKv = generator.generateTestKeyValues(NUMBER_OF_KV, includesTags);
for (DataBlockEncoding encoding : DataBlockEncoding.values()) {
// Off heap block data support not added for PREFIX_TREE DBE yet.
// TODO remove this once support is added. HBASE-12298
if (this.useOffheapData && encoding == DataBlockEncoding.PREFIX_TREE) continue;
if (encoding.getEncoder() == null) {
continue;
}
DataBlockEncoder encoder = encoding.getEncoder();
ByteBuffer encodedBuffer =
encodeKeyValues(
encoding,
sampleKv,
getEncodingContext(Compression.Algorithm.NONE, encoding),
this.useOffheapData);
HFileContext meta =
new HFileContextBuilder()
.withHBaseCheckSum(false)
.withIncludesMvcc(includesMemstoreTS)
.withIncludesTags(includesTags)
.withCompression(Compression.Algorithm.NONE)
.build();
DataBlockEncoder.EncodedSeeker seeker =
encoder.createSeeker(
CellComparator.COMPARATOR, encoder.newDataBlockDecodingContext(meta));
seeker.setCurrentBuffer(new SingleByteBuff(encodedBuffer));
int i = 0;
do {
KeyValue expectedKeyValue = sampleKv.get(i);
Cell cell = seeker.getCell();
if (CellComparator.COMPARATOR.compareKeyIgnoresMvcc(expectedKeyValue, cell) != 0) {
int commonPrefix =
CellUtil.findCommonPrefixInFlatKey(expectedKeyValue, cell, false, true);
fail(
String.format(
"next() produces wrong results "
+ "encoder: %s i: %d commonPrefix: %d"
+ "\n expected %s\n actual %s",
encoder.toString(),
i,
commonPrefix,
Bytes.toStringBinary(
expectedKeyValue.getBuffer(),
expectedKeyValue.getKeyOffset(),
expectedKeyValue.getKeyLength()),
CellUtil.toString(cell, false)));
}
i++;
} while (seeker.next());
}
}
public static String toString(byte[][] values) {
if (values == null) {
return "null";
}
StringBuilder buf = new StringBuilder("[");
for (byte[] value : values) {
buf.append(Bytes.toStringBinary(value));
buf.append(',');
}
buf.setCharAt(buf.length() - 1, ']');
return buf.toString();
}
/**
* Update a record in the database. Any field/value pairs in the specified values HashMap will be
* written into the record with the specified record key, overwriting any existing values with the
* same field name.
*
* @param table The name of the table
* @param key The record key of the record to write
* @param values A HashMap of field/value pairs to update in the record
* @return Zero on success, a non-zero error code on error
*/
@Override
public Status update(String table, String key, HashMap<String, ByteIterator> values) {
// if this is a "new" table, init HTable object. Else, use existing one
if (!tableName.equals(table)) {
currentTable = null;
try {
getHTable(table);
tableName = table;
} catch (IOException e) {
System.err.println("Error accessing HBase table: " + e);
return Status.ERROR;
}
}
if (debug) {
System.out.println("Setting up put for key: " + key);
}
Put p = new Put(Bytes.toBytes(key));
p.setDurability(durability);
for (Map.Entry<String, ByteIterator> entry : values.entrySet()) {
byte[] value = entry.getValue().toArray();
if (debug) {
System.out.println(
"Adding field/value "
+ entry.getKey()
+ "/"
+ Bytes.toStringBinary(value)
+ " to put request");
}
p.addColumn(columnFamilyBytes, Bytes.toBytes(entry.getKey()), value);
}
try {
if (clientSideBuffering) {
Preconditions.checkNotNull(bufferedMutator);
bufferedMutator.mutate(p);
} else {
currentTable.put(p);
}
} catch (IOException e) {
if (debug) {
System.err.println("Error doing put: " + e);
}
return Status.ERROR;
} catch (ConcurrentModificationException e) {
// do nothing for now...hope this is rare
return Status.ERROR;
}
return Status.OK;
}
private ReplicateWALEntryResponse replayToServer(List<Entry> entries, int timeout)
throws IOException {
// check whether we should still replay this entry. If the regions are changed, or the
// entry is not coming form the primary region, filter it out because we do not need it.
// Regions can change because of (1) region split (2) region merge (3) table recreated
boolean skip = false;
if (!Bytes.equals(
location.getRegionInfo().getEncodedNameAsBytes(), initialEncodedRegionName)) {
skip = true;
}
if (!entries.isEmpty() && !skip) {
Entry[] entriesArray = new Entry[entries.size()];
entriesArray = entries.toArray(entriesArray);
// set the region name for the target region replica
Pair<AdminProtos.ReplicateWALEntryRequest, CellScanner> p =
ReplicationProtbufUtil.buildReplicateWALEntryRequest(
entriesArray, location.getRegionInfo().getEncodedNameAsBytes(), null, null, null);
try {
PayloadCarryingRpcController controller =
rpcControllerFactory.newController(p.getSecond());
controller.setCallTimeout(timeout);
controller.setPriority(tableName);
return stub.replay(controller, p.getFirst());
} catch (ServiceException se) {
throw ProtobufUtil.getRemoteException(se);
}
}
if (skip) {
if (LOG.isTraceEnabled()) {
LOG.trace(
"Skipping "
+ entries.size()
+ " entries in table "
+ tableName
+ " because located region "
+ location.getRegionInfo().getEncodedName()
+ " is different than the original region "
+ Bytes.toStringBinary(initialEncodedRegionName)
+ " from WALEdit");
for (Entry entry : entries) {
LOG.trace("Skipping : " + entry);
}
}
skippedEntries.addAndGet(entries.size());
}
return ReplicateWALEntryResponse.newBuilder().build();
}
private boolean authorize(
List<TablePermission> perms, byte[] table, KeyValue kv, TablePermission.Action action) {
if (perms != null) {
for (TablePermission p : perms) {
if (p.implies(table, kv, action)) {
return true;
}
}
} else if (LOG.isDebugEnabled()) {
LOG.debug("No permissions for authorize() check, table=" + Bytes.toStringBinary(table));
}
return false;
}
