private HRegionInfo nextRegion() throws IOException {
try {
Result results = getMetaRow();
if (results == null) {
return null;
}
byte[] regionInfoValue =
results.getValue(HConstants.CATALOG_FAMILY, HConstants.REGIONINFO_QUALIFIER);
if (regionInfoValue == null || regionInfoValue.length == 0) {
throw new NoSuchElementException(
"meta region entry missing "
+ Bytes.toString(HConstants.CATALOG_FAMILY)
+ ":"
+ Bytes.toString(HConstants.REGIONINFO_QUALIFIER));
}
HRegionInfo region = Writables.getHRegionInfo(regionInfoValue);
if (!Bytes.equals(region.getTableName(), this.tableName)) {
return null;
}
return region;
} catch (IOException e) {
e = RemoteExceptionHandler.checkIOException(e);
LOG.error("meta scanner error", e);
metaScanner.close();
throw e;
}
}
/** Apply column family options such as Bloom filters, compression, and data block encoding. */
protected void applyColumnFamilyOptions(byte[] tableName, byte[][] columnFamilies)
throws IOException {
HBaseAdmin admin = new HBaseAdmin(conf);
HTableDescriptor tableDesc = admin.getTableDescriptor(tableName);
LOG.info("Disabling table " + Bytes.toString(tableName));
admin.disableTable(tableName);
for (byte[] cf : columnFamilies) {
HColumnDescriptor columnDesc = tableDesc.getFamily(cf);
boolean isNewCf = columnDesc == null;
if (isNewCf) {
columnDesc = new HColumnDescriptor(cf);
}
if (bloomType != null) {
columnDesc.setBloomFilterType(bloomType);
}
if (compressAlgo != null) {
columnDesc.setCompressionType(compressAlgo);
}
if (dataBlockEncodingAlgo != null) {
columnDesc.setDataBlockEncoding(dataBlockEncodingAlgo);
columnDesc.setEncodeOnDisk(!encodeInCacheOnly);
}
if (inMemoryCF) {
columnDesc.setInMemory(inMemoryCF);
}
if (isNewCf) {
admin.addColumn(tableName, columnDesc);
} else {
admin.modifyColumn(tableName, columnDesc);
}
}
LOG.info("Enabling table " + Bytes.toString(tableName));
admin.enableTable(tableName);
}
/**
* Get HTD from HDFS.
*
* @param fs
* @param hbaseRootDir
* @param tableName
* @return Descriptor or null if none found.
* @throws IOException
*/
public static HTableDescriptor getTableDescriptor(
FileSystem fs, Path hbaseRootDir, byte[] tableName) throws IOException {
HTableDescriptor htd = null;
try {
TableDescriptorModtime tdmt =
getTableDescriptorModtime(fs, hbaseRootDir, Bytes.toString(tableName));
htd = tdmt == null ? null : tdmt.getTableDescriptor();
} catch (NullPointerException e) {
LOG.debug(
"Exception during readTableDecriptor. Current table name = " + Bytes.toString(tableName),
e);
}
return htd;
}
/*
* @param number Number to use as suffix.
* @return Returns zero-prefixed 5-byte wide decimal version of passed
* number (Does absolute in case number is negative).
*/
static String formatTableInfoSequenceId(final int number) {
byte[] b = new byte[WIDTH_OF_SEQUENCE_ID];
int d = Math.abs(number);
for (int i = b.length - 1; i >= 0; i--) {
b[i] = (byte) ((d % 10) + '0');
d /= 10;
}
return Bytes.toString(b);
}
/**
* A lower level API, return ID integer from raw value bytes. In case of not found
*
* <p>- if roundingFlag=0, throw IllegalArgumentException; <br>
* - if roundingFlag<0, the closest smaller ID integer if exist; <br>
* - if roundingFlag>0, the closest bigger ID integer if exist. <br>
*
* <p>Bypassing the cache layer, this could be significantly slower than getIdFromValue(T value).
*
* @throws IllegalArgumentException if value is not found in dictionary and rounding is off; or if
* rounding cannot find a smaller or bigger ID
*/
public final int getIdFromValueBytes(byte[] value, int offset, int len, int roundingFlag)
throws IllegalArgumentException {
if (isNullByteForm(value, offset, len)) return nullId();
else {
int id = getIdFromValueBytesImpl(value, offset, len, roundingFlag);
if (id == -1)
throw new IllegalArgumentException(
"Value '"
+ Bytes.toString(value, offset, len)
+ "' ("
+ Bytes.toStringBinary(value, offset, len)
+ ") not exists!");
return id;
}
}
// Parse mutate info into a map of <column name> => <update action>
private Map<String, MutationType> parseMutateInfo(byte[] mutateInfo) {
Map<String, MutationType> mi = new HashMap<String, MutationType>();
if (mutateInfo != null) {
String mutateInfoStr = Bytes.toString(mutateInfo);
String[] mutations = mutateInfoStr.split("#");
for (String mutation : mutations) {
if (mutation.isEmpty()) continue;
Preconditions.checkArgument(mutation.contains(":"), "Invalid mutation info " + mutation);
int p = mutation.indexOf(":");
String column = mutation.substring(0, p);
MutationType type = MutationType.valueOf(Integer.parseInt(mutation.substring(p + 1)));
mi.put(column, type);
}
}
return mi;
}
protected Merger(Configuration conf, FileSystem fs, final byte[] tableName) throws IOException {
this.conf = conf;
this.fs = fs;
this.maxFilesize =
conf.getLong("hbase.hregion.max.filesize", HConstants.DEFAULT_MAX_FILE_SIZE);
this.tabledir =
new Path(
fs.makeQualified(new Path(conf.get(HConstants.HBASE_DIR))),
Bytes.toString(tableName));
this.htd = FSTableDescriptors.getTableDescriptor(this.fs, this.tabledir);
Path logdir =
new Path(
tabledir, "merge_" + System.currentTimeMillis() + HConstants.HREGION_LOGDIR_NAME);
Path oldLogDir = new Path(tabledir, HConstants.HREGION_OLDLOGDIR_NAME);
this.hlog = new HLog(fs, logdir, oldLogDir, conf);
}
private void printLocations(Result r) {
RegionLocations rl = null;
if (r == null) {
LOG.info("FAILED FOR null Result");
return;
}
LOG.info("FAILED FOR " + resultToString(r) + " Stale " + r.isStale());
if (r.getRow() == null) {
return;
}
try {
rl = ((ClusterConnection) connection).locateRegion(tableName, r.getRow(), true, true);
} catch (IOException e) {
LOG.warn("Couldn't get locations for row " + Bytes.toString(r.getRow()));
}
HRegionLocation locations[] = rl.getRegionLocations();
for (HRegionLocation h : locations) {
LOG.info("LOCATION " + h);
}
}
/**
* Verifies the result from get or scan using the dataGenerator (that was presumably also used to
* generate said result).
*
* @param verifyValues verify that values in the result make sense for row/cf/column combination
* @param verifyCfAndColumnIntegrity verify that cf/column set in the result is complete. Note
* that to use this multiPut should be used, or verification has to happen after writes,
* otherwise there can be races.
* @return
*/
public boolean verifyResultAgainstDataGenerator(
Result result, boolean verifyValues, boolean verifyCfAndColumnIntegrity) {
String rowKeyStr = Bytes.toString(result.getRow());
// See if we have any data at all.
if (result.isEmpty()) {
LOG.error("Error checking data for key [" + rowKeyStr + "], no data returned");
printLocations(result);
return false;
}
if (!verifyValues && !verifyCfAndColumnIntegrity) {
return true; // as long as we have something, we are good.
}
// See if we have all the CFs.
byte[][] expectedCfs = dataGenerator.getColumnFamilies();
if (verifyCfAndColumnIntegrity && (expectedCfs.length != result.getMap().size())) {
LOG.error(
"Error checking data for key ["
+ rowKeyStr
+ "], bad family count: "
+ result.getMap().size());
printLocations(result);
return false;
}
// Verify each column family from get in the result.
for (byte[] cf : result.getMap().keySet()) {
String cfStr = Bytes.toString(cf);
Map<byte[], byte[]> columnValues = result.getFamilyMap(cf);
if (columnValues == null) {
LOG.error(
"Error checking data for key [" + rowKeyStr + "], no data for family [" + cfStr + "]]");
printLocations(result);
return false;
}
Map<String, MutationType> mutateInfo = null;
if (verifyCfAndColumnIntegrity || verifyValues) {
if (!columnValues.containsKey(MUTATE_INFO)) {
LOG.error(
"Error checking data for key ["
+ rowKeyStr
+ "], column family ["
+ cfStr
+ "], column ["
+ Bytes.toString(MUTATE_INFO)
+ "]; value is not found");
printLocations(result);
return false;
}
long cfHash = Arrays.hashCode(cf);
// Verify deleted columns, and make up column counts if deleted
byte[] mutateInfoValue = columnValues.remove(MUTATE_INFO);
mutateInfo = parseMutateInfo(mutateInfoValue);
for (Map.Entry<String, MutationType> mutate : mutateInfo.entrySet()) {
if (mutate.getValue() == MutationType.DELETE) {
byte[] column = Bytes.toBytes(mutate.getKey());
long columnHash = Arrays.hashCode(column);
long hashCode = cfHash + columnHash;
if (hashCode % 2 == 0) {
if (columnValues.containsKey(column)) {
LOG.error(
"Error checking data for key ["
+ rowKeyStr
+ "], column family ["
+ cfStr
+ "], column ["
+ mutate.getKey()
+ "]; should be deleted");
printLocations(result);
return false;
}
byte[] hashCodeBytes = Bytes.toBytes(hashCode);
columnValues.put(column, hashCodeBytes);
}
}
}
// Verify increment
if (!columnValues.containsKey(INCREMENT)) {
LOG.error(
"Error checking data for key ["
+ rowKeyStr
+ "], column family ["
+ cfStr
+ "], column ["
+ Bytes.toString(INCREMENT)
+ "]; value is not found");
printLocations(result);
return false;
}
long currentValue = Bytes.toLong(columnValues.remove(INCREMENT));
if (verifyValues) {
long amount = mutateInfo.isEmpty() ? 0 : cfHash;
long originalValue = Arrays.hashCode(result.getRow());
long extra = currentValue - originalValue;
if (extra != 0 && (amount == 0 || extra % amount != 0)) {
LOG.error(
"Error checking data for key ["
+ rowKeyStr
+ "], column family ["
+ cfStr
+ "], column [increment], extra ["
+ extra
+ "], amount ["
+ amount
+ "]");
printLocations(result);
return false;
}
if (amount != 0 && extra != amount) {
LOG.warn(
"Warning checking data for key ["
+ rowKeyStr
+ "], column family ["
+ cfStr
+ "], column [increment], incremented ["
+ (extra / amount)
+ "] times");
}
}
// See if we have correct columns.
if (verifyCfAndColumnIntegrity
&& !dataGenerator.verify(result.getRow(), cf, columnValues.keySet())) {
String colsStr = "";
for (byte[] col : columnValues.keySet()) {
if (colsStr.length() > 0) {
colsStr += ", ";
}
colsStr += "[" + Bytes.toString(col) + "]";
}
LOG.error(
"Error checking data for key ["
+ rowKeyStr
+ "], bad columns for family ["
+ cfStr
+ "]: "
+ colsStr);
printLocations(result);
return false;
}
// See if values check out.
if (verifyValues) {
for (Map.Entry<byte[], byte[]> kv : columnValues.entrySet()) {
String column = Bytes.toString(kv.getKey());
MutationType mutation = mutateInfo.get(column);
boolean verificationNeeded = true;
byte[] bytes = kv.getValue();
if (mutation != null) {
boolean mutationVerified = true;
long columnHash = Arrays.hashCode(kv.getKey());
long hashCode = cfHash + columnHash;
byte[] hashCodeBytes = Bytes.toBytes(hashCode);
if (mutation == MutationType.APPEND) {
int offset = bytes.length - hashCodeBytes.length;
mutationVerified =
offset > 0
&& Bytes.equals(
hashCodeBytes,
0,
hashCodeBytes.length,
bytes,
offset,
hashCodeBytes.length);
if (mutationVerified) {
int n = 1;
while (true) {
int newOffset = offset - hashCodeBytes.length;
if (newOffset < 0
|| !Bytes.equals(
hashCodeBytes,
0,
hashCodeBytes.length,
bytes,
newOffset,
hashCodeBytes.length)) {
break;
}
offset = newOffset;
n++;
}
if (n > 1) {
LOG.warn(
"Warning checking data for key ["
+ rowKeyStr
+ "], column family ["
+ cfStr
+ "], column ["
+ column
+ "], appended ["
+ n
+ "] times");
}
byte[] dest = new byte[offset];
System.arraycopy(bytes, 0, dest, 0, offset);
bytes = dest;
}
} else if (hashCode % 2 == 0) { // checkAndPut
mutationVerified = Bytes.equals(bytes, hashCodeBytes);
verificationNeeded = false;
}
if (!mutationVerified) {
LOG.error(
"Error checking data for key ["
+ rowKeyStr
+ "], mutation checking failed for column family ["
+ cfStr
+ "], column ["
+ column
+ "]; mutation ["
+ mutation
+ "], hashCode ["
+ hashCode
+ "], verificationNeeded ["
+ verificationNeeded
+ "]");
printLocations(result);
return false;
}
} // end of mutation checking
if (verificationNeeded
&& !dataGenerator.verify(result.getRow(), cf, kv.getKey(), bytes)) {
LOG.error(
"Error checking data for key ["
+ rowKeyStr
+ "], column family ["
+ cfStr
+ "], column ["
+ column
+ "], mutation ["
+ mutation
+ "]; value of length "
+ bytes.length);
printLocations(result);
return false;
}
}
}
}
}
return true;
}
/* (non-Javadoc)
* @see org.apache.hadoop.hbase.TableDescriptors#getHTableDescriptor(java.lang.String)
*/
@Override
public HTableDescriptor get(final byte[] tablename) throws IOException {
return get(Bytes.toString(tablename));
}
