static TableDescriptorModtime getTableDescriptorModtime(
FileSystem fs, Path hbaseRootDir, String tableName) throws NullPointerException, IOException {
// ignore both -ROOT- and .META. tables
if (Bytes.compareTo(Bytes.toBytes(tableName), HConstants.ROOT_TABLE_NAME) == 0
|| Bytes.compareTo(Bytes.toBytes(tableName), HConstants.META_TABLE_NAME) == 0) {
return null;
}
return getTableDescriptorModtime(fs, FSUtils.getTablePath(hbaseRootDir, tableName));
}
@Test
public void testHTableDescriptors() throws IOException, InterruptedException {
final String name = "testHTableDescriptors";
FileSystem fs = FileSystem.get(UTIL.getConfiguration());
// Cleanup old tests if any debris laying around.
Path rootdir = new Path(UTIL.getDataTestDir(), name);
final int count = 10;
// Write out table infos.
for (int i = 0; i < count; i++) {
HTableDescriptor htd = new HTableDescriptor(name + i);
createHTDInFS(fs, rootdir, htd);
}
FSTableDescriptors htds =
new FSTableDescriptors(fs, rootdir) {
@Override
public HTableDescriptor get(byte[] tablename)
throws TableExistsException, FileNotFoundException, IOException {
LOG.info(Bytes.toString(tablename) + ", cachehits=" + this.cachehits);
return super.get(tablename);
}
};
for (int i = 0; i < count; i++) {
assertTrue(htds.get(Bytes.toBytes(name + i)) != null);
}
for (int i = 0; i < count; i++) {
assertTrue(htds.get(Bytes.toBytes(name + i)) != null);
}
// Update the table infos
for (int i = 0; i < count; i++) {
HTableDescriptor htd = new HTableDescriptor(name + i);
htd.addFamily(new HColumnDescriptor("" + i));
FSTableDescriptors.updateHTableDescriptor(fs, rootdir, htd);
}
// Wait a while so mod time we write is for sure different.
Thread.sleep(100);
for (int i = 0; i < count; i++) {
assertTrue(htds.get(Bytes.toBytes(name + i)) != null);
}
for (int i = 0; i < count; i++) {
assertTrue(htds.get(Bytes.toBytes(name + i)) != null);
}
assertEquals(count * 4, htds.invocations);
assertTrue(
"expected=" + (count * 2) + ", actual=" + htds.cachehits, htds.cachehits >= (count * 2));
assertTrue(htds.get(HConstants.ROOT_TABLE_NAME) != null);
assertEquals(htds.invocations, count * 4 + 1);
assertTrue(
"expected=" + ((count * 2) + 1) + ", actual=" + htds.cachehits,
htds.cachehits >= ((count * 2) + 1));
}
@Test(expected = IllegalArgumentException.class)
public void testEmptyNamespaceName() {
for (String nn : emptyNames) {
TableName.isLegalNamespaceName(Bytes.toBytes(nn));
fail("invalid Namespace name " + nn + " should have failed with IllegalArgumentException");
}
}
@Test(expected = IllegalArgumentException.class)
public void testEmptyTableName() {
for (String tn : emptyNames) {
TableName.isLegalFullyQualifiedTableName(Bytes.toBytes(tn));
fail("invalid tablename " + tn + " should have failed with IllegalArgumentException");
}
}
@Test
public void testTTL() throws Exception {
TableName tableName = TableName.valueOf("testTTL");
if (TEST_UTIL.getHBaseAdmin().tableExists(tableName)) {
TEST_UTIL.deleteTable(tableName);
}
HTableDescriptor desc = new HTableDescriptor(tableName);
HColumnDescriptor hcd = new HColumnDescriptor(F).setMaxVersions(10).setTimeToLive(1);
desc.addFamily(hcd);
TEST_UTIL.getHBaseAdmin().createTable(desc);
Table t = new HTable(new Configuration(TEST_UTIL.getConfiguration()), tableName);
long now = EnvironmentEdgeManager.currentTime();
ManualEnvironmentEdge me = new ManualEnvironmentEdge();
me.setValue(now);
EnvironmentEdgeManagerTestHelper.injectEdge(me);
// 2s in the past
long ts = now - 2000;
// Set the TTL override to 3s
Put p = new Put(R);
p.setAttribute("ttl", new byte[] {});
p.add(F, tableName.getName(), Bytes.toBytes(3000L));
t.put(p);
p = new Put(R);
p.add(F, Q, ts, Q);
t.put(p);
p = new Put(R);
p.add(F, Q, ts + 1, Q);
t.put(p);
// these two should be expired but for the override
// (their ts was 2s in the past)
Get g = new Get(R);
g.setMaxVersions(10);
Result r = t.get(g);
// still there?
assertEquals(2, r.size());
TEST_UTIL.flush(tableName);
TEST_UTIL.compact(tableName, true);
g = new Get(R);
g.setMaxVersions(10);
r = t.get(g);
// still there?
assertEquals(2, r.size());
// roll time forward 2s.
me.setValue(now + 2000);
// now verify that data eventually does expire
g = new Get(R);
g.setMaxVersions(10);
r = t.get(g);
// should be gone now
assertEquals(0, r.size());
t.close();
}
/**
* Get the directory to archive a store directory
*
* @param conf {@link Configuration} to read for the archive directory name
* @param tableName table name under which the store currently lives
* @param regionName region encoded name under which the store currently lives
* @param familyName name of the family in the store
* @return {@link Path} to the directory to archive the given store or <tt>null</tt> if it should
* not be archived
*/
public static Path getStoreArchivePath(
final Configuration conf,
final TableName tableName,
final String regionName,
final String familyName)
throws IOException {
Path tableArchiveDir = getTableArchivePath(conf, tableName);
return HStore.getStoreHomedir(tableArchiveDir, regionName, Bytes.toBytes(familyName));
}
@Test(expected = IllegalArgumentException.class)
public void testInvalidNamespace() {
for (String tn : invalidNamespace) {
TableName.isLegalFullyQualifiedTableName(Bytes.toBytes(tn));
fail(
"invalid namespace "
+ tn
+ " should have failed with IllegalArgumentException for namespace");
}
}
@Test
public void testIllegalHTableNames() {
for (String tn : illegalTableNames) {
try {
TableName.isLegalFullyQualifiedTableName(Bytes.toBytes(tn));
fail("invalid tablename " + tn + " should have failed");
} catch (Exception e) {
// expected
}
}
}
private static void writeHTD(final FileSystem fs, final Path p, final HTableDescriptor htd)
throws IOException {
FSDataOutputStream out = fs.create(p, false);
try {
htd.write(out);
out.write('\n');
out.write('\n');
out.write(Bytes.toBytes(htd.toString()));
} finally {
out.close();
}
}
@Test
public void testBaseCases() throws Exception {
TableName tableName = TableName.valueOf("baseCases");
if (TEST_UTIL.getHBaseAdmin().tableExists(tableName)) {
TEST_UTIL.deleteTable(tableName);
}
Table t = TEST_UTIL.createTable(tableName, F, 1);
// set the version override to 2
Put p = new Put(R);
p.setAttribute("versions", new byte[] {});
p.add(F, tableName.getName(), Bytes.toBytes(2));
t.put(p);
long now = EnvironmentEdgeManager.currentTime();
// insert 2 versions
p = new Put(R);
p.add(F, Q, now, Q);
t.put(p);
p = new Put(R);
p.add(F, Q, now + 1, Q);
t.put(p);
Get g = new Get(R);
g.setMaxVersions(10);
Result r = t.get(g);
assertEquals(2, r.size());
TEST_UTIL.flush(tableName);
TEST_UTIL.compact(tableName, true);
// both version are still visible even after a flush/compaction
g = new Get(R);
g.setMaxVersions(10);
r = t.get(g);
assertEquals(2, r.size());
// insert a 3rd version
p = new Put(R);
p.add(F, Q, now + 2, Q);
t.put(p);
g = new Get(R);
g.setMaxVersions(10);
r = t.get(g);
// still only two version visible
assertEquals(2, r.size());
t.close();
}
@Test
public void testLegalHTableNames() {
for (String tn : legalTableNames) {
TableName.isLegalFullyQualifiedTableName(Bytes.toBytes(tn));
}
}
/**
* Verifies the result from get or scan using the dataGenerator (that was presumably also used to
* generate said result).
*
* @param verifyValues verify that values in the result make sense for row/cf/column combination
* @param verifyCfAndColumnIntegrity verify that cf/column set in the result is complete. Note
* that to use this multiPut should be used, or verification has to happen after writes,
* otherwise there can be races.
* @return
*/
public boolean verifyResultAgainstDataGenerator(
Result result, boolean verifyValues, boolean verifyCfAndColumnIntegrity) {
String rowKeyStr = Bytes.toString(result.getRow());
// See if we have any data at all.
if (result.isEmpty()) {
LOG.error("Error checking data for key [" + rowKeyStr + "], no data returned");
printLocations(result);
return false;
}
if (!verifyValues && !verifyCfAndColumnIntegrity) {
return true; // as long as we have something, we are good.
}
// See if we have all the CFs.
byte[][] expectedCfs = dataGenerator.getColumnFamilies();
if (verifyCfAndColumnIntegrity && (expectedCfs.length != result.getMap().size())) {
LOG.error(
"Error checking data for key ["
+ rowKeyStr
+ "], bad family count: "
+ result.getMap().size());
printLocations(result);
return false;
}
// Verify each column family from get in the result.
for (byte[] cf : result.getMap().keySet()) {
String cfStr = Bytes.toString(cf);
Map<byte[], byte[]> columnValues = result.getFamilyMap(cf);
if (columnValues == null) {
LOG.error(
"Error checking data for key [" + rowKeyStr + "], no data for family [" + cfStr + "]]");
printLocations(result);
return false;
}
Map<String, MutationType> mutateInfo = null;
if (verifyCfAndColumnIntegrity || verifyValues) {
if (!columnValues.containsKey(MUTATE_INFO)) {
LOG.error(
"Error checking data for key ["
+ rowKeyStr
+ "], column family ["
+ cfStr
+ "], column ["
+ Bytes.toString(MUTATE_INFO)
+ "]; value is not found");
printLocations(result);
return false;
}
long cfHash = Arrays.hashCode(cf);
// Verify deleted columns, and make up column counts if deleted
byte[] mutateInfoValue = columnValues.remove(MUTATE_INFO);
mutateInfo = parseMutateInfo(mutateInfoValue);
for (Map.Entry<String, MutationType> mutate : mutateInfo.entrySet()) {
if (mutate.getValue() == MutationType.DELETE) {
byte[] column = Bytes.toBytes(mutate.getKey());
long columnHash = Arrays.hashCode(column);
long hashCode = cfHash + columnHash;
if (hashCode % 2 == 0) {
if (columnValues.containsKey(column)) {
LOG.error(
"Error checking data for key ["
+ rowKeyStr
+ "], column family ["
+ cfStr
+ "], column ["
+ mutate.getKey()
+ "]; should be deleted");
printLocations(result);
return false;
}
byte[] hashCodeBytes = Bytes.toBytes(hashCode);
columnValues.put(column, hashCodeBytes);
}
}
}
// Verify increment
if (!columnValues.containsKey(INCREMENT)) {
LOG.error(
"Error checking data for key ["
+ rowKeyStr
+ "], column family ["
+ cfStr
+ "], column ["
+ Bytes.toString(INCREMENT)
+ "]; value is not found");
printLocations(result);
return false;
}
long currentValue = Bytes.toLong(columnValues.remove(INCREMENT));
if (verifyValues) {
long amount = mutateInfo.isEmpty() ? 0 : cfHash;
long originalValue = Arrays.hashCode(result.getRow());
long extra = currentValue - originalValue;
if (extra != 0 && (amount == 0 || extra % amount != 0)) {
LOG.error(
"Error checking data for key ["
+ rowKeyStr
+ "], column family ["
+ cfStr
+ "], column [increment], extra ["
+ extra
+ "], amount ["
+ amount
+ "]");
printLocations(result);
return false;
}
if (amount != 0 && extra != amount) {
LOG.warn(
"Warning checking data for key ["
+ rowKeyStr
+ "], column family ["
+ cfStr
+ "], column [increment], incremented ["
+ (extra / amount)
+ "] times");
}
}
// See if we have correct columns.
if (verifyCfAndColumnIntegrity
&& !dataGenerator.verify(result.getRow(), cf, columnValues.keySet())) {
String colsStr = "";
for (byte[] col : columnValues.keySet()) {
if (colsStr.length() > 0) {
colsStr += ", ";
}
colsStr += "[" + Bytes.toString(col) + "]";
}
LOG.error(
"Error checking data for key ["
+ rowKeyStr
+ "], bad columns for family ["
+ cfStr
+ "]: "
+ colsStr);
printLocations(result);
return false;
}
// See if values check out.
if (verifyValues) {
for (Map.Entry<byte[], byte[]> kv : columnValues.entrySet()) {
String column = Bytes.toString(kv.getKey());
MutationType mutation = mutateInfo.get(column);
boolean verificationNeeded = true;
byte[] bytes = kv.getValue();
if (mutation != null) {
boolean mutationVerified = true;
long columnHash = Arrays.hashCode(kv.getKey());
long hashCode = cfHash + columnHash;
byte[] hashCodeBytes = Bytes.toBytes(hashCode);
if (mutation == MutationType.APPEND) {
int offset = bytes.length - hashCodeBytes.length;
mutationVerified =
offset > 0
&& Bytes.equals(
hashCodeBytes,
0,
hashCodeBytes.length,
bytes,
offset,
hashCodeBytes.length);
if (mutationVerified) {
int n = 1;
while (true) {
int newOffset = offset - hashCodeBytes.length;
if (newOffset < 0
|| !Bytes.equals(
hashCodeBytes,
0,
hashCodeBytes.length,
bytes,
newOffset,
hashCodeBytes.length)) {
break;
}
offset = newOffset;
n++;
}
if (n > 1) {
LOG.warn(
"Warning checking data for key ["
+ rowKeyStr
+ "], column family ["
+ cfStr
+ "], column ["
+ column
+ "], appended ["
+ n
+ "] times");
}
byte[] dest = new byte[offset];
System.arraycopy(bytes, 0, dest, 0, offset);
bytes = dest;
}
} else if (hashCode % 2 == 0) { // checkAndPut
mutationVerified = Bytes.equals(bytes, hashCodeBytes);
verificationNeeded = false;
}
if (!mutationVerified) {
LOG.error(
"Error checking data for key ["
+ rowKeyStr
+ "], mutation checking failed for column family ["
+ cfStr
+ "], column ["
+ column
+ "]; mutation ["
+ mutation
+ "], hashCode ["
+ hashCode
+ "], verificationNeeded ["
+ verificationNeeded
+ "]");
printLocations(result);
return false;
}
} // end of mutation checking
if (verificationNeeded
&& !dataGenerator.verify(result.getRow(), cf, kv.getKey(), bytes)) {
LOG.error(
"Error checking data for key ["
+ rowKeyStr
+ "], column family ["
+ cfStr
+ "], column ["
+ column
+ "], mutation ["
+ mutation
+ "]; value of length "
+ bytes.length);
printLocations(result);
return false;
}
}
}
}
}
return true;
}
public RowId(String key, long createTime, long modifiedTime, byte state) {
init(Bytes.toBytes(key), createTime, modifiedTime, state);
}
/**
* A command-line utility that reads, writes, and verifies data. Unlike {@link
* PerformanceEvaluation}, this tool validates the data written, and supports simultaneously writing
* and reading the same set of keys.
*/
public class LoadTestTool extends AbstractHBaseTool {
private static final Log LOG = LogFactory.getLog(LoadTestTool.class);
/** Table name for the test */
protected byte[] tableName;
/** Table name to use of not overridden on the command line */
protected static final String DEFAULT_TABLE_NAME = "cluster_test";
/** Column family used by the test */
protected static byte[] COLUMN_FAMILY = Bytes.toBytes("test_cf");
/** Column families used by the test */
protected static final byte[][] COLUMN_FAMILIES = {COLUMN_FAMILY};
/** The default data size if not specified */
protected static final int DEFAULT_DATA_SIZE = 64;
/** The number of reader/writer threads if not specified */
protected static final int DEFAULT_NUM_THREADS = 20;
/** Usage string for the load option */
protected static final String OPT_USAGE_LOAD =
"<avg_cols_per_key>:<avg_data_size>" + "[:<#threads=" + DEFAULT_NUM_THREADS + ">]";
/** Usa\ge string for the read option */
protected static final String OPT_USAGE_READ =
"<verify_percent>[:<#threads=" + DEFAULT_NUM_THREADS + ">]";
protected static final String OPT_USAGE_BLOOM =
"Bloom filter type, one of " + Arrays.toString(StoreFile.BloomType.values());
protected static final String OPT_USAGE_COMPRESSION =
"Compression type, " + "one of " + Arrays.toString(Compression.Algorithm.values());
public static final String OPT_DATA_BLOCK_ENCODING_USAGE =
"Encoding algorithm (e.g. prefix "
+ "compression) to use for data blocks in the test column family, "
+ "one of "
+ Arrays.toString(DataBlockEncoding.values())
+ ".";
public static final String OPT_INMEMORY = "in_memory";
public static final String OPT_USAGE_IN_MEMORY =
"Tries to keep the HFiles of the CF "
+ "inmemory as far as possible. Not guaranteed that reads are always served from inmemory";
private static final String OPT_BLOOM = "bloom";
private static final String OPT_COMPRESSION = "compression";
public static final String OPT_DATA_BLOCK_ENCODING =
HColumnDescriptor.DATA_BLOCK_ENCODING.toLowerCase();
public static final String OPT_ENCODE_IN_CACHE_ONLY = "encode_in_cache_only";
public static final String OPT_ENCODE_IN_CACHE_ONLY_USAGE =
"If this is specified, data blocks will only be encoded in block " + "cache but not on disk";
protected static final String OPT_KEY_WINDOW = "key_window";
protected static final String OPT_WRITE = "write";
protected static final String OPT_MAX_READ_ERRORS = "max_read_errors";
protected static final String OPT_MULTIPUT = "multiput";
protected static final String OPT_NUM_KEYS = "num_keys";
protected static final String OPT_READ = "read";
protected static final String OPT_START_KEY = "start_key";
protected static final String OPT_TABLE_NAME = "tn";
protected static final String OPT_ZK_QUORUM = "zk";
protected static final String OPT_SKIP_INIT = "skip_init";
protected static final String OPT_INIT_ONLY = "init_only";
private static final String NUM_TABLES = "num_tables";
protected static final long DEFAULT_START_KEY = 0;
/** This will be removed as we factor out the dependency on command line */
protected CommandLine cmd;
protected MultiThreadedWriter writerThreads = null;
protected MultiThreadedReader readerThreads = null;
protected long startKey, endKey;
protected boolean isWrite, isRead;
// Column family options
protected DataBlockEncoding dataBlockEncodingAlgo;
protected boolean encodeInCacheOnly;
protected Compression.Algorithm compressAlgo;
protected StoreFile.BloomType bloomType;
private boolean inMemoryCF;
// Writer options
protected int numWriterThreads = DEFAULT_NUM_THREADS;
protected int minColsPerKey, maxColsPerKey;
protected int minColDataSize = DEFAULT_DATA_SIZE, maxColDataSize = DEFAULT_DATA_SIZE;
protected boolean isMultiPut;
// Reader options
private int numReaderThreads = DEFAULT_NUM_THREADS;
private int keyWindow = MultiThreadedReader.DEFAULT_KEY_WINDOW;
private int maxReadErrors = MultiThreadedReader.DEFAULT_MAX_ERRORS;
private int verifyPercent;
private int numTables = 1;
// TODO: refactor LoadTestToolImpl somewhere to make the usage from tests less bad,
// console tool itself should only be used from console.
protected boolean isSkipInit = false;
protected boolean isInitOnly = false;
protected String[] splitColonSeparated(String option, int minNumCols, int maxNumCols) {
String optVal = cmd.getOptionValue(option);
String[] cols = optVal.split(":");
if (cols.length < minNumCols || cols.length > maxNumCols) {
throw new IllegalArgumentException(
"Expected at least "
+ minNumCols
+ " columns but no more than "
+ maxNumCols
+ " in the colon-separated value '"
+ optVal
+ "' of the "
+ "-"
+ option
+ " option");
}
return cols;
}
protected int getNumThreads(String numThreadsStr) {
return parseInt(numThreadsStr, 1, Short.MAX_VALUE);
}
/** Apply column family options such as Bloom filters, compression, and data block encoding. */
protected void applyColumnFamilyOptions(byte[] tableName, byte[][] columnFamilies)
throws IOException {
HBaseAdmin admin = new HBaseAdmin(conf);
HTableDescriptor tableDesc = admin.getTableDescriptor(tableName);
LOG.info("Disabling table " + Bytes.toString(tableName));
admin.disableTable(tableName);
for (byte[] cf : columnFamilies) {
HColumnDescriptor columnDesc = tableDesc.getFamily(cf);
boolean isNewCf = columnDesc == null;
if (isNewCf) {
columnDesc = new HColumnDescriptor(cf);
}
if (bloomType != null) {
columnDesc.setBloomFilterType(bloomType);
}
if (compressAlgo != null) {
columnDesc.setCompressionType(compressAlgo);
}
if (dataBlockEncodingAlgo != null) {
columnDesc.setDataBlockEncoding(dataBlockEncodingAlgo);
columnDesc.setEncodeOnDisk(!encodeInCacheOnly);
}
if (inMemoryCF) {
columnDesc.setInMemory(inMemoryCF);
}
if (isNewCf) {
admin.addColumn(tableName, columnDesc);
} else {
admin.modifyColumn(tableName, columnDesc);
}
}
LOG.info("Enabling table " + Bytes.toString(tableName));
admin.enableTable(tableName);
}
@Override
protected void addOptions() {
addOptWithArg(
OPT_ZK_QUORUM, "ZK quorum as comma-separated host names " + "without port numbers");
addOptWithArg(OPT_TABLE_NAME, "The name of the table to read or write");
addOptWithArg(OPT_WRITE, OPT_USAGE_LOAD);
addOptWithArg(OPT_READ, OPT_USAGE_READ);
addOptNoArg(OPT_INIT_ONLY, "Initialize the test table only, don't do any loading");
addOptWithArg(OPT_BLOOM, OPT_USAGE_BLOOM);
addOptWithArg(OPT_COMPRESSION, OPT_USAGE_COMPRESSION);
addOptWithArg(OPT_DATA_BLOCK_ENCODING, OPT_DATA_BLOCK_ENCODING_USAGE);
addOptWithArg(
OPT_MAX_READ_ERRORS,
"The maximum number of read errors "
+ "to tolerate before terminating all reader threads. The default is "
+ MultiThreadedReader.DEFAULT_MAX_ERRORS
+ ".");
addOptWithArg(
OPT_KEY_WINDOW,
"The 'key window' to maintain between "
+ "reads and writes for concurrent write/read workload. The default "
+ "is "
+ MultiThreadedReader.DEFAULT_KEY_WINDOW
+ ".");
addOptNoArg(
OPT_MULTIPUT,
"Whether to use multi-puts as opposed to " + "separate puts for every column in a row");
addOptNoArg(OPT_ENCODE_IN_CACHE_ONLY, OPT_ENCODE_IN_CACHE_ONLY_USAGE);
addOptNoArg(OPT_INMEMORY, OPT_USAGE_IN_MEMORY);
addOptWithArg(OPT_NUM_KEYS, "The number of keys to read/write");
addOptWithArg(
OPT_START_KEY,
"The first key to read/write "
+ "(a 0-based index). The default value is "
+ DEFAULT_START_KEY
+ ".");
addOptNoArg(OPT_SKIP_INIT, "Skip the initialization; assume test table " + "already exists");
addOptWithArg(
NUM_TABLES,
"A positive integer number. When a number n is speicfied, load test "
+ "tool will load n table parallely. -tn parameter value becomes "
+ "table name prefix. Each table name is in format <tn>_1...<tn>_n");
}
@Override
protected void processOptions(CommandLine cmd) {
this.cmd = cmd;
tableName = Bytes.toBytes(cmd.getOptionValue(OPT_TABLE_NAME, DEFAULT_TABLE_NAME));
isWrite = cmd.hasOption(OPT_WRITE);
isRead = cmd.hasOption(OPT_READ);
isInitOnly = cmd.hasOption(OPT_INIT_ONLY);
if (!isWrite && !isRead && !isInitOnly) {
throw new IllegalArgumentException(
"Either -" + OPT_WRITE + " or " + "-" + OPT_READ + " has to be specified");
}
if (isInitOnly && (isRead || isWrite)) {
throw new IllegalArgumentException(
OPT_INIT_ONLY
+ " cannot be specified with"
+ " either -"
+ OPT_WRITE
+ " or -"
+ OPT_READ);
}
if (!isInitOnly) {
if (!cmd.hasOption(OPT_NUM_KEYS)) {
throw new IllegalArgumentException(
OPT_NUM_KEYS + " must be specified in " + "read or write mode");
}
startKey =
parseLong(
cmd.getOptionValue(OPT_START_KEY, String.valueOf(DEFAULT_START_KEY)),
0,
Long.MAX_VALUE);
long numKeys = parseLong(cmd.getOptionValue(OPT_NUM_KEYS), 1, Long.MAX_VALUE - startKey);
endKey = startKey + numKeys;
isSkipInit = cmd.hasOption(OPT_SKIP_INIT);
System.out.println("Key range: [" + startKey + ".." + (endKey - 1) + "]");
}
encodeInCacheOnly = cmd.hasOption(OPT_ENCODE_IN_CACHE_ONLY);
parseColumnFamilyOptions(cmd);
if (isWrite) {
String[] writeOpts = splitColonSeparated(OPT_WRITE, 2, 3);
int colIndex = 0;
minColsPerKey = 1;
maxColsPerKey = 2 * Integer.parseInt(writeOpts[colIndex++]);
int avgColDataSize = parseInt(writeOpts[colIndex++], 1, Integer.MAX_VALUE);
minColDataSize = avgColDataSize / 2;
maxColDataSize = avgColDataSize * 3 / 2;
if (colIndex < writeOpts.length) {
numWriterThreads = getNumThreads(writeOpts[colIndex++]);
}
isMultiPut = cmd.hasOption(OPT_MULTIPUT);
System.out.println("Multi-puts: " + isMultiPut);
System.out.println("Columns per key: " + minColsPerKey + ".." + maxColsPerKey);
System.out.println("Data size per column: " + minColDataSize + ".." + maxColDataSize);
}
if (isRead) {
String[] readOpts = splitColonSeparated(OPT_READ, 1, 2);
int colIndex = 0;
verifyPercent = parseInt(readOpts[colIndex++], 0, 100);
if (colIndex < readOpts.length) {
numReaderThreads = getNumThreads(readOpts[colIndex++]);
}
if (cmd.hasOption(OPT_MAX_READ_ERRORS)) {
maxReadErrors = parseInt(cmd.getOptionValue(OPT_MAX_READ_ERRORS), 0, Integer.MAX_VALUE);
}
if (cmd.hasOption(OPT_KEY_WINDOW)) {
keyWindow = parseInt(cmd.getOptionValue(OPT_KEY_WINDOW), 0, Integer.MAX_VALUE);
}
System.out.println("Percent of keys to verify: " + verifyPercent);
System.out.println("Reader threads: " + numReaderThreads);
}
numTables = 1;
if (cmd.hasOption(NUM_TABLES)) {
numTables = parseInt(cmd.getOptionValue(NUM_TABLES), 1, Short.MAX_VALUE);
}
}
protected void parseColumnFamilyOptions(CommandLine cmd) {
String dataBlockEncodingStr = cmd.getOptionValue(OPT_DATA_BLOCK_ENCODING);
dataBlockEncodingAlgo =
dataBlockEncodingStr == null ? null : DataBlockEncoding.valueOf(dataBlockEncodingStr);
if (dataBlockEncodingAlgo == DataBlockEncoding.NONE && encodeInCacheOnly) {
throw new IllegalArgumentException(
"-"
+ OPT_ENCODE_IN_CACHE_ONLY
+ " "
+ "does not make sense when data block encoding is not used");
}
String compressStr = cmd.getOptionValue(OPT_COMPRESSION);
compressAlgo =
compressStr == null
? Compression.Algorithm.NONE
: Compression.Algorithm.valueOf(compressStr);
String bloomStr = cmd.getOptionValue(OPT_BLOOM);
bloomType = bloomStr == null ? null : StoreFile.BloomType.valueOf(bloomStr);
inMemoryCF = cmd.hasOption(OPT_INMEMORY);
}
public void initTestTable() throws IOException {
HBaseTestingUtility.createPreSplitLoadTestTable(
conf, tableName, COLUMN_FAMILY, compressAlgo, dataBlockEncodingAlgo);
applyColumnFamilyOptions(tableName, COLUMN_FAMILIES);
}
@Override
protected int doWork() throws IOException {
if (numTables > 1) {
return parallelLoadTables();
} else {
return loadTable();
}
}
protected int loadTable() throws IOException {
if (cmd.hasOption(OPT_ZK_QUORUM)) {
conf.set(HConstants.ZOOKEEPER_QUORUM, cmd.getOptionValue(OPT_ZK_QUORUM));
}
if (isInitOnly) {
LOG.info("Initializing only; no reads or writes");
initTestTable();
return 0;
}
if (!isSkipInit) {
initTestTable();
}
LoadTestDataGenerator dataGen =
new MultiThreadedAction.DefaultDataGenerator(
minColDataSize, maxColDataSize, minColsPerKey, maxColsPerKey, COLUMN_FAMILY);
if (isWrite) {
writerThreads = new MultiThreadedWriter(dataGen, conf, tableName);
writerThreads.setMultiPut(isMultiPut);
}
if (isRead) {
readerThreads = new MultiThreadedReader(dataGen, conf, tableName, verifyPercent);
readerThreads.setMaxErrors(maxReadErrors);
readerThreads.setKeyWindow(keyWindow);
}
if (isRead && isWrite) {
LOG.info("Concurrent read/write workload: making readers aware of the " + "write point");
readerThreads.linkToWriter(writerThreads);
}
if (isWrite) {
System.out.println("Starting to write data...");
writerThreads.start(startKey, endKey, numWriterThreads);
}
if (isRead) {
System.out.println("Starting to read data...");
readerThreads.start(startKey, endKey, numReaderThreads);
}
if (isWrite) {
writerThreads.waitForFinish();
}
if (isRead) {
readerThreads.waitForFinish();
}
boolean success = true;
if (isWrite) {
success = success && writerThreads.getNumWriteFailures() == 0;
}
if (isRead) {
success =
success
&& readerThreads.getNumReadErrors() == 0
&& readerThreads.getNumReadFailures() == 0;
}
return success ? EXIT_SUCCESS : this.EXIT_FAILURE;
}
public static void main(String[] args) {
new LoadTestTool().doStaticMain(args);
}
/**
* When NUM_TABLES is specified, the function starts multiple worker threads which individually
* start a LoadTestTool instance to load a table. Each table name is in format <tn>_<index>. For
* example, "-tn test -num_tables 2" , table names will be "test_1", "test_2"
*
* @throws IOException
*/
private int parallelLoadTables() throws IOException {
// create new command args
String tableName = cmd.getOptionValue(OPT_TABLE_NAME, DEFAULT_TABLE_NAME);
String[] newArgs = null;
if (!cmd.hasOption(LoadTestTool.OPT_TABLE_NAME)) {
newArgs = new String[cmdLineArgs.length + 2];
newArgs[0] = "-" + LoadTestTool.OPT_TABLE_NAME;
for (int i = 0; i < cmdLineArgs.length; i++) {
newArgs[i + 2] = cmdLineArgs[i];
}
} else {
newArgs = cmdLineArgs;
}
int tableNameValueIndex = -1;
for (int j = 0; j < newArgs.length; j++) {
if (newArgs[j].endsWith(OPT_TABLE_NAME)) {
tableNameValueIndex = j + 1;
} else if (newArgs[j].endsWith(NUM_TABLES)) {
// change NUM_TABLES to 1 so that each worker loads one table
newArgs[j + 1] = "1";
}
}
// starting to load multiple tables
List<WorkerThread> workers = new ArrayList<WorkerThread>();
for (int i = 0; i < numTables; i++) {
String[] workerArgs = newArgs.clone();
workerArgs[tableNameValueIndex] = tableName + "_" + (i + 1);
WorkerThread worker = new WorkerThread(i, workerArgs);
workers.add(worker);
LOG.info(worker + " starting");
worker.start();
}
// wait for all workers finish
LOG.info("Waiting for worker threads to finish");
for (WorkerThread t : workers) {
try {
t.join();
} catch (InterruptedException ie) {
IOException iie = new InterruptedIOException();
iie.initCause(ie);
throw iie;
}
checkForErrors();
}
return EXIT_SUCCESS;
}
// If an exception is thrown by one of worker threads, it will be
// stored here.
protected AtomicReference<Throwable> thrown = new AtomicReference<Throwable>();
private void workerThreadError(Throwable t) {
thrown.compareAndSet(null, t);
}
/** Check for errors in the writer threads. If any is found, rethrow it. */
private void checkForErrors() throws IOException {
Throwable thrown = this.thrown.get();
if (thrown == null) return;
if (thrown instanceof IOException) {
throw (IOException) thrown;
} else {
throw new RuntimeException(thrown);
}
}
class WorkerThread extends Thread {
private String[] workerArgs;
WorkerThread(int i, String[] args) {
super("WorkerThread-" + i);
workerArgs = args;
}
public void run() {
try {
int ret = ToolRunner.run(HBaseConfiguration.create(), new LoadTestTool(), workerArgs);
if (ret != 0) {
throw new RuntimeException("LoadTestTool exit with non-zero return code.");
}
} catch (Exception ex) {
LOG.error("Error in worker thread", ex);
workerThreadError(ex);
}
}
}
}
@Override
protected void processOptions(CommandLine cmd) {
this.cmd = cmd;
tableName = Bytes.toBytes(cmd.getOptionValue(OPT_TABLE_NAME, DEFAULT_TABLE_NAME));
isWrite = cmd.hasOption(OPT_WRITE);
isRead = cmd.hasOption(OPT_READ);
isInitOnly = cmd.hasOption(OPT_INIT_ONLY);
if (!isWrite && !isRead && !isInitOnly) {
throw new IllegalArgumentException(
"Either -" + OPT_WRITE + " or " + "-" + OPT_READ + " has to be specified");
}
if (isInitOnly && (isRead || isWrite)) {
throw new IllegalArgumentException(
OPT_INIT_ONLY
+ " cannot be specified with"
+ " either -"
+ OPT_WRITE
+ " or -"
+ OPT_READ);
}
if (!isInitOnly) {
if (!cmd.hasOption(OPT_NUM_KEYS)) {
throw new IllegalArgumentException(
OPT_NUM_KEYS + " must be specified in " + "read or write mode");
}
startKey =
parseLong(
cmd.getOptionValue(OPT_START_KEY, String.valueOf(DEFAULT_START_KEY)),
0,
Long.MAX_VALUE);
long numKeys = parseLong(cmd.getOptionValue(OPT_NUM_KEYS), 1, Long.MAX_VALUE - startKey);
endKey = startKey + numKeys;
isSkipInit = cmd.hasOption(OPT_SKIP_INIT);
System.out.println("Key range: [" + startKey + ".." + (endKey - 1) + "]");
}
encodeInCacheOnly = cmd.hasOption(OPT_ENCODE_IN_CACHE_ONLY);
parseColumnFamilyOptions(cmd);
if (isWrite) {
String[] writeOpts = splitColonSeparated(OPT_WRITE, 2, 3);
int colIndex = 0;
minColsPerKey = 1;
maxColsPerKey = 2 * Integer.parseInt(writeOpts[colIndex++]);
int avgColDataSize = parseInt(writeOpts[colIndex++], 1, Integer.MAX_VALUE);
minColDataSize = avgColDataSize / 2;
maxColDataSize = avgColDataSize * 3 / 2;
if (colIndex < writeOpts.length) {
numWriterThreads = getNumThreads(writeOpts[colIndex++]);
}
isMultiPut = cmd.hasOption(OPT_MULTIPUT);
System.out.println("Multi-puts: " + isMultiPut);
System.out.println("Columns per key: " + minColsPerKey + ".." + maxColsPerKey);
System.out.println("Data size per column: " + minColDataSize + ".." + maxColDataSize);
}
if (isRead) {
String[] readOpts = splitColonSeparated(OPT_READ, 1, 2);
int colIndex = 0;
verifyPercent = parseInt(readOpts[colIndex++], 0, 100);
if (colIndex < readOpts.length) {
numReaderThreads = getNumThreads(readOpts[colIndex++]);
}
if (cmd.hasOption(OPT_MAX_READ_ERRORS)) {
maxReadErrors = parseInt(cmd.getOptionValue(OPT_MAX_READ_ERRORS), 0, Integer.MAX_VALUE);
}
if (cmd.hasOption(OPT_KEY_WINDOW)) {
keyWindow = parseInt(cmd.getOptionValue(OPT_KEY_WINDOW), 0, Integer.MAX_VALUE);
}
System.out.println("Percent of keys to verify: " + verifyPercent);
System.out.println("Reader threads: " + numReaderThreads);
}
numTables = 1;
if (cmd.hasOption(NUM_TABLES)) {
numTables = parseInt(cmd.getOptionValue(NUM_TABLES), 1, Short.MAX_VALUE);
}
}
@Category({MiscTests.class, MediumTests.class})
@RunWith(Parameterized.class)
public class TestCoprocessorScanPolicy {
final Log LOG = LogFactory.getLog(getClass());
protected static final HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();
private static final byte[] F = Bytes.toBytes("fam");
private static final byte[] Q = Bytes.toBytes("qual");
private static final byte[] R = Bytes.toBytes("row");
@BeforeClass
public static void setUpBeforeClass() throws Exception {
Configuration conf = TEST_UTIL.getConfiguration();
conf.setStrings(CoprocessorHost.REGION_COPROCESSOR_CONF_KEY, ScanObserver.class.getName());
TEST_UTIL.startMiniCluster();
}
@AfterClass
public static void tearDownAfterClass() throws Exception {
TEST_UTIL.shutdownMiniCluster();
}
@Parameters
public static Collection<Object[]> parameters() {
return HBaseTestingUtility.BOOLEAN_PARAMETERIZED;
}
public TestCoprocessorScanPolicy(boolean parallelSeekEnable) {
TEST_UTIL
.getMiniHBaseCluster()
.getConf()
.setBoolean(StoreScanner.STORESCANNER_PARALLEL_SEEK_ENABLE, parallelSeekEnable);
}
@Test
public void testBaseCases() throws Exception {
TableName tableName = TableName.valueOf("baseCases");
if (TEST_UTIL.getHBaseAdmin().tableExists(tableName)) {
TEST_UTIL.deleteTable(tableName);
}
Table t = TEST_UTIL.createTable(tableName, F, 1);
// set the version override to 2
Put p = new Put(R);
p.setAttribute("versions", new byte[] {});
p.add(F, tableName.getName(), Bytes.toBytes(2));
t.put(p);
long now = EnvironmentEdgeManager.currentTime();
// insert 2 versions
p = new Put(R);
p.add(F, Q, now, Q);
t.put(p);
p = new Put(R);
p.add(F, Q, now + 1, Q);
t.put(p);
Get g = new Get(R);
g.setMaxVersions(10);
Result r = t.get(g);
assertEquals(2, r.size());
TEST_UTIL.flush(tableName);
TEST_UTIL.compact(tableName, true);
// both version are still visible even after a flush/compaction
g = new Get(R);
g.setMaxVersions(10);
r = t.get(g);
assertEquals(2, r.size());
// insert a 3rd version
p = new Put(R);
p.add(F, Q, now + 2, Q);
t.put(p);
g = new Get(R);
g.setMaxVersions(10);
r = t.get(g);
// still only two version visible
assertEquals(2, r.size());
t.close();
}
@Test
public void testTTL() throws Exception {
TableName tableName = TableName.valueOf("testTTL");
if (TEST_UTIL.getHBaseAdmin().tableExists(tableName)) {
TEST_UTIL.deleteTable(tableName);
}
HTableDescriptor desc = new HTableDescriptor(tableName);
HColumnDescriptor hcd = new HColumnDescriptor(F).setMaxVersions(10).setTimeToLive(1);
desc.addFamily(hcd);
TEST_UTIL.getHBaseAdmin().createTable(desc);
Table t = new HTable(new Configuration(TEST_UTIL.getConfiguration()), tableName);
long now = EnvironmentEdgeManager.currentTime();
ManualEnvironmentEdge me = new ManualEnvironmentEdge();
me.setValue(now);
EnvironmentEdgeManagerTestHelper.injectEdge(me);
// 2s in the past
long ts = now - 2000;
// Set the TTL override to 3s
Put p = new Put(R);
p.setAttribute("ttl", new byte[] {});
p.add(F, tableName.getName(), Bytes.toBytes(3000L));
t.put(p);
p = new Put(R);
p.add(F, Q, ts, Q);
t.put(p);
p = new Put(R);
p.add(F, Q, ts + 1, Q);
t.put(p);
// these two should be expired but for the override
// (their ts was 2s in the past)
Get g = new Get(R);
g.setMaxVersions(10);
Result r = t.get(g);
// still there?
assertEquals(2, r.size());
TEST_UTIL.flush(tableName);
TEST_UTIL.compact(tableName, true);
g = new Get(R);
g.setMaxVersions(10);
r = t.get(g);
// still there?
assertEquals(2, r.size());
// roll time forward 2s.
me.setValue(now + 2000);
// now verify that data eventually does expire
g = new Get(R);
g.setMaxVersions(10);
r = t.get(g);
// should be gone now
assertEquals(0, r.size());
t.close();
}
public static class ScanObserver extends BaseRegionObserver {
private Map<TableName, Long> ttls = new HashMap<TableName, Long>();
private Map<TableName, Integer> versions = new HashMap<TableName, Integer>();
// lame way to communicate with the coprocessor,
// since it is loaded by a different class loader
@Override
public void prePut(
final ObserverContext<RegionCoprocessorEnvironment> c,
final Put put,
final WALEdit edit,
final Durability durability)
throws IOException {
if (put.getAttribute("ttl") != null) {
Cell cell = put.getFamilyCellMap().values().iterator().next().get(0);
KeyValue kv = KeyValueUtil.ensureKeyValue(cell);
ttls.put(TableName.valueOf(kv.getQualifier()), Bytes.toLong(kv.getValue()));
c.bypass();
} else if (put.getAttribute("versions") != null) {
Cell cell = put.getFamilyCellMap().values().iterator().next().get(0);
KeyValue kv = KeyValueUtil.ensureKeyValue(cell);
versions.put(TableName.valueOf(kv.getQualifier()), Bytes.toInt(kv.getValue()));
c.bypass();
}
}
@Override
public InternalScanner preFlushScannerOpen(
final ObserverContext<RegionCoprocessorEnvironment> c,
Store store,
KeyValueScanner memstoreScanner,
InternalScanner s)
throws IOException {
Long newTtl = ttls.get(store.getTableName());
if (newTtl != null) {
System.out.println("PreFlush:" + newTtl);
}
Integer newVersions = versions.get(store.getTableName());
ScanInfo oldSI = store.getScanInfo();
HColumnDescriptor family = store.getFamily();
ScanInfo scanInfo =
new ScanInfo(
family.getName(),
family.getMinVersions(),
newVersions == null ? family.getMaxVersions() : newVersions,
newTtl == null ? oldSI.getTtl() : newTtl,
family.getKeepDeletedCells(),
oldSI.getTimeToPurgeDeletes(),
oldSI.getComparator());
Scan scan = new Scan();
scan.setMaxVersions(newVersions == null ? oldSI.getMaxVersions() : newVersions);
return new StoreScanner(
store,
scanInfo,
scan,
Collections.singletonList(memstoreScanner),
ScanType.COMPACT_RETAIN_DELETES,
store.getSmallestReadPoint(),
HConstants.OLDEST_TIMESTAMP);
}
@Override
public InternalScanner preCompactScannerOpen(
final ObserverContext<RegionCoprocessorEnvironment> c,
Store store,
List<? extends KeyValueScanner> scanners,
ScanType scanType,
long earliestPutTs,
InternalScanner s)
throws IOException {
Long newTtl = ttls.get(store.getTableName());
Integer newVersions = versions.get(store.getTableName());
ScanInfo oldSI = store.getScanInfo();
HColumnDescriptor family = store.getFamily();
ScanInfo scanInfo =
new ScanInfo(
family.getName(),
family.getMinVersions(),
newVersions == null ? family.getMaxVersions() : newVersions,
newTtl == null ? oldSI.getTtl() : newTtl,
family.getKeepDeletedCells(),
oldSI.getTimeToPurgeDeletes(),
oldSI.getComparator());
Scan scan = new Scan();
scan.setMaxVersions(newVersions == null ? oldSI.getMaxVersions() : newVersions);
return new StoreScanner(
store, scanInfo, scan, scanners, scanType, store.getSmallestReadPoint(), earliestPutTs);
}
@Override
public KeyValueScanner preStoreScannerOpen(
final ObserverContext<RegionCoprocessorEnvironment> c,
Store store,
final Scan scan,
final NavigableSet<byte[]> targetCols,
KeyValueScanner s)
throws IOException {
TableName tn = store.getTableName();
if (!tn.isSystemTable()) {
Long newTtl = ttls.get(store.getTableName());
Integer newVersions = versions.get(store.getTableName());
ScanInfo oldSI = store.getScanInfo();
HColumnDescriptor family = store.getFamily();
ScanInfo scanInfo =
new ScanInfo(
family.getName(),
family.getMinVersions(),
newVersions == null ? family.getMaxVersions() : newVersions,
newTtl == null ? oldSI.getTtl() : newTtl,
family.getKeepDeletedCells(),
oldSI.getTimeToPurgeDeletes(),
oldSI.getComparator());
return new StoreScanner(
store,
scanInfo,
scan,
targetCols,
((HStore) store).getHRegion().getReadpoint(IsolationLevel.READ_COMMITTED));
} else {
return s;
}
}
}
}
