public HBaseStoreManager(org.apache.commons.configuration.Configuration config)
throws StorageException {
super(config, PORT_DEFAULT);
this.tableName = config.getString(TABLE_NAME_KEY, TABLE_NAME_DEFAULT);
this.hconf = HBaseConfiguration.create();
for (Map.Entry<String, String> confEntry : HBASE_CONFIGURATION.entrySet()) {
if (config.containsKey(confEntry.getKey())) {
hconf.set(confEntry.getValue(), config.getString(confEntry.getKey()));
}
}
// Copy a subset of our commons config into a Hadoop config
org.apache.commons.configuration.Configuration hbCommons =
config.subset(HBASE_CONFIGURATION_NAMESPACE);
@SuppressWarnings("unchecked") // I hope commons-config eventually fixes this
Iterator<String> keys = hbCommons.getKeys();
int keysLoaded = 0;
while (keys.hasNext()) {
String key = keys.next();
String value = hbCommons.getString(key);
logger.debug("HBase configuration: setting {}={}", key, value);
hconf.set(key, value);
keysLoaded++;
}
logger.debug("HBase configuration: set a total of {} configuration values", keysLoaded);
connectionPool = new HTablePool(hconf, connectionPoolSize);
openStores = new ConcurrentHashMap<String, HBaseKeyColumnValueStore>();
// TODO: allowing publicly mutate fields is bad, should be fixed
features = new StoreFeatures();
features.supportsScan = true;
features.supportsBatchMutation = true;
features.supportsTransactions = false;
features.supportsConsistentKeyOperations = true;
features.supportsLocking = false;
features.isKeyOrdered = false;
features.isDistributed = true;
features.hasLocalKeyPartition = false;
}
public int run(String[] args) throws Exception {
Configuration argConf = getConf();
// JobConf conf = new JobConf(diffdb.class);
Configuration config = HBaseConfiguration.create();
HBaseAdmin hbAdmin = new HBaseAdmin(config);
dbutil db_util = new dbutil(config);
HTable runTable = new HTable(config, "gestore_runs");
Get runGet = new Get(argConf.get("id").getBytes());
Result pipeline = runTable.get(runGet);
NavigableMap<byte[], byte[]> pipeMap = pipeline.getFamilyMap("d".getBytes());
Map.Entry<byte[], byte[]> results = pipeMap.pollFirstEntry();
HashMap<String, HashMap<String, String>> resultMap =
new HashMap<String, HashMap<String, String>>();
while (results != null) {
String resultKey = new String(results.getKey());
String resultValue = new String(results.getValue());
String field = "type";
HashMap<String, String> tempMap = new HashMap<String, String>();
String entry = resultKey;
if (resultKey.endsWith("_db_timestamp")) {
field = "db_timestamp";
entry = resultKey.substring(0, resultKey.lastIndexOf("_db_timestamp"));
} else if (resultKey.endsWith("_filename")) {
field = "filename";
entry = resultKey.substring(0, resultKey.lastIndexOf("_filename"));
} else if (resultKey.endsWith("_regex")) {
field = "regex";
entry = resultKey.substring(0, resultKey.lastIndexOf("_regex"));
}
if (resultMap.containsKey(entry)) {
tempMap = resultMap.get(entry);
}
tempMap.put(field, resultValue);
resultMap.put(entry, tempMap);
// System.out.println("Key: " + resultKey + " Value: " + resultValue);
results = pipeMap.pollFirstEntry();
}
for (String key : resultMap.keySet()) {
System.out.println("File ID: " + key);
for (String subKey : resultMap.get(key).keySet()) {
// System.out.println("\t " + subKey + "\t\t" + resultMap.get(key).get(subKey));
System.out.format(" %1$-20s %2$s\n", subKey, resultMap.get(key).get(subKey));
}
}
return 0;
}
private static String getConfigString(Configuration config) {
String output = "";
Iterator<Map.Entry<String, String>> iterConfig = config.iterator();
while (iterConfig.hasNext()) {
Map.Entry<String, String> curEntry = iterConfig.next();
output = output + "Key: \t" + curEntry.getKey() + "\nValue: \t" + curEntry.getValue() + "\n";
}
return output;
}
/**
* Convert Titan internal Mutation representation into HBase native commands.
*
* @param mutations Mutations to convert into HBase commands.
* @param putTimestamp The timestamp to use for Put commands.
* @param delTimestamp The timestamp to use for Delete commands.
* @return Commands sorted by key converted from Titan internal representation.
*/
private static Map<ByteBuffer, Pair<Put, Delete>> convertToCommands(
Map<String, Map<ByteBuffer, KCVMutation>> mutations,
final long putTimestamp,
final long delTimestamp) {
Map<ByteBuffer, Pair<Put, Delete>> commandsPerKey =
new HashMap<ByteBuffer, Pair<Put, Delete>>();
for (Map.Entry<String, Map<ByteBuffer, KCVMutation>> entry : mutations.entrySet()) {
byte[] cfName = entry.getKey().getBytes();
for (Map.Entry<ByteBuffer, KCVMutation> m : entry.getValue().entrySet()) {
ByteBuffer key = m.getKey();
KCVMutation mutation = m.getValue();
Pair<Put, Delete> commands = commandsPerKey.get(key);
if (commands == null) {
commands = new Pair<Put, Delete>();
commandsPerKey.put(key, commands);
}
if (mutation.hasDeletions()) {
if (commands.getSecond() == null)
commands.setSecond(new Delete(ByteBufferUtil.getArray(key), delTimestamp, null));
for (ByteBuffer b : mutation.getDeletions()) {
commands.getSecond().deleteColumns(cfName, ByteBufferUtil.getArray(b), delTimestamp);
}
}
if (mutation.hasAdditions()) {
if (commands.getFirst() == null)
commands.setFirst(new Put(ByteBufferUtil.getArray(key), putTimestamp));
for (Entry e : mutation.getAdditions()) {
commands
.getFirst()
.add(
cfName,
ByteBufferUtil.getArray(e.getColumn()),
putTimestamp,
ByteBufferUtil.getArray(e.getValue()));
}
}
}
}
return commandsPerKey;
}
public void printStats() throws IOException {
Admin admin = connection.getAdmin();
ClusterStatus status =
admin.getClusterStatus(); // co ClusterStatusExample-1-GetStatus Get the cluster status.
System.out.println("Cluster Status:\n--------------");
System.out.println("HBase Version: " + status.getHBaseVersion());
System.out.println("Version: " + status.getVersion());
System.out.println("Cluster ID: " + status.getClusterId());
System.out.println("Master: " + status.getMaster());
System.out.println("No. Backup Masters: " + status.getBackupMastersSize());
System.out.println("Backup Masters: " + status.getBackupMasters());
System.out.println("No. Live Servers: " + status.getServersSize());
System.out.println("Servers: " + status.getServers());
System.out.println("No. Dead Servers: " + status.getDeadServers());
System.out.println("Dead Servers: " + status.getDeadServerNames());
System.out.println("No. Regions: " + status.getRegionsCount());
System.out.println("Regions in Transition: " + status.getRegionsInTransition());
System.out.println("No. Requests: " + status.getRequestsCount());
System.out.println("Avg Load: " + status.getAverageLoad());
System.out.println("Balancer On: " + status.getBalancerOn());
System.out.println("Is Balancer On: " + status.isBalancerOn());
System.out.println("Master Coprocessors: " + Arrays.asList(status.getMasterCoprocessors()));
System.out.println("\nServer Info:\n--------------");
for (ServerName server :
status
.getServers()) { // co ClusterStatusExample-2-ServerInfo Iterate over the included
// server instances.
System.out.println("Hostname: " + server.getHostname());
System.out.println("Host and Port: " + server.getHostAndPort());
System.out.println("Server Name: " + server.getServerName());
System.out.println("RPC Port: " + server.getPort());
System.out.println("Start Code: " + server.getStartcode());
ServerLoad load =
status.getLoad(
server); // co ClusterStatusExample-3-ServerLoad Retrieve the load details for the
// current server.
System.out.println("\nServer Load:\n--------------");
System.out.println("Info Port: " + load.getInfoServerPort());
System.out.println("Load: " + load.getLoad());
System.out.println("Max Heap (MB): " + load.getMaxHeapMB());
System.out.println("Used Heap (MB): " + load.getUsedHeapMB());
System.out.println("Memstore Size (MB): " + load.getMemstoreSizeInMB());
System.out.println("No. Regions: " + load.getNumberOfRegions());
System.out.println("No. Requests: " + load.getNumberOfRequests());
System.out.println("Total No. Requests: " + load.getTotalNumberOfRequests());
System.out.println("No. Requests per Sec: " + load.getRequestsPerSecond());
System.out.println("No. Read Requests: " + load.getReadRequestsCount());
System.out.println("No. Write Requests: " + load.getWriteRequestsCount());
System.out.println("No. Stores: " + load.getStores());
System.out.println("Store Size Uncompressed (MB): " + load.getStoreUncompressedSizeMB());
System.out.println("No. Storefiles: " + load.getStorefiles());
System.out.println("Storefile Size (MB): " + load.getStorefileSizeInMB());
System.out.println("Storefile Index Size (MB): " + load.getStorefileIndexSizeInMB());
System.out.println("Root Index Size: " + load.getRootIndexSizeKB());
System.out.println("Total Bloom Size: " + load.getTotalStaticBloomSizeKB());
System.out.println("Total Index Size: " + load.getTotalStaticIndexSizeKB());
System.out.println("Current Compacted Cells: " + load.getCurrentCompactedKVs());
System.out.println("Total Compacting Cells: " + load.getTotalCompactingKVs());
System.out.println("Coprocessors1: " + Arrays.asList(load.getRegionServerCoprocessors()));
System.out.println("Coprocessors2: " + Arrays.asList(load.getRsCoprocessors()));
System.out.println("Replication Load Sink: " + load.getReplicationLoadSink());
System.out.println("Replication Load Source: " + load.getReplicationLoadSourceList());
System.out.println("\nRegion Load:\n--------------");
for (Map.Entry<byte[], RegionLoad>
entry : // co ClusterStatusExample-4-Regions Iterate over the region details of the
// current server.
load.getRegionsLoad().entrySet()) {
System.out.println("Region: " + Bytes.toStringBinary(entry.getKey()));
RegionLoad regionLoad =
entry
.getValue(); // co ClusterStatusExample-5-RegionLoad Get the load details for the
// current region.
System.out.println("Name: " + Bytes.toStringBinary(regionLoad.getName()));
System.out.println("Name (as String): " + regionLoad.getNameAsString());
System.out.println("No. Requests: " + regionLoad.getRequestsCount());
System.out.println("No. Read Requests: " + regionLoad.getReadRequestsCount());
System.out.println("No. Write Requests: " + regionLoad.getWriteRequestsCount());
System.out.println("No. Stores: " + regionLoad.getStores());
System.out.println("No. Storefiles: " + regionLoad.getStorefiles());
System.out.println("Data Locality: " + regionLoad.getDataLocality());
System.out.println("Storefile Size (MB): " + regionLoad.getStorefileSizeMB());
System.out.println("Storefile Index Size (MB): " + regionLoad.getStorefileIndexSizeMB());
System.out.println("Memstore Size (MB): " + regionLoad.getMemStoreSizeMB());
System.out.println("Root Index Size: " + regionLoad.getRootIndexSizeKB());
System.out.println("Total Bloom Size: " + regionLoad.getTotalStaticBloomSizeKB());
System.out.println("Total Index Size: " + regionLoad.getTotalStaticIndexSizeKB());
System.out.println("Current Compacted Cells: " + regionLoad.getCurrentCompactedKVs());
System.out.println("Total Compacting Cells: " + regionLoad.getTotalCompactingKVs());
System.out.println();
}
}
}
public QueryResult getSimpleVariantsByRegion(
Region region, String sourceId, QueryOptions options) {
Long start, end, dbstart, dbend;
start = System.currentTimeMillis();
boolean includeStats;
boolean includeEffects;
if (!options.containsKey("stats") && !options.containsKey("effects")) {
includeStats = true;
includeEffects = true;
} else {
includeStats = options.containsKey("stats") && options.getBoolean("stats");
includeEffects = options.containsKey("effects") && options.getBoolean("effects");
}
QueryResult<Variant> queryResult =
new QueryResult<>(
String.format("%s:%d-%d", region.getChromosome(), region.getStart(), region.getEnd()));
List<Variant> results = new ArrayList<>();
String startRow = buildRowkey(region.getChromosome(), Long.toString(region.getStart()));
String stopRow = buildRowkey(region.getChromosome(), Long.toString(region.getEnd()));
BasicDBObject query =
new BasicDBObject("position", new BasicDBObject("$gte", startRow).append("$lte", stopRow))
.append("sources.sourceId", sourceId);
DBCollection collection = db.getCollection("variants");
dbstart = System.currentTimeMillis();
DBCursor variantInStudies = collection.find(query);
dbend = System.currentTimeMillis();
queryResult.setDbTime(dbend - dbstart);
for (DBObject result : variantInStudies) {
String[] rowkeyParts = result.get("position").toString().split("_");
String chromosome = rowkeyParts[0].replaceFirst("^0+(?!$)", "");
int position = Integer.parseInt(rowkeyParts[1]);
BasicDBList studies = (BasicDBList) result.get("sources");
BasicDBObject st = (BasicDBObject) studies.get(0);
String ref = (String) st.get("ref");
String alt = StringUtils.join((ArrayList<String>) st.get("alt"), ",");
// TODO Needs rework
Variant variant = new Variant(chromosome, position, position, ref, alt);
// Set stats informations
if (includeStats) {
VariantStats stats = new VariantStats();
BasicDBObject mongoStats = (BasicDBObject) st.get("stats");
stats.setMaf((float) (double) mongoStats.get("maf"));
stats.setMafAllele((String) mongoStats.get("alleleMaf"));
stats.setMissingGenotypes((int) mongoStats.get("missing"));
List<Genotype> genotypeCount = new ArrayList<>();
for (BasicDBObject s : (List<BasicDBObject>) mongoStats.get("genotypeCount")) {
for (Map.Entry<String, Object> entry : s.entrySet()) {
Genotype genotype = new Genotype(entry.getKey());
genotype.setCount((Integer) entry.getValue());
genotypeCount.add(genotype);
}
}
stats.setGenotypes(genotypeCount);
variant.setStats(stats);
}
// TODO Set consequence type names
if (includeEffects) {
BasicDBList mongoEffects = (BasicDBList) st.get("effects");
if (mongoEffects != null) {
for (Object e : mongoEffects) {
String effectObo = e.toString();
VariantEffect effect = new VariantEffect();
effect.setConsequenceTypeObo(effectObo);
variant.addEffect(effect);
}
}
}
results.add(variant);
}
queryResult.setResult(results);
queryResult.setNumResults(results.size());
end = System.currentTimeMillis();
queryResult.setTime(end - start);
return queryResult;
}