/*
* Should we check all assignments or just not in RIT?
*/
public static void waitUntilAssigned(HBaseAdmin admin, HRegionInfo region)
throws IOException, InterruptedException {
long timeout = admin.getConfiguration().getLong("hbase.hbck.assign.timeout", 120000);
long expiration = timeout + System.currentTimeMillis();
while (System.currentTimeMillis() < expiration) {
try {
Map<String, RegionState> rits = admin.getClusterStatus().getRegionsInTransition();
if (rits.keySet() != null && !rits.keySet().contains(region.getEncodedName())) {
// yay! no longer RIT
return;
}
// still in rit
LOG.info("Region still in transition, waiting for " + "it to become assigned: " + region);
} catch (IOException e) {
LOG.warn("Exception when waiting for region to become assigned," + " retrying", e);
}
Thread.sleep(1000);
}
throw new IOException(
"Region "
+ region
+ " failed to move out of "
+ "transition within timeout "
+ timeout
+ "ms");
}
@BeforeClass
public static void setUpOnce() throws Exception {
miniCluster = System.getProperty("cluster.type").equals("mini");
securedCluster = System.getProperty("cluster.secured").equals("true");
System.out.println("realCluster - " + !miniCluster);
System.out.println("securedCluster - " + securedCluster);
Util.setLoggingThreshold("ERROR");
if (miniCluster) {
if (hbase == null) {
hbase = new HBaseTestingUtility();
conf = hbase.getConfiguration();
conf.set("zookeeper.session.timeout", "3600000");
conf.set("dfs.client.socket-timeout", "3600000");
if (securedCluster) {
hbase.startMiniCluster(RS_COUNT);
hbase.waitTableEnabled(AccessControlLists.ACL_TABLE_NAME, 30000L);
admin = new HBaseAdminWrapper(conf);
} else {
hbase.startMiniCluster(RS_COUNT);
admin = hbase.getHBaseAdmin();
}
}
} else {
if (admin == null) {
final String argsFileName =
securedCluster
? "../../testClusterRealSecured.args"
: "../../testClusterRealNonSecured.args";
if (!Util.isFile(argsFileName)) {
throw new IllegalStateException(
"You have to define args file " + argsFileName + " for tests.");
}
String[] testArgs = {argsFileName};
Args args = new TestArgs(testArgs);
admin = HBaseClient.getAdmin(args);
conf = admin.getConfiguration();
RS_COUNT = getServerNameList().size();
}
}
previousBalancerRunning = admin.setBalancerRunning(false, true);
hConnection = HConnectionManager.createConnection(conf);
USER_RW = User.createUserForTesting(conf, "rwuser", new String[0]);
}
@AfterClass
public static void tearDownAfterClass() throws Exception {
if (lilyProxy != null) lilyProxy.stop();
TestHomeUtil.cleanupTestHome(tmpDir);
if (lilyProxy.getMode() == LilyProxy.Mode.CONNECT
|| lilyProxy.getMode() == LilyProxy.Mode.HADOOP_CONNECT) {
// We're in connect mode, drop the tables again so that the remainder of the tests
// don't have the overhead of the extra splits
Configuration conf = HBaseConfiguration.create();
conf.set("hbase.zookeeper.quorum", "localhost");
HBaseAdmin hbaseAdmin = new HBaseAdmin(conf);
for (String tableName : TABLE_NAMES) {
if (hbaseAdmin.tableExists(tableName)) {
hbaseAdmin.disableTable(tableName);
hbaseAdmin.deleteTable(tableName);
}
}
HConnectionManager.deleteConnection(hbaseAdmin.getConfiguration(), true);
}
}
@BeforeClass
public static void setUpBeforeClass() throws Exception {
lilyProxy = new LilyProxy();
//
// Make multiple record table splits
//
if (lilyProxy.getMode() == LilyProxy.Mode.CONNECT
|| lilyProxy.getMode() == LilyProxy.Mode.HADOOP_CONNECT) {
// The tables will likely already exist and not be recreated, hence we won't be able to change
// the number of regions. Therefore, drop them.
Configuration conf = HBaseConfiguration.create();
conf.set("hbase.zookeeper.quorum", "localhost");
HBaseAdmin hbaseAdmin = new HBaseAdmin(conf);
for (String tableName : TABLE_NAMES) {
if (hbaseAdmin.tableExists(tableName)) {
hbaseAdmin.disableTable(tableName);
hbaseAdmin.deleteTable(tableName);
}
}
HConnectionManager.deleteConnection(hbaseAdmin.getConfiguration(), true);
}
// Temp dir where we will create conf dir
tmpDir = TestHomeUtil.createTestHome("lily-tablesplit-test-");
File customConfDir = setupConfDirectory(tmpDir);
String oldCustomConfDir = setProperty("lily.conf.customdir", customConfDir.getAbsolutePath());
String oldRestoreTemplate = setProperty("lily.lilyproxy.restoretemplatedir", "false");
try {
lilyProxy.start();
} finally {
// Make sure the properties won't be used by later-running tests
setProperty("lily.conf.customdir", oldCustomConfDir);
setProperty("lily.lilyproxy.restoretemplatedir", oldRestoreTemplate);
}
}
/**
* Contacts a region server and waits up to hbase.hbck.close.timeout ms (default 120s) to close
* the region. This bypasses the active hmaster.
*/
public static void closeRegionSilentlyAndWait(
HBaseAdmin admin, ServerName server, HRegionInfo region)
throws IOException, InterruptedException {
HConnection connection = admin.getConnection();
AdminService.BlockingInterface rs = connection.getAdmin(server);
try {
ProtobufUtil.closeRegion(rs, server, region.getRegionName(), false);
} catch (IOException e) {
LOG.warn("Exception when closing region: " + region.getRegionNameAsString(), e);
}
long timeout = admin.getConfiguration().getLong("hbase.hbck.close.timeout", 120000);
long expiration = timeout + System.currentTimeMillis();
while (System.currentTimeMillis() < expiration) {
try {
HRegionInfo rsRegion = ProtobufUtil.getRegionInfo(rs, region.getRegionName());
if (rsRegion == null) return;
} catch (IOException ioe) {
return;
}
Thread.sleep(1000);
}
throw new IOException("Region " + region + " failed to close within" + " timeout " + timeout);
}
@Override
public QueryResult<Variant> getAllVariantsByRegionAndStudy(
Region region, String sourceId, QueryOptions options) {
Long start, end, dbstart, dbend;
start = System.currentTimeMillis();
QueryResult<Variant> queryResult =
new QueryResult<>(
String.format("%s:%d-%d", region.getChromosome(), region.getStart(), region.getEnd()));
List<Variant> results = new LinkedList<>();
boolean includeSamples;
boolean includeStats;
boolean includeEffects;
if (!options.containsKey("samples")
&& !options.containsKey("stats")
&& !options.containsKey("effects")) {
includeSamples = true;
includeStats = true;
includeEffects = true;
} else {
includeSamples = options.containsKey("samples") && options.getBoolean("samples");
includeStats = options.containsKey("stats") && options.getBoolean("stats");
includeEffects = options.containsKey("effects") && options.getBoolean("effects");
}
try {
String startRow = buildRowkey(region.getChromosome(), Long.toString(region.getStart()));
String stopRow = buildRowkey(region.getChromosome(), Long.toString(region.getEnd()));
HTable table = new HTable(admin.getConfiguration(), tableName);
dbstart = System.currentTimeMillis();
Scan regionScan = new Scan(startRow.getBytes(), stopRow.getBytes());
ResultScanner scanres = table.getScanner(regionScan);
dbend = System.currentTimeMillis();
queryResult.setDbTime(dbend - dbstart);
// Iterate over results and, optionally, their samples and statistics
for (Result result : scanres) {
String[] rowkeyParts = new String(result.getRow(), CHARSET_UTF_8).split("_");
String chromosome = rowkeyParts[0].replaceFirst("^0+(?!$)", "");
int position = Integer.parseInt(rowkeyParts[1]);
// Get basic result fields from Protocol Buffers message
NavigableMap<byte[], byte[]> infoMap = result.getFamilyMap("i".getBytes());
byte[] byteInfo = infoMap.get((sourceId + "_data").getBytes());
VariantFieldsProtos.VariantInfo protoInfo =
VariantFieldsProtos.VariantInfo.parseFrom(byteInfo);
String reference = protoInfo.getReference();
String alternate = StringUtils.join(protoInfo.getAlternateList(), ",");
String format = StringUtils.join(protoInfo.getFormatList(), ":");
Variant variant = new Variant(chromosome, position, position, reference, alternate);
// Set samples if requested
if (includeSamples) {
NavigableMap<byte[], byte[]> sampleMap = result.getFamilyMap("d".getBytes());
Map<String, Map<String, String>> resultSampleMap = new HashMap<>();
// Set samples
for (byte[] s : sampleMap.keySet()) {
String sampleName = (new String(s, CHARSET_UTF_8)).replaceAll(sourceId + "_", "");
VariantFieldsProtos.VariantSample sample =
VariantFieldsProtos.VariantSample.parseFrom(sampleMap.get(s));
String sample1 = sample.getSample();
String[] values = sample1.split(":");
String[] fields = format.split(":");
Map<String, String> singleSampleMap = new HashMap<>();
for (int i = 0; i < fields.length; i++) {
singleSampleMap.put(fields[i], values[i]);
}
// TODO
// variant.addSampleData(sampleName, singleSampleMap);
}
}
// Set stats if requested
if (includeStats) {
byte[] byteStats = infoMap.get((sourceId + "_stats").getBytes());
VariantFieldsProtos.VariantStats protoStats =
VariantFieldsProtos.VariantStats.parseFrom(byteStats);
VariantStats variantStats =
new VariantStats(
chromosome,
position,
reference,
alternate,
protoStats.getMaf(),
protoStats.getMgf(),
protoStats.getMafAllele(),
protoStats.getMgfGenotype(),
protoStats.getMissingAlleles(),
protoStats.getMissingGenotypes(),
protoStats.getMendelianErrors(),
protoStats.getIsIndel(),
protoStats.getCasesPercentDominant(),
protoStats.getControlsPercentDominant(),
protoStats.getCasesPercentRecessive(),
protoStats.getControlsPercentRecessive());
variant.setStats(variantStats);
}
if (includeEffects) {
QueryResult<VariantEffect> queryEffects = getEffectsByVariant(variant, options);
variant.setEffect(queryEffects.getResult());
}
results.add(variant);
}
} catch (IOException e) {
System.err.println(e.getClass().getName() + ": " + e.getMessage());
}
queryResult.setResult(results);
queryResult.setNumResults(results.size());
end = System.currentTimeMillis();
queryResult.setTime(end - start);
return queryResult;
}
public List<Variant> getRecordSimpleStats(
String study, int missing_gt, float maf, String maf_allele) {
BasicDBObject compare =
new BasicDBObject("studies.stats.allele_maf", maf_allele)
.append("studies.stats.MAF", maf)
.append("studies.stats.missing", missing_gt);
List<Get> hbaseQuery = new ArrayList<>();
DBCollection collection = db.getCollection("variants");
Iterator<DBObject> result = collection.find(compare);
String chromosome = new String();
while (result.hasNext()) {
DBObject variant = result.next();
String position = variant.get("_id").toString();
// hbase query construction
Get get = new Get(position.getBytes());
hbaseQuery.add(get);
}
// Complete results, from HBase
tableName = study;
effectTableName = tableName + "effect";
Map<String, Variant> resultsMap = new HashMap<>();
try {
HTable table = new HTable(admin.getConfiguration(), tableName);
HTable effectTable = new HTable(admin.getConfiguration(), effectTableName);
Result[] hbaseResultEffect = effectTable.get(hbaseQuery);
Result[] hbaseResultStats = table.get(hbaseQuery);
// List<Variant> results = new LinkedList<>();
for (Result r : hbaseResultStats) {
String position = new String(r.getRow(), CHARSET_UTF_8);
String[] aux = position.split("_");
String inner_position = aux[1];
String chr = aux[0];
// position parsing
if (chr.startsWith("0")) {
chr = chr.substring(1);
}
while (inner_position.startsWith("0")) {
inner_position = inner_position.substring(1);
}
List<VariantFieldsProtos.VariantSample> samples = new LinkedList<>();
NavigableMap<byte[], byte[]> infoMap = r.getFamilyMap("i".getBytes());
byte[] byteStats = infoMap.get((study + "_stats").getBytes());
VariantFieldsProtos.VariantStats stats =
VariantFieldsProtos.VariantStats.parseFrom(byteStats);
byte[] byteInfo = infoMap.get((study + "_data").getBytes());
VariantFieldsProtos.VariantInfo info = VariantFieldsProtos.VariantInfo.parseFrom(byteInfo);
String alternate = StringUtils.join(info.getAlternateList(), ", ");
String reference = info.getReference();
Variant partialResult =
new Variant(
chr,
Integer.parseInt(inner_position),
Integer.parseInt(inner_position),
reference,
alternate);
String format = StringUtils.join(info.getFormatList(), ":");
NavigableMap<byte[], byte[]> sampleMap = r.getFamilyMap("d".getBytes());
Map<String, Map<String, String>> resultSampleMap = new HashMap<>();
// StringBuilder sampleRaw = new StringBuilder();
for (byte[] s : sampleMap.keySet()) {
String qual = (new String(s, CHARSET_UTF_8)).replaceAll(study + "_", "");
VariantFieldsProtos.VariantSample sample =
VariantFieldsProtos.VariantSample.parseFrom(sampleMap.get(s));
String sample1 = sample.getSample();
String[] values = sample1.split(":");
String[] fields = format.split(":");
Map<String, String> singleSampleMap = new HashMap<>();
for (int i = 0; i < fields.length; i++) {
singleSampleMap.put(fields[i], values[i]);
}
resultSampleMap.put(qual, singleSampleMap);
}
VariantStats variantStats =
new VariantStats(
chromosome,
Integer.parseInt(inner_position),
reference,
alternate,
stats.getMaf(),
stats.getMgf(),
stats.getMafAllele(),
stats.getMgfGenotype(),
stats.getMissingAlleles(),
stats.getMissingGenotypes(),
stats.getMendelianErrors(),
stats.getIsIndel(),
stats.getCasesPercentDominant(),
stats.getControlsPercentDominant(),
stats.getCasesPercentRecessive(),
stats.getControlsPercentRecessive());
partialResult.setStats(variantStats);
resultsMap.put(new String(r.getRow(), CHARSET_UTF_8), partialResult);
}
for (Result r : hbaseResultEffect) {
if (!r.isEmpty()) {
NavigableMap<byte[], byte[]> effectMap = r.getFamilyMap("e".getBytes());
Variant partialResult = resultsMap.get(new String(r.getRow(), CHARSET_UTF_8));
System.out.println("Recuperado " + partialResult.toString());
String s = partialResult.getReference() + "_" + partialResult.getAlternate();
VariantEffectProtos.EffectInfo effectInfo =
VariantEffectProtos.EffectInfo.parseFrom(effectMap.get(s.getBytes()));
VariantEffect variantEffect =
new VariantEffect(
partialResult.getChromosome(),
(int) partialResult.getStart(),
partialResult.getReference(),
partialResult.getAlternate(),
effectInfo.getFeatureId(),
effectInfo.getFeatureName(),
effectInfo.getFeatureType(),
effectInfo.getFeatureBiotype(),
effectInfo.getFeatureChromosome(),
effectInfo.getFeatureStart(),
effectInfo.getFeatureEnd(),
effectInfo.getFeatureStrand(),
effectInfo.getSnpId(),
effectInfo.getAncestral(),
effectInfo.getAlternative(),
effectInfo.getGeneId(),
effectInfo.getTranscriptId(),
effectInfo.getGeneName(),
effectInfo.getConsequenceType(),
effectInfo.getConsequenceTypeObo(),
effectInfo.getConsequenceTypeDesc(),
effectInfo.getConsequenceTypeType(),
effectInfo.getAaPosition(),
effectInfo.getAminoacidChange(),
effectInfo.getCodonChange());
resultsMap.put(new String(r.getRow(), CHARSET_UTF_8), partialResult);
}
}
} catch (InvalidProtocolBufferException e) {
System.err.println(e.getClass().getName() + ": " + e.getMessage());
} catch (IOException e) {
System.err.println(e.getClass().getName() + ": " + e.getMessage());
}
List<Variant> results = new ArrayList<>(resultsMap.values());
return results;
}