/* (non-Javadoc)
* @see com.hazelcast.core.MapLoader#loadAllKeys()
*/
@Override
public Set<String> loadAllKeys() {
Set<String> keySet = null;
if (allowLoadAll) {
keySet = new HashSet<String>();
HTableInterface hti = null;
try {
hti = pool.getTable(tableName);
Scan s = new Scan();
s.addColumn(family, qualifier);
ResultScanner rs = hti.getScanner(s);
Result r = null;
while ((r = rs.next()) != null) {
String k = new String(r.getRow());
keySet.add(k);
}
} catch (IOException e) {
LOG.error("IOException while loading all keys", e);
} finally {
if (hti != null) {
pool.putTable(hti);
}
}
}
return keySet;
}
public static void deleteTest(String tableStr) {
try {
Configuration conf = HBaseConfiguration.create();
byte[] tableName = Bytes.toBytes(tableStr);
HConnection hConnection = HConnectionManager.createConnection(conf);
HTableInterface table = hConnection.getTable(tableName);
byte[] startRow = Bytes.toBytes("rowKey_1");
byte[] stopRow = Bytes.toBytes("rowKey_3");
byte[] family = f0;
Scan scan = new Scan();
scan.addFamily(family);
scan.setMaxVersions(1);
// scan.setStartRow(startRow);
// scan.setStopRow(stopRow);
ResultScanner scanner = table.getScanner(scan);
Result result = scanner.next();
List<Delete> delete = new ArrayList<Delete>();
while (result != null) {
Delete del = new Delete(result.getRow());
delete.add(del);
result = scanner.next();
}
table.delete(delete);
System.out.println("delete done");
table.close(); // very important
} catch (IOException e) {
e.printStackTrace();
}
}
@Override
public Result next() throws IOException {
// If the scanner is closed and there's nothing left in the cache, next is a
// no-op.
if (cache.size() == 0 && this.closed) {
return null;
}
if (cache.size() == 0) {
Result[] values = null;
long remainingResultSize = maxScannerResultSize;
int countdown = this.caching;
boolean currentRegionDone = false;
// Values == null means server-side filter has determined we must STOP
while (remainingResultSize > 0
&& countdown > 0
&& nextScanner(countdown, values == null, currentRegionDone)) {
// Server returns a null values if scanning is to stop. Else,
// returns an empty array if scanning is to go on and we've just
// exhausted current region.
values = this.caller.callWithRetries(smallScanCallable, scannerTimeout);
this.currentRegion = smallScanCallable.getHRegionInfo();
long currentTime = System.currentTimeMillis();
if (this.scanMetrics != null) {
this.scanMetrics.sumOfMillisSecBetweenNexts.addAndGet(currentTime - lastNext);
}
lastNext = currentTime;
if (values != null && values.length > 0) {
for (int i = 0; i < values.length; i++) {
Result rs = values[i];
if (i == 0
&& this.skipRowOfFirstResult != null
&& Bytes.equals(skipRowOfFirstResult, rs.getRow())) {
// Skip the first result
continue;
}
cache.add(rs);
for (Cell kv : rs.rawCells()) {
remainingResultSize -= KeyValueUtil.ensureKeyValue(kv).heapSize();
}
countdown--;
this.lastResult = rs;
}
}
currentRegionDone = countdown > 0;
}
}
if (cache.size() > 0) {
return cache.poll();
}
// if we exhausted this scanner before calling close, write out the scan
// metrics
writeScanMetrics();
return null;
}
/*
* (non-Javadoc)
*
* @see com.hazelcast.core.MapLoader#loadAll(java.util.Collection)
*/
@Override
public Map<String, String> loadAll(Collection<String> keys) {
if (allowLoadAll) {
List<Get> gets = new ArrayList<Get>(keys.size());
for (String k : keys) {
Get g = new Get(Bytes.toBytes(k));
gets.add(g);
}
HTableInterface table = null;
Map<String, String> kvMap = new HashMap<String, String>();
try {
table = pool.getTable(tableName);
table.get(gets);
Result[] result = table.get(gets);
for (Result r : result) {
byte[] value = r.getValue(family, qualifier);
if (value != null) {
if (outputFormatType == StoreFormatType.SMILE) {
kvMap.put(
new String(r.getRow()),
jsonSmileConverter.convertFromSmile(r.getValue(family, qualifier)));
} else {
kvMap.put(new String(r.getRow()), new String(r.getValue(family, qualifier)));
}
}
}
} catch (IOException e) {
LOG.error("IOException while getting values", e);
} finally {
if (pool != null && table != null) {
pool.putTable(table);
}
}
}
return null;
}
public static void scanTest(String tableStr) {
try {
Configuration conf = HBaseConfiguration.create();
byte[] tableName = Bytes.toBytes(tableStr);
HConnection hConnection = HConnectionManager.createConnection(conf);
HTableInterface table = hConnection.getTable(tableName);
byte[] startRow = Bytes.toBytes("rowKey_0");
byte[] stopRow = Bytes.toBytes("rowKey_6");
byte[] family = f0;
Scan scan = new Scan();
scan.addFamily(family);
scan.setMaxVersions(1);
// scan.setStartRow(startRow);
// scan.setStopRow(stopRow);
int count = 0;
ResultScanner scanner = table.getScanner(scan);
Result result = scanner.next();
while (result != null) {
String rowKey = Bytes.toString(result.getRow());
NavigableMap<byte[], byte[]> m = result.getFamilyMap(family);
if (m == null || m.isEmpty()) {
System.err.println("Empty." + m);
return;
}
for (Map.Entry<byte[], byte[]> entry : m.entrySet()) {
String qualifier = Bytes.toString(entry.getKey());
String value = Bytes.toString(entry.getValue());
System.out.println(rowKey + ":" + qualifier + ":" + value);
}
result = scanner.next();
count++;
System.out.println("-----------------------------");
}
table.close(); // very important
System.out.println("count:" + count);
} catch (IOException e) {
e.printStackTrace();
}
}
/**
* Deletes the specified table with all its columns. ATTENTION: Invoking this method will delete
* the table if it exists and therefore causes data loss.
*/
@Override
public void clearStorage() throws StorageException {
HBaseAdmin adm = getAdminInterface();
try { // first of all, check if table exists, if not - we are done
if (!adm.tableExists(tableName)) {
logger.debug("clearStorage() called before table {} was created, skipping.", tableName);
return;
}
} catch (IOException e) {
throw new TemporaryStorageException(e);
}
HTable table = null;
try {
table = new HTable(hconf, tableName);
Scan scan = new Scan();
scan.setBatch(100);
scan.setCacheBlocks(false);
scan.setCaching(2000);
ResultScanner scanner = null;
try {
scanner = table.getScanner(scan);
for (Result res : scanner) {
table.delete(new Delete(res.getRow()));
}
} finally {
IOUtils.closeQuietly(scanner);
}
} catch (IOException e) {
throw new TemporaryStorageException(e);
} finally {
IOUtils.closeQuietly(table);
}
}
@Test
public void scan_by_prefix_date() throws IOException {
FilterList fl = new FilterList();
Filter filter =
new RowFilter(CompareFilter.CompareOp.EQUAL, new SubstringComparator("20150903"));
fl.addFilter(filter);
Scan scan = new Scan();
scan.setFilter(fl);
long t1 = System.currentTimeMillis();
ResultScanner rs = errorTable.getScanner(scan);
Result result;
int count = 0;
while ((result = rs.next()) != null) {
System.out.println("rowkey=" + new String(result.getRow()));
System.out.println(
"value=" + new String(result.getValue(fBytes, Bytes.toBytes("stacktrace"))));
System.out.println();
count++;
}
long t2 = System.currentTimeMillis();
System.out.println("count=" + count + ",t2 - t1=" + ((t2 - t1) / 1000));
}
@Override
public QueryResult<Variant> getAllVariantsByRegionAndStudy(
Region region, String sourceId, QueryOptions options) {
Long start, end, dbstart, dbend;
start = System.currentTimeMillis();
QueryResult<Variant> queryResult =
new QueryResult<>(
String.format("%s:%d-%d", region.getChromosome(), region.getStart(), region.getEnd()));
List<Variant> results = new LinkedList<>();
boolean includeSamples;
boolean includeStats;
boolean includeEffects;
if (!options.containsKey("samples")
&& !options.containsKey("stats")
&& !options.containsKey("effects")) {
includeSamples = true;
includeStats = true;
includeEffects = true;
} else {
includeSamples = options.containsKey("samples") && options.getBoolean("samples");
includeStats = options.containsKey("stats") && options.getBoolean("stats");
includeEffects = options.containsKey("effects") && options.getBoolean("effects");
}
try {
String startRow = buildRowkey(region.getChromosome(), Long.toString(region.getStart()));
String stopRow = buildRowkey(region.getChromosome(), Long.toString(region.getEnd()));
HTable table = new HTable(admin.getConfiguration(), tableName);
dbstart = System.currentTimeMillis();
Scan regionScan = new Scan(startRow.getBytes(), stopRow.getBytes());
ResultScanner scanres = table.getScanner(regionScan);
dbend = System.currentTimeMillis();
queryResult.setDbTime(dbend - dbstart);
// Iterate over results and, optionally, their samples and statistics
for (Result result : scanres) {
String[] rowkeyParts = new String(result.getRow(), CHARSET_UTF_8).split("_");
String chromosome = rowkeyParts[0].replaceFirst("^0+(?!$)", "");
int position = Integer.parseInt(rowkeyParts[1]);
// Get basic result fields from Protocol Buffers message
NavigableMap<byte[], byte[]> infoMap = result.getFamilyMap("i".getBytes());
byte[] byteInfo = infoMap.get((sourceId + "_data").getBytes());
VariantFieldsProtos.VariantInfo protoInfo =
VariantFieldsProtos.VariantInfo.parseFrom(byteInfo);
String reference = protoInfo.getReference();
String alternate = StringUtils.join(protoInfo.getAlternateList(), ",");
String format = StringUtils.join(protoInfo.getFormatList(), ":");
Variant variant = new Variant(chromosome, position, position, reference, alternate);
// Set samples if requested
if (includeSamples) {
NavigableMap<byte[], byte[]> sampleMap = result.getFamilyMap("d".getBytes());
Map<String, Map<String, String>> resultSampleMap = new HashMap<>();
// Set samples
for (byte[] s : sampleMap.keySet()) {
String sampleName = (new String(s, CHARSET_UTF_8)).replaceAll(sourceId + "_", "");
VariantFieldsProtos.VariantSample sample =
VariantFieldsProtos.VariantSample.parseFrom(sampleMap.get(s));
String sample1 = sample.getSample();
String[] values = sample1.split(":");
String[] fields = format.split(":");
Map<String, String> singleSampleMap = new HashMap<>();
for (int i = 0; i < fields.length; i++) {
singleSampleMap.put(fields[i], values[i]);
}
// TODO
// variant.addSampleData(sampleName, singleSampleMap);
}
}
// Set stats if requested
if (includeStats) {
byte[] byteStats = infoMap.get((sourceId + "_stats").getBytes());
VariantFieldsProtos.VariantStats protoStats =
VariantFieldsProtos.VariantStats.parseFrom(byteStats);
VariantStats variantStats =
new VariantStats(
chromosome,
position,
reference,
alternate,
protoStats.getMaf(),
protoStats.getMgf(),
protoStats.getMafAllele(),
protoStats.getMgfGenotype(),
protoStats.getMissingAlleles(),
protoStats.getMissingGenotypes(),
protoStats.getMendelianErrors(),
protoStats.getIsIndel(),
protoStats.getCasesPercentDominant(),
protoStats.getControlsPercentDominant(),
protoStats.getCasesPercentRecessive(),
protoStats.getControlsPercentRecessive());
variant.setStats(variantStats);
}
if (includeEffects) {
QueryResult<VariantEffect> queryEffects = getEffectsByVariant(variant, options);
variant.setEffect(queryEffects.getResult());
}
results.add(variant);
}
} catch (IOException e) {
System.err.println(e.getClass().getName() + ": " + e.getMessage());
}
queryResult.setResult(results);
queryResult.setNumResults(results.size());
end = System.currentTimeMillis();
queryResult.setTime(end - start);
return queryResult;
}
public List<Variant> getRecordSimpleStats(
String study, int missing_gt, float maf, String maf_allele) {
BasicDBObject compare =
new BasicDBObject("studies.stats.allele_maf", maf_allele)
.append("studies.stats.MAF", maf)
.append("studies.stats.missing", missing_gt);
List<Get> hbaseQuery = new ArrayList<>();
DBCollection collection = db.getCollection("variants");
Iterator<DBObject> result = collection.find(compare);
String chromosome = new String();
while (result.hasNext()) {
DBObject variant = result.next();
String position = variant.get("_id").toString();
// hbase query construction
Get get = new Get(position.getBytes());
hbaseQuery.add(get);
}
// Complete results, from HBase
tableName = study;
effectTableName = tableName + "effect";
Map<String, Variant> resultsMap = new HashMap<>();
try {
HTable table = new HTable(admin.getConfiguration(), tableName);
HTable effectTable = new HTable(admin.getConfiguration(), effectTableName);
Result[] hbaseResultEffect = effectTable.get(hbaseQuery);
Result[] hbaseResultStats = table.get(hbaseQuery);
// List<Variant> results = new LinkedList<>();
for (Result r : hbaseResultStats) {
String position = new String(r.getRow(), CHARSET_UTF_8);
String[] aux = position.split("_");
String inner_position = aux[1];
String chr = aux[0];
// position parsing
if (chr.startsWith("0")) {
chr = chr.substring(1);
}
while (inner_position.startsWith("0")) {
inner_position = inner_position.substring(1);
}
List<VariantFieldsProtos.VariantSample> samples = new LinkedList<>();
NavigableMap<byte[], byte[]> infoMap = r.getFamilyMap("i".getBytes());
byte[] byteStats = infoMap.get((study + "_stats").getBytes());
VariantFieldsProtos.VariantStats stats =
VariantFieldsProtos.VariantStats.parseFrom(byteStats);
byte[] byteInfo = infoMap.get((study + "_data").getBytes());
VariantFieldsProtos.VariantInfo info = VariantFieldsProtos.VariantInfo.parseFrom(byteInfo);
String alternate = StringUtils.join(info.getAlternateList(), ", ");
String reference = info.getReference();
Variant partialResult =
new Variant(
chr,
Integer.parseInt(inner_position),
Integer.parseInt(inner_position),
reference,
alternate);
String format = StringUtils.join(info.getFormatList(), ":");
NavigableMap<byte[], byte[]> sampleMap = r.getFamilyMap("d".getBytes());
Map<String, Map<String, String>> resultSampleMap = new HashMap<>();
// StringBuilder sampleRaw = new StringBuilder();
for (byte[] s : sampleMap.keySet()) {
String qual = (new String(s, CHARSET_UTF_8)).replaceAll(study + "_", "");
VariantFieldsProtos.VariantSample sample =
VariantFieldsProtos.VariantSample.parseFrom(sampleMap.get(s));
String sample1 = sample.getSample();
String[] values = sample1.split(":");
String[] fields = format.split(":");
Map<String, String> singleSampleMap = new HashMap<>();
for (int i = 0; i < fields.length; i++) {
singleSampleMap.put(fields[i], values[i]);
}
resultSampleMap.put(qual, singleSampleMap);
}
VariantStats variantStats =
new VariantStats(
chromosome,
Integer.parseInt(inner_position),
reference,
alternate,
stats.getMaf(),
stats.getMgf(),
stats.getMafAllele(),
stats.getMgfGenotype(),
stats.getMissingAlleles(),
stats.getMissingGenotypes(),
stats.getMendelianErrors(),
stats.getIsIndel(),
stats.getCasesPercentDominant(),
stats.getControlsPercentDominant(),
stats.getCasesPercentRecessive(),
stats.getControlsPercentRecessive());
partialResult.setStats(variantStats);
resultsMap.put(new String(r.getRow(), CHARSET_UTF_8), partialResult);
}
for (Result r : hbaseResultEffect) {
if (!r.isEmpty()) {
NavigableMap<byte[], byte[]> effectMap = r.getFamilyMap("e".getBytes());
Variant partialResult = resultsMap.get(new String(r.getRow(), CHARSET_UTF_8));
System.out.println("Recuperado " + partialResult.toString());
String s = partialResult.getReference() + "_" + partialResult.getAlternate();
VariantEffectProtos.EffectInfo effectInfo =
VariantEffectProtos.EffectInfo.parseFrom(effectMap.get(s.getBytes()));
VariantEffect variantEffect =
new VariantEffect(
partialResult.getChromosome(),
(int) partialResult.getStart(),
partialResult.getReference(),
partialResult.getAlternate(),
effectInfo.getFeatureId(),
effectInfo.getFeatureName(),
effectInfo.getFeatureType(),
effectInfo.getFeatureBiotype(),
effectInfo.getFeatureChromosome(),
effectInfo.getFeatureStart(),
effectInfo.getFeatureEnd(),
effectInfo.getFeatureStrand(),
effectInfo.getSnpId(),
effectInfo.getAncestral(),
effectInfo.getAlternative(),
effectInfo.getGeneId(),
effectInfo.getTranscriptId(),
effectInfo.getGeneName(),
effectInfo.getConsequenceType(),
effectInfo.getConsequenceTypeObo(),
effectInfo.getConsequenceTypeDesc(),
effectInfo.getConsequenceTypeType(),
effectInfo.getAaPosition(),
effectInfo.getAminoacidChange(),
effectInfo.getCodonChange());
resultsMap.put(new String(r.getRow(), CHARSET_UTF_8), partialResult);
}
}
} catch (InvalidProtocolBufferException e) {
System.err.println(e.getClass().getName() + ": " + e.getMessage());
} catch (IOException e) {
System.err.println(e.getClass().getName() + ": " + e.getMessage());
}
List<Variant> results = new ArrayList<>(resultsMap.values());
return results;
}