@Override
public void setOwner(ORecord owner) {
if (owner != null && this.owner != null && !this.owner.equals(owner)) {
throw new IllegalStateException(
"This data structure is owned by document "
+ owner
+ " if you want to use it in other document create new rid bag instance and copy content of current one.");
}
if (this.owner != null) {
for (OIdentifiable entry : newEntries.keySet()) {
ORecordInternal.unTrack(this.owner, entry);
}
for (OIdentifiable entry : changes.keySet()) {
ORecordInternal.unTrack(this.owner, entry);
}
}
this.owner = owner;
if (this.owner != null) {
for (OIdentifiable entry : newEntries.keySet()) {
ORecordInternal.track(this.owner, entry);
}
for (OIdentifiable entry : changes.keySet()) {
ORecordInternal.track(this.owner, entry);
}
}
}
public void testForwardIteration() throws IOException {
final int records = 10000;
long seed = System.currentTimeMillis();
MersenneTwisterFast mersenneTwisterFast = new MersenneTwisterFast(seed);
System.out.println("testForwardIteration seed : " + seed);
NavigableMap<OClusterPosition, byte[]> positionRecordMap = new TreeMap<OClusterPosition, byte[]>();
ORecordVersion recordVersion = OVersionFactory.instance().createVersion();
recordVersion.increment();
recordVersion.increment();
for (int i = 0; i < records; i++) {
int recordSize = mersenneTwisterFast.nextInt(2 * OClusterPage.MAX_RECORD_SIZE) + 1;
byte[] record = new byte[recordSize];
mersenneTwisterFast.nextBytes(record);
final OPhysicalPosition physicalPosition = paginatedCluster.createRecord(record, recordVersion, (byte) 2);
positionRecordMap.put(physicalPosition.clusterPosition, record);
}
Iterator<OClusterPosition> positionIterator = positionRecordMap.keySet().iterator();
while (positionIterator.hasNext()) {
OClusterPosition clusterPosition = positionIterator.next();
if (mersenneTwisterFast.nextBoolean()) {
Assert.assertTrue(paginatedCluster.deleteRecord(clusterPosition));
positionIterator.remove();
}
}
OPhysicalPosition physicalPosition = new OPhysicalPosition();
physicalPosition.clusterPosition = OClusterPositionFactory.INSTANCE.valueOf(0);
OPhysicalPosition[] positions = paginatedCluster.ceilingPositions(physicalPosition);
Assert.assertTrue(positions.length > 0);
int counter = 0;
for (OClusterPosition testedPosition : positionRecordMap.keySet()) {
Assert.assertTrue(positions.length > 0);
Assert.assertEquals(positions[0].clusterPosition, testedPosition);
OPhysicalPosition positionToFind = positions[0];
positions = paginatedCluster.higherPositions(positionToFind);
counter++;
}
Assert.assertEquals(paginatedCluster.getEntries(), counter);
Assert.assertEquals(paginatedCluster.getFirstPosition(), positionRecordMap.firstKey());
Assert.assertEquals(paginatedCluster.getLastPosition(), positionRecordMap.lastKey());
}
@Override
public Set<Element> get(Object key) {
Get get = new Get(ByteArraySerializer.fromObject(key));
Result result;
try {
result = backingTable.get(get);
} catch (IOException e) {
LOG.severe("Cannot get from backing table");
e.printStackTrace();
return null;
}
NavigableMap<byte[], byte[]> map = result.getFamilyMap(Bytes.toBytes(VALUES));
if (null == map) return null;
HashSet<Element> elementHashSet = new HashSet<Element>();
for (byte[] byteArray : map.keySet()) {
if (indexClass.equals(HVertex.class) || indexClass.equals(Vertex.class)) {
HVertex hVertex = new HVertex(hGraph);
hVertex.setId(byteArray);
elementHashSet.add(hVertex);
} else {
final HEdge hEdge = new HEdge(hGraph);
hEdge.setId(byteArray);
elementHashSet.add(hEdge);
}
}
return elementHashSet;
}
/**
* Save the state of the {@code TreeSet} instance to a stream (that is, serialize it).
*
* @serialData Emits the comparator used to order this set, or {@code null} if it obeys its
* elements' natural ordering (Object), followed by the size of the set (the number of
* elements it contains) (int), followed by all of its elements (each an Object) in order (as
* determined by the set's Comparator, or by the elements' natural ordering if the set has no
* Comparator).
*/
private void writeObject(java.io.ObjectOutputStream s) throws java.io.IOException {
// Write out any hidden stuff
s.defaultWriteObject();
// Write out Comparator
s.writeObject(m.comparator());
// Write out size
s.writeInt(m.size());
// Write out all elements in the proper order.
for (Iterator i = m.keySet().iterator(); i.hasNext(); ) s.writeObject(i.next());
}
@Override
public Set<String> keySet() {
return configMap.keySet();
}
private int getChangesSerializedSize() {
Set<OIdentifiable> changedIds = new HashSet<OIdentifiable>(changes.keySet());
changedIds.addAll(newEntries.keySet());
return ChangeSerializationHelper.INSTANCE.getChangesSerializedSize(changedIds.size());
}
@Override
public QueryResult<Variant> getAllVariantsByRegionAndStudy(
Region region, String sourceId, QueryOptions options) {
Long start, end, dbstart, dbend;
start = System.currentTimeMillis();
QueryResult<Variant> queryResult =
new QueryResult<>(
String.format("%s:%d-%d", region.getChromosome(), region.getStart(), region.getEnd()));
List<Variant> results = new LinkedList<>();
boolean includeSamples;
boolean includeStats;
boolean includeEffects;
if (!options.containsKey("samples")
&& !options.containsKey("stats")
&& !options.containsKey("effects")) {
includeSamples = true;
includeStats = true;
includeEffects = true;
} else {
includeSamples = options.containsKey("samples") && options.getBoolean("samples");
includeStats = options.containsKey("stats") && options.getBoolean("stats");
includeEffects = options.containsKey("effects") && options.getBoolean("effects");
}
try {
String startRow = buildRowkey(region.getChromosome(), Long.toString(region.getStart()));
String stopRow = buildRowkey(region.getChromosome(), Long.toString(region.getEnd()));
HTable table = new HTable(admin.getConfiguration(), tableName);
dbstart = System.currentTimeMillis();
Scan regionScan = new Scan(startRow.getBytes(), stopRow.getBytes());
ResultScanner scanres = table.getScanner(regionScan);
dbend = System.currentTimeMillis();
queryResult.setDbTime(dbend - dbstart);
// Iterate over results and, optionally, their samples and statistics
for (Result result : scanres) {
String[] rowkeyParts = new String(result.getRow(), CHARSET_UTF_8).split("_");
String chromosome = rowkeyParts[0].replaceFirst("^0+(?!$)", "");
int position = Integer.parseInt(rowkeyParts[1]);
// Get basic result fields from Protocol Buffers message
NavigableMap<byte[], byte[]> infoMap = result.getFamilyMap("i".getBytes());
byte[] byteInfo = infoMap.get((sourceId + "_data").getBytes());
VariantFieldsProtos.VariantInfo protoInfo =
VariantFieldsProtos.VariantInfo.parseFrom(byteInfo);
String reference = protoInfo.getReference();
String alternate = StringUtils.join(protoInfo.getAlternateList(), ",");
String format = StringUtils.join(protoInfo.getFormatList(), ":");
Variant variant = new Variant(chromosome, position, position, reference, alternate);
// Set samples if requested
if (includeSamples) {
NavigableMap<byte[], byte[]> sampleMap = result.getFamilyMap("d".getBytes());
Map<String, Map<String, String>> resultSampleMap = new HashMap<>();
// Set samples
for (byte[] s : sampleMap.keySet()) {
String sampleName = (new String(s, CHARSET_UTF_8)).replaceAll(sourceId + "_", "");
VariantFieldsProtos.VariantSample sample =
VariantFieldsProtos.VariantSample.parseFrom(sampleMap.get(s));
String sample1 = sample.getSample();
String[] values = sample1.split(":");
String[] fields = format.split(":");
Map<String, String> singleSampleMap = new HashMap<>();
for (int i = 0; i < fields.length; i++) {
singleSampleMap.put(fields[i], values[i]);
}
// TODO
// variant.addSampleData(sampleName, singleSampleMap);
}
}
// Set stats if requested
if (includeStats) {
byte[] byteStats = infoMap.get((sourceId + "_stats").getBytes());
VariantFieldsProtos.VariantStats protoStats =
VariantFieldsProtos.VariantStats.parseFrom(byteStats);
VariantStats variantStats =
new VariantStats(
chromosome,
position,
reference,
alternate,
protoStats.getMaf(),
protoStats.getMgf(),
protoStats.getMafAllele(),
protoStats.getMgfGenotype(),
protoStats.getMissingAlleles(),
protoStats.getMissingGenotypes(),
protoStats.getMendelianErrors(),
protoStats.getIsIndel(),
protoStats.getCasesPercentDominant(),
protoStats.getControlsPercentDominant(),
protoStats.getCasesPercentRecessive(),
protoStats.getControlsPercentRecessive());
variant.setStats(variantStats);
}
if (includeEffects) {
QueryResult<VariantEffect> queryEffects = getEffectsByVariant(variant, options);
variant.setEffect(queryEffects.getResult());
}
results.add(variant);
}
} catch (IOException e) {
System.err.println(e.getClass().getName() + ": " + e.getMessage());
}
queryResult.setResult(results);
queryResult.setNumResults(results.size());
end = System.currentTimeMillis();
queryResult.setTime(end - start);
return queryResult;
}
public List<Variant> getRecordSimpleStats(
String study, int missing_gt, float maf, String maf_allele) {
BasicDBObject compare =
new BasicDBObject("studies.stats.allele_maf", maf_allele)
.append("studies.stats.MAF", maf)
.append("studies.stats.missing", missing_gt);
List<Get> hbaseQuery = new ArrayList<>();
DBCollection collection = db.getCollection("variants");
Iterator<DBObject> result = collection.find(compare);
String chromosome = new String();
while (result.hasNext()) {
DBObject variant = result.next();
String position = variant.get("_id").toString();
// hbase query construction
Get get = new Get(position.getBytes());
hbaseQuery.add(get);
}
// Complete results, from HBase
tableName = study;
effectTableName = tableName + "effect";
Map<String, Variant> resultsMap = new HashMap<>();
try {
HTable table = new HTable(admin.getConfiguration(), tableName);
HTable effectTable = new HTable(admin.getConfiguration(), effectTableName);
Result[] hbaseResultEffect = effectTable.get(hbaseQuery);
Result[] hbaseResultStats = table.get(hbaseQuery);
// List<Variant> results = new LinkedList<>();
for (Result r : hbaseResultStats) {
String position = new String(r.getRow(), CHARSET_UTF_8);
String[] aux = position.split("_");
String inner_position = aux[1];
String chr = aux[0];
// position parsing
if (chr.startsWith("0")) {
chr = chr.substring(1);
}
while (inner_position.startsWith("0")) {
inner_position = inner_position.substring(1);
}
List<VariantFieldsProtos.VariantSample> samples = new LinkedList<>();
NavigableMap<byte[], byte[]> infoMap = r.getFamilyMap("i".getBytes());
byte[] byteStats = infoMap.get((study + "_stats").getBytes());
VariantFieldsProtos.VariantStats stats =
VariantFieldsProtos.VariantStats.parseFrom(byteStats);
byte[] byteInfo = infoMap.get((study + "_data").getBytes());
VariantFieldsProtos.VariantInfo info = VariantFieldsProtos.VariantInfo.parseFrom(byteInfo);
String alternate = StringUtils.join(info.getAlternateList(), ", ");
String reference = info.getReference();
Variant partialResult =
new Variant(
chr,
Integer.parseInt(inner_position),
Integer.parseInt(inner_position),
reference,
alternate);
String format = StringUtils.join(info.getFormatList(), ":");
NavigableMap<byte[], byte[]> sampleMap = r.getFamilyMap("d".getBytes());
Map<String, Map<String, String>> resultSampleMap = new HashMap<>();
// StringBuilder sampleRaw = new StringBuilder();
for (byte[] s : sampleMap.keySet()) {
String qual = (new String(s, CHARSET_UTF_8)).replaceAll(study + "_", "");
VariantFieldsProtos.VariantSample sample =
VariantFieldsProtos.VariantSample.parseFrom(sampleMap.get(s));
String sample1 = sample.getSample();
String[] values = sample1.split(":");
String[] fields = format.split(":");
Map<String, String> singleSampleMap = new HashMap<>();
for (int i = 0; i < fields.length; i++) {
singleSampleMap.put(fields[i], values[i]);
}
resultSampleMap.put(qual, singleSampleMap);
}
VariantStats variantStats =
new VariantStats(
chromosome,
Integer.parseInt(inner_position),
reference,
alternate,
stats.getMaf(),
stats.getMgf(),
stats.getMafAllele(),
stats.getMgfGenotype(),
stats.getMissingAlleles(),
stats.getMissingGenotypes(),
stats.getMendelianErrors(),
stats.getIsIndel(),
stats.getCasesPercentDominant(),
stats.getControlsPercentDominant(),
stats.getCasesPercentRecessive(),
stats.getControlsPercentRecessive());
partialResult.setStats(variantStats);
resultsMap.put(new String(r.getRow(), CHARSET_UTF_8), partialResult);
}
for (Result r : hbaseResultEffect) {
if (!r.isEmpty()) {
NavigableMap<byte[], byte[]> effectMap = r.getFamilyMap("e".getBytes());
Variant partialResult = resultsMap.get(new String(r.getRow(), CHARSET_UTF_8));
System.out.println("Recuperado " + partialResult.toString());
String s = partialResult.getReference() + "_" + partialResult.getAlternate();
VariantEffectProtos.EffectInfo effectInfo =
VariantEffectProtos.EffectInfo.parseFrom(effectMap.get(s.getBytes()));
VariantEffect variantEffect =
new VariantEffect(
partialResult.getChromosome(),
(int) partialResult.getStart(),
partialResult.getReference(),
partialResult.getAlternate(),
effectInfo.getFeatureId(),
effectInfo.getFeatureName(),
effectInfo.getFeatureType(),
effectInfo.getFeatureBiotype(),
effectInfo.getFeatureChromosome(),
effectInfo.getFeatureStart(),
effectInfo.getFeatureEnd(),
effectInfo.getFeatureStrand(),
effectInfo.getSnpId(),
effectInfo.getAncestral(),
effectInfo.getAlternative(),
effectInfo.getGeneId(),
effectInfo.getTranscriptId(),
effectInfo.getGeneName(),
effectInfo.getConsequenceType(),
effectInfo.getConsequenceTypeObo(),
effectInfo.getConsequenceTypeDesc(),
effectInfo.getConsequenceTypeType(),
effectInfo.getAaPosition(),
effectInfo.getAminoacidChange(),
effectInfo.getCodonChange());
resultsMap.put(new String(r.getRow(), CHARSET_UTF_8), partialResult);
}
}
} catch (InvalidProtocolBufferException e) {
System.err.println(e.getClass().getName() + ": " + e.getMessage());
} catch (IOException e) {
System.err.println(e.getClass().getName() + ": " + e.getMessage());
}
List<Variant> results = new ArrayList<>(resultsMap.values());
return results;
}
/**
* Returns an Iterator on the elements of this {@code TreeSet}.
*
* @return an Iterator on the elements of this {@code TreeSet}.
* @see Iterator
*/
@Override
public Iterator<E> iterator() {
return backingMap.keySet().iterator();
}