@Override
public PColumnFamily getColumnFamily(String familyName) throws ColumnFamilyNotFoundException {
PColumnFamily family = familyByString.get(familyName);
if (family == null) {
throw new ColumnFamilyNotFoundException(familyName);
}
return family;
}
@Override
public PColumnFamily getColumnFamily(byte[] familyBytes) throws ColumnFamilyNotFoundException {
PColumnFamily family = familyByBytes.get(familyBytes);
if (family == null) {
String familyName = Bytes.toString(familyBytes);
throw new ColumnFamilyNotFoundException(familyName);
}
return family;
}
public List<Variant> getRecordSimpleStats(
String study, int missing_gt, float maf, String maf_allele) {
BasicDBObject compare =
new BasicDBObject("studies.stats.allele_maf", maf_allele)
.append("studies.stats.MAF", maf)
.append("studies.stats.missing", missing_gt);
List<Get> hbaseQuery = new ArrayList<>();
DBCollection collection = db.getCollection("variants");
Iterator<DBObject> result = collection.find(compare);
String chromosome = new String();
while (result.hasNext()) {
DBObject variant = result.next();
String position = variant.get("_id").toString();
// hbase query construction
Get get = new Get(position.getBytes());
hbaseQuery.add(get);
}
// Complete results, from HBase
tableName = study;
effectTableName = tableName + "effect";
Map<String, Variant> resultsMap = new HashMap<>();
try {
HTable table = new HTable(admin.getConfiguration(), tableName);
HTable effectTable = new HTable(admin.getConfiguration(), effectTableName);
Result[] hbaseResultEffect = effectTable.get(hbaseQuery);
Result[] hbaseResultStats = table.get(hbaseQuery);
// List<Variant> results = new LinkedList<>();
for (Result r : hbaseResultStats) {
String position = new String(r.getRow(), CHARSET_UTF_8);
String[] aux = position.split("_");
String inner_position = aux[1];
String chr = aux[0];
// position parsing
if (chr.startsWith("0")) {
chr = chr.substring(1);
}
while (inner_position.startsWith("0")) {
inner_position = inner_position.substring(1);
}
List<VariantFieldsProtos.VariantSample> samples = new LinkedList<>();
NavigableMap<byte[], byte[]> infoMap = r.getFamilyMap("i".getBytes());
byte[] byteStats = infoMap.get((study + "_stats").getBytes());
VariantFieldsProtos.VariantStats stats =
VariantFieldsProtos.VariantStats.parseFrom(byteStats);
byte[] byteInfo = infoMap.get((study + "_data").getBytes());
VariantFieldsProtos.VariantInfo info = VariantFieldsProtos.VariantInfo.parseFrom(byteInfo);
String alternate = StringUtils.join(info.getAlternateList(), ", ");
String reference = info.getReference();
Variant partialResult =
new Variant(
chr,
Integer.parseInt(inner_position),
Integer.parseInt(inner_position),
reference,
alternate);
String format = StringUtils.join(info.getFormatList(), ":");
NavigableMap<byte[], byte[]> sampleMap = r.getFamilyMap("d".getBytes());
Map<String, Map<String, String>> resultSampleMap = new HashMap<>();
// StringBuilder sampleRaw = new StringBuilder();
for (byte[] s : sampleMap.keySet()) {
String qual = (new String(s, CHARSET_UTF_8)).replaceAll(study + "_", "");
VariantFieldsProtos.VariantSample sample =
VariantFieldsProtos.VariantSample.parseFrom(sampleMap.get(s));
String sample1 = sample.getSample();
String[] values = sample1.split(":");
String[] fields = format.split(":");
Map<String, String> singleSampleMap = new HashMap<>();
for (int i = 0; i < fields.length; i++) {
singleSampleMap.put(fields[i], values[i]);
}
resultSampleMap.put(qual, singleSampleMap);
}
VariantStats variantStats =
new VariantStats(
chromosome,
Integer.parseInt(inner_position),
reference,
alternate,
stats.getMaf(),
stats.getMgf(),
stats.getMafAllele(),
stats.getMgfGenotype(),
stats.getMissingAlleles(),
stats.getMissingGenotypes(),
stats.getMendelianErrors(),
stats.getIsIndel(),
stats.getCasesPercentDominant(),
stats.getControlsPercentDominant(),
stats.getCasesPercentRecessive(),
stats.getControlsPercentRecessive());
partialResult.setStats(variantStats);
resultsMap.put(new String(r.getRow(), CHARSET_UTF_8), partialResult);
}
for (Result r : hbaseResultEffect) {
if (!r.isEmpty()) {
NavigableMap<byte[], byte[]> effectMap = r.getFamilyMap("e".getBytes());
Variant partialResult = resultsMap.get(new String(r.getRow(), CHARSET_UTF_8));
System.out.println("Recuperado " + partialResult.toString());
String s = partialResult.getReference() + "_" + partialResult.getAlternate();
VariantEffectProtos.EffectInfo effectInfo =
VariantEffectProtos.EffectInfo.parseFrom(effectMap.get(s.getBytes()));
VariantEffect variantEffect =
new VariantEffect(
partialResult.getChromosome(),
(int) partialResult.getStart(),
partialResult.getReference(),
partialResult.getAlternate(),
effectInfo.getFeatureId(),
effectInfo.getFeatureName(),
effectInfo.getFeatureType(),
effectInfo.getFeatureBiotype(),
effectInfo.getFeatureChromosome(),
effectInfo.getFeatureStart(),
effectInfo.getFeatureEnd(),
effectInfo.getFeatureStrand(),
effectInfo.getSnpId(),
effectInfo.getAncestral(),
effectInfo.getAlternative(),
effectInfo.getGeneId(),
effectInfo.getTranscriptId(),
effectInfo.getGeneName(),
effectInfo.getConsequenceType(),
effectInfo.getConsequenceTypeObo(),
effectInfo.getConsequenceTypeDesc(),
effectInfo.getConsequenceTypeType(),
effectInfo.getAaPosition(),
effectInfo.getAminoacidChange(),
effectInfo.getCodonChange());
resultsMap.put(new String(r.getRow(), CHARSET_UTF_8), partialResult);
}
}
} catch (InvalidProtocolBufferException e) {
System.err.println(e.getClass().getName() + ": " + e.getMessage());
} catch (IOException e) {
System.err.println(e.getClass().getName() + ": " + e.getMessage());
}
List<Variant> results = new ArrayList<>(resultsMap.values());
return results;
}
public static List<KeyValue> getAllTrainInfo(Configuration config, String date) {
List<KeyValue> result = new ArrayList<>();
String strJson = null;
BufferedWriter writer = null;
Table table = null;
try (Connection connect = ConnectionFactory.createConnection(config);
Admin admin = connect.getAdmin()) {
TableName tablename = TableName.valueOf(TABLE_NAME);
if (!admin.tableExists(tablename)) {
System.out.println("Table does not exist.");
return null;
}
table = connect.getTable(tablename);
Put put = null;
String start = null;
String end = null;
writer = new BufferedWriter(new FileWriter(new File(strConfig), true));
for (KeyValue item : lstAllProcessStation) {
start = (String) item.getKey();
end = (String) item.getValue();
try {
try {
Thread.sleep(200);
} catch (InterruptedException e1) {
e1.printStackTrace();
}
System.out.println("process : " + start + ":" + end);
strJson = getFromAPIX(mapStationCode.get(start), mapStationCode.get(end), date);
writer.write(start + ":" + end);
writer.newLine();
} catch (Exception e) {
System.out.println(start + ":" + end + "error");
e.printStackTrace();
break;
}
JSONObject jo = new JSONObject(strJson);
if (jo.has("httpstatus") && (jo.getInt("httpstatus") == 200)) {
JSONObject joData = jo.getJSONObject("data");
if (joData.has("flag") && joData.getBoolean("flag")) {
result.add(new DefaultKeyValue(start, end));
// 插入到hbase
String rowkey = start + ":" + end;
put = new Put(rowkey.getBytes());
put.addColumn(
CF_JSON.getBytes(), "json".getBytes(), joData.toString().getBytes("utf-8"));
table.put(put);
System.out.println("start " + start + "\t end " + end + "\t has ticket");
}
}
}
} catch (IOException e) {
e.printStackTrace();
} finally {
if (writer != null) {
try {
writer.flush();
writer.close();
} catch (IOException e) {
e.printStackTrace();
}
}
if (table != null) {
try {
table.close();
} catch (IOException e) {
e.printStackTrace();
}
}
}
return result;
}