/**
* Sorts lookups based on their dependencies with each other. Non-lookups (i.e. everything except
* #match, #setChoice, and #mapChoice) are in no particular order in this ordering, since they can
* always be inferred later to occur at the final step after all other variables are bound.
*
* @return
*/
private Stream<KApply> getSortedLookups() {
List<Tuple2<KApply, KApply>> edges = new ArrayList<>();
for (KApply k1 : state) {
Multiset<KVariable> rhsVars = HashMultiset.create();
if (k1.klabel().name().equals("Set:in")) {
continue;
}
gatherVars(k1.klist().items().get(1), rhsVars);
for (KApply k2 : state) {
Multiset<KVariable> lhsVars = HashMultiset.create();
if (k2.klabel().name().equals("Set:in")) {
continue;
}
gatherVars(k2.klist().items().get(0), lhsVars);
for (KVariable var : rhsVars) {
if (lhsVars.contains(var)) {
if (k1 != k2) {
edges.add(Tuple2.apply(k2, k1));
break;
}
}
}
}
}
List<KApply> topologicalSorted = mutable(TopologicalSort.tsort(immutable(edges)).toList());
return state
.stream()
.sorted((k1, k2) -> (topologicalSorted.indexOf(k1) - topologicalSorted.indexOf(k2)));
}
KVariable newDotVariable(Sort sort) {
KVariable newLabel;
do {
newLabel = KVariable("_" + (counter++), Att().add("sort", sort));
} while (vars.contains(newLabel));
vars.add(newLabel);
return newLabel;
}
@Override
public boolean matchesSafely(final Iterable<? super K> actual) {
final Multiset<?> comparisonMultiSet = HashMultiset.create(comparisonIterable);
final Multiset<?> actualMultiSet = HashMultiset.create(actual);
for (final Object key : actualMultiSet.elementSet()) {
if (!comparisonMultiSet.contains(key)
|| comparisonMultiSet.count(key) < actualMultiSet.count(key)) {
return false;
}
}
return true;
}
private Iterable<?> notRepeated(
Iterable<? super K> actual, Iterable<? super K> comparisonIterable) {
final Multiset<?> comparisonMultiSet = HashMultiset.create(comparisonIterable);
final Multiset<?> actualMultiSet = HashMultiset.create(actual);
final Set<Object> notRepeated = newHashSet();
for (final Object key : actualMultiSet.elementSet()) {
if (!comparisonMultiSet.contains(key)
|| comparisonMultiSet.count(key) < actualMultiSet.count(key)) {
notRepeated.add(key);
}
}
return notRepeated;
}
public void convertToNetwork() throws IOException, InvalidFormatException {
container = MyFileImporter.container;
container.setEdgeDefault(EdgeDefault.UNDIRECTED);
String firstDelimiter;
String secondDelimiter;
firstDelimiter = Utils.getCharacter(MyFileImporter.firstConnectorDelimiter);
secondDelimiter = Utils.getCharacter(MyFileImporter.secondConnectorDelimiter);
boolean oneTypeOfAgent =
MyFileImporter.getFirstConnectedAgent().equals(MyFileImporter.getSecondConnectedAgent());
nbColumnFirstAgent = MyFileImporter.firstConnectedAgentIndex;
nbColumnSecondAgent = MyFileImporter.secondConnectedAgentIndex;
Integer lineCounter = 0;
InputStream inp;
inp = new FileInputStream(fileName);
Workbook wb = WorkbookFactory.create(inp);
Row row;
Sheet sheet = wb.getSheet(sheetName);
int startingRow;
if (MyFileImporter.headersPresent) {
startingRow = 1;
} else {
startingRow = 0;
}
Set<String> linesFirstAgent = new HashSet();
Set<String> linesSecondAgent = new HashSet();
for (int i = startingRow; i <= sheet.getLastRowNum(); i++) {
row = sheet.getRow(i);
if (row == null) {
break;
}
Cell cell = row.getCell(nbColumnFirstAgent);
if (cell == null) {
Issue issue =
new Issue(
"problem with line "
+ lineCounter
+ " (empty column "
+ MyFileImporter.getFirstConnectedAgent()
+ "). It was skipped in the conversion",
Issue.Level.WARNING);
MyFileImporter.getStaticReport().logIssue(issue);
continue;
}
String firstAgent = row.getCell(nbColumnFirstAgent).getStringCellValue();
if (firstAgent == null || firstAgent.isEmpty()) {
Issue issue =
new Issue(
"problem with line "
+ lineCounter
+ " (empty column "
+ MyFileImporter.getFirstConnectedAgent()
+ "). It was skipped in the conversion",
Issue.Level.WARNING);
MyFileImporter.getStaticReport().logIssue(issue);
continue;
}
if (MyFileImporter.removeDuplicates) {
boolean newLine = linesFirstAgent.add(firstAgent);
if (!newLine) {
continue;
}
}
String secondAgent = null;
if (!oneTypeOfAgent) {
cell = row.getCell(nbColumnSecondAgent);
if (cell == null) {
Issue issue =
new Issue(
"problem with line "
+ lineCounter
+ " (empty column "
+ MyFileImporter.getFirstConnectedAgent()
+ "). It was skipped in the conversion",
Issue.Level.WARNING);
MyFileImporter.getStaticReport().logIssue(issue);
continue;
}
secondAgent = row.getCell(nbColumnSecondAgent).getStringCellValue();
if (secondAgent == null || secondAgent.isEmpty()) {
Issue issue =
new Issue(
"problem with line "
+ lineCounter
+ " (empty column "
+ MyFileImporter.getSecondConnectedAgent()
+ "). It was skipped in the conversion",
Issue.Level.WARNING);
MyFileImporter.getStaticReport().logIssue(issue);
continue;
}
if (MyFileImporter.removeDuplicates) {
boolean newLine = linesFirstAgent.add(firstAgent);
if (!newLine) {
continue;
}
}
}
lineCounter++;
String[] firstAgentSplit;
String[] secondAgentSplit;
if (firstDelimiter != null) {
firstAgentSplit = firstAgent.trim().split(firstDelimiter);
} else {
firstAgentSplit = new String[1];
firstAgentSplit[0] = firstAgent;
}
for (String node : firstAgentSplit) {
nodesFirst.add(node.trim());
}
if (!oneTypeOfAgent) {
if (secondDelimiter != null) {
secondAgentSplit = secondAgent.trim().split(secondDelimiter);
} else {
secondAgentSplit = new String[1];
secondAgentSplit[0] = secondAgent;
}
for (String node : secondAgentSplit) {
nodesSecond.add(node.trim());
}
} else {
secondAgentSplit = null;
}
String[] both = ArrayUtils.addAll(firstAgentSplit, secondAgentSplit);
// let's find all connections between all the tags for this picture
Utils usefulTools = new Utils();
List<String> connections = usefulTools.getListOfLinks(both, MyFileImporter.removeSelfLoops);
edges.addAll(connections);
}
NodeDraft node;
AttributeTable atNodes = container.getAttributeModel().getNodeTable();
AttributeColumn acFrequency = atNodes.addColumn("frequency", AttributeType.INT);
AttributeColumn acType = atNodes.addColumn("type", AttributeType.STRING);
for (String n : nodesFirst.elementSet()) {
node = container.factory().newNodeDraft();
node.setId(n);
node.setLabel(n);
node.addAttributeValue(acFrequency, nodesFirst.count(n));
node.addAttributeValue(acType, MyFileImporter.getFirstConnectedAgent());
container.addNode(node);
}
for (String n : nodesSecond.elementSet()) {
node = container.factory().newNodeDraft();
node.setId(n);
node.setLabel(n);
node.addAttributeValue(acFrequency, nodesSecond.count(n));
node.addAttributeValue(acType, MyFileImporter.getSecondConnectedAgent());
container.addNode(node);
}
// loop for edges
Integer idEdge = 0;
EdgeDraft edge;
for (String e : edges.elementSet()) {
System.out.println("edge: " + e);
String sourceNode = e.split("\\|")[0];
String targetNode = e.split("\\|")[1];
if (!MyFileImporter.innerLinksIncluded) {
if ((nodesFirst.contains(sourceNode) & nodesFirst.contains(targetNode))
|| (nodesSecond.contains(sourceNode) & nodesSecond.contains(targetNode))) {
continue;
}
}
edge = container.factory().newEdgeDraft();
idEdge = idEdge + 1;
edge.setSource(container.getNode(sourceNode));
edge.setTarget(container.getNode(targetNode));
edge.setWeight((float) edges.count(e));
edge.setId(String.valueOf(idEdge));
edge.setType(EdgeDraft.EdgeType.UNDIRECTED);
container.addEdge(edge);
}
}
/**
* The core scoring method for statistics queries
*
* @param statisticsQuery query to be peformed on statisticsStorage
* @param statisticsStorage core data for Statistics qeries
* @param scoringExps an out parameter.
* <p>- If null, experiment counts result of statisticsQuery should be returned. if - If
* non-null, it serves as a flag that an optimised statisticsQuery should be performed to just
* collect Experiments for which non-zero counts exist for Statistics query. A typical call
* scenario in this case is just one efv per statisticsQuery, in which we can both: 1. check
* if the efv Attribute itself is a scoring one 2. map this Attribute and Experimeants in
* scoringExps to efo terms - via the reverse mapping efv-experiment-> efo term in EfoIndex
* (c.f. atlasStatisticsQueryService.getScoringAttributesForGenes())
* @return Multiset of aggregated experiment counts, where the set of scores genes is intersected
* across statisticsQuery.getConditions(), and union-ed across attributes within each
* condition in statisticsQuery.getConditions().
*/
public static Multiset<Integer> scoreQuery(
StatisticsQueryCondition statisticsQuery,
final StatisticsStorage statisticsStorage,
Set<ExperimentInfo> scoringExps) {
// gatherScoringExpsOnly -> experiment counts should be calculated for statisticsQuery
// !gatherScoringExpsOnly -> scoring experiments should be collected (into scoringExps) only
boolean gatherScoringExpsOnly = scoringExps != null;
Set<StatisticsQueryOrConditions<StatisticsQueryCondition>> andStatisticsQueryConditions =
statisticsQuery.getConditions();
Multiset<Integer> results = null;
if (andStatisticsQueryConditions.isEmpty()) { // End of recursion
Set<Integer> bioEntityIdRestrictionSet = statisticsQuery.getBioEntityIdRestrictionSet();
Set<EfAttribute> attributes = statisticsQuery.getAttributes();
if (attributes.isEmpty()) {
// No attributes were provided - we have to use pre-computed scores across all attributes
Multiset<Integer> scoresAcrossAllEfos =
statisticsStorage.getScoresAcrossAllEfos(statisticsQuery.getStatisticsType());
results = intersect(scoresAcrossAllEfos, bioEntityIdRestrictionSet);
} else {
results = HashMultiset.create();
setQueryExperiments(statisticsQuery, statisticsStorage);
// For each experiment in the query, traverse through all attributes and add all gene
// indexes into one ConciseSet. This way a gene can score
// only once for a single experiment - across all OR attributes in this query. Once all
// attributes have been traversed for a single experiment,
// add ConciseSet to Multiset results
for (ExperimentInfo exp : statisticsQuery.getExperiments()) {
FastSet statsForExperiment = new FastSet();
for (EfAttribute attr : attributes) {
Map<ExperimentInfo, ConciseSet> expsToStats =
getStatisticsForAttribute(
statisticsQuery.getStatisticsType(), attr, statisticsStorage);
if (expsToStats != null) {
if (expsToStats.isEmpty()) {
log.debug(
"Failed to retrieve stats for stat: "
+ statisticsQuery.getStatisticsType()
+ " and attr: "
+ attr);
} else {
if (expsToStats.get(exp) != null) {
if (!gatherScoringExpsOnly) {
statsForExperiment.addAll(
intersect(expsToStats.get(exp), bioEntityIdRestrictionSet));
} else if (containsAtLeastOne(expsToStats.get(exp), bioEntityIdRestrictionSet)) {
// exp contains at least one non-zero score for at least one gene index in
// bioEntityIdRestrictionSet -> add it to scoringExps
scoringExps.add(exp);
}
} else {
log.debug(
"Failed to retrieve stats for stat: "
+ statisticsQuery.getStatisticsType()
+ " exp: "
+ exp.getAccession()
+ " and attr: "
+ attr);
}
}
}
}
if (!gatherScoringExpsOnly) {
results.addAll(statsForExperiment);
}
}
}
} else {
// run over all AND conditions, do "OR" inside (cf. scoreOrStatisticsQueryConditions()) ,
// "AND"'ing over the whole thing
for (StatisticsQueryOrConditions<StatisticsQueryCondition> orConditions :
andStatisticsQueryConditions) {
// Pass gene restriction set down to orConditions
orConditions.setGeneRestrictionSet(statisticsQuery.getBioEntityIdRestrictionSet());
// process OR conditions
Multiset<Integer> condGenes =
getScoresForOrConditions(orConditions, statisticsStorage, scoringExps);
if (results == null) results = condGenes;
else {
Iterator<Multiset.Entry<Integer>> resultGenes = results.entrySet().iterator();
while (resultGenes.hasNext()) {
Multiset.Entry<Integer> entry = resultGenes.next();
if (!condGenes.contains(
entry.getElement())) // AND operation between different top query conditions
resultGenes.remove();
else
// for all gene ids belonging to intersection of all conditions seen so far, we
// accumulate experiment counts
results.setCount(
entry.getElement(), entry.getCount() + condGenes.count(entry.getElement()));
}
}
}
}
if (results == null) {
results = HashMultiset.create();
}
return results;
}
public void writeCoverageReport(
final PrintStream pStatisticsOutput, final ReachedSet pReached, final CFA pCfa) {
if (!enabled) {
return;
}
Multiset<FunctionEntryNode> reachedLocations = getFunctionEntriesFromReached(pReached);
Map<String, FileCoverageInformation> infosPerFile = new HashMap<>();
// Add information about existing functions
for (FunctionEntryNode entryNode : pCfa.getAllFunctionHeads()) {
final FileLocation loc = entryNode.getFileLocation();
if (loc.getStartingLineNumber() == 0) {
// dummy location
continue;
}
final String functionName = entryNode.getFunctionName();
final FileCoverageInformation infos = getFileInfoTarget(loc, infosPerFile);
final int startingLine = loc.getStartingLineInOrigin();
final int endingLine = loc.getEndingLineInOrigin();
infos.addExistingFunction(functionName, startingLine, endingLine);
if (reachedLocations.contains(entryNode)) {
infos.addVisitedFunction(entryNode.getFunctionName(), reachedLocations.count(entryNode));
}
}
// Add information about existing locations
for (CFANode node : pCfa.getAllNodes()) {
for (int i = 0; i < node.getNumLeavingEdges(); i++) {
handleExistedEdge(node.getLeavingEdge(i), infosPerFile);
}
}
Set<CFANode> reachedNodes =
from(pReached).transform(EXTRACT_LOCATION).filter(notNull()).toSet();
// Add information about visited locations
for (AbstractState state : pReached) {
ARGState argState = AbstractStates.extractStateByType(state, ARGState.class);
if (argState != null) {
for (ARGState child : argState.getChildren()) {
if (!child.isCovered()) {
List<CFAEdge> edges = argState.getEdgesToChild(child);
if (edges.size() > 1) {
for (CFAEdge innerEdge : edges) {
handleCoveredEdge(innerEdge, infosPerFile);
}
// BAM produces paths with no edge connection thus the list will be empty
} else if (!edges.isEmpty()) {
handleCoveredEdge(Iterables.getOnlyElement(edges), infosPerFile);
}
}
}
} else {
// Simple kind of analysis
// Cover all edges from reached nodes
// It is less precise, but without ARG it is impossible to know what path we chose
CFANode node = AbstractStates.extractLocation(state);
for (int i = 0; i < node.getNumLeavingEdges(); i++) {
CFAEdge edge = node.getLeavingEdge(i);
if (reachedNodes.contains(edge.getSuccessor())) {
handleCoveredEdge(edge, infosPerFile);
}
}
}
}
for (CoverageWriter w : reportWriters) {
w.write(infosPerFile, pStatisticsOutput);
}
}