private static void sortIntoList(
Node n,
Block b,
ArrayList<Node> result,
NodeMap<Block> nodeMap,
NodeBitMap unprocessed,
Node excludeNode) {
assert unprocessed.isMarked(n) : n;
unprocessed.clear(n);
assert nodeMap.get(n) == b;
if (n instanceof PhiNode) {
return;
}
for (Node input : n.inputs()) {
if (nodeMap.get(input) == b && unprocessed.isMarked(input) && input != excludeNode) {
sortIntoList(input, b, result, nodeMap, unprocessed, excludeNode);
}
}
if (n instanceof ProxyNode) {
// Skip proxy nodes.
} else {
result.add(n);
}
}
private void visitForward(NodeBitMap visited, Node node) {
if (node != null && !visited.isMarked(node)) {
visited.mark(node);
if (node.predecessor() != null) {
visitForward(visited, node.predecessor());
}
if (node instanceof MergeNode) {
// make sure that the cfg predecessors of a MergeNode are processed first
MergeNode merge = (MergeNode) node;
for (int i = 0; i < merge.forwardEndCount(); i++) {
visitForward(visited, merge.forwardEndAt(i));
}
}
for (Node input : node.inputs()) {
visitForward(visited, input);
}
if (node instanceof LoopBeginNode) {
LoopBeginNode loopBegin = (LoopBeginNode) node;
for (LoopEndNode loopEnd : loopBegin.loopEnds()) {
visitForward(visited, loopEnd);
}
}
nodes.add(node);
}
}
private static void checkWatchList(
ArrayList<FloatingReadNode> watchList,
LocationIdentity identity,
Block b,
ArrayList<Node> result,
NodeMap<Block> nodeMap,
NodeBitMap unprocessed) {
assert identity.isMutable();
if (identity.isAny()) {
for (FloatingReadNode r : watchList) {
if (unprocessed.isMarked(r)) {
sortIntoList(r, b, result, nodeMap, unprocessed, null);
}
}
watchList.clear();
} else {
int index = 0;
while (index < watchList.size()) {
FloatingReadNode r = watchList.get(index);
LocationIdentity locationIdentity = r.getLocationIdentity();
assert locationIdentity.isMutable();
if (unprocessed.isMarked(r)) {
if (identity.overlaps(locationIdentity)) {
sortIntoList(r, b, result, nodeMap, unprocessed, null);
} else {
++index;
continue;
}
}
int lastIndex = watchList.size() - 1;
watchList.set(index, watchList.get(lastIndex));
watchList.remove(lastIndex);
}
}
}
@Override
public void postprocess() {
if (anchor != null && OptEliminateGuards.getValue()) {
for (GuardNode guard : anchor.asNode().usages().filter(GuardNode.class)) {
if (activeGuards.isMarkedAndGrow(guard)) {
activeGuards.clear(guard);
}
}
}
}
private boolean accept(T n) {
if (n == null) {
return true;
}
if (visited == null) {
visited = n.graph().createNodeBitMap();
}
boolean accept = !visited.isMarked(n);
visited.mark(n);
return accept;
}
private void visitBackward(NodeBitMap visited, Node node) {
if (node != null && !visited.isMarked(node)) {
visited.mark(node);
for (Node successor : node.successors()) {
visitBackward(visited, successor);
}
for (Node usage : node.usages()) {
visitBackward(visited, usage);
}
nodes.add(node);
}
}
@Override
public GuardingNode createGuard(
FixedNode before,
LogicNode condition,
DeoptimizationReason deoptReason,
DeoptimizationAction action,
JavaConstant speculation,
boolean negated) {
if (OptEliminateGuards.getValue()) {
for (Node usage : condition.usages()) {
if (!activeGuards.isNew(usage)
&& activeGuards.isMarked(usage)
&& ((GuardNode) usage).isNegated() == negated) {
return (GuardNode) usage;
}
}
}
StructuredGraph graph = before.graph();
if (!condition.graph().getGuardsStage().allowsFloatingGuards()) {
FixedGuardNode fixedGuard =
graph.add(new FixedGuardNode(condition, deoptReason, action, speculation, negated));
graph.addBeforeFixed(before, fixedGuard);
DummyGuardHandle handle = graph.add(new DummyGuardHandle(fixedGuard));
fixedGuard.lower(this);
GuardingNode result = handle.getGuard();
handle.safeDelete();
return result;
} else {
GuardNode newGuard =
graph.unique(
new GuardNode(condition, guardAnchor, deoptReason, action, negated, speculation));
if (OptEliminateGuards.getValue()) {
activeGuards.markAndGrow(newGuard);
}
return newGuard;
}
}
protected void calcLatestBlock(
Block earliestBlock,
SchedulingStrategy strategy,
Node currentNode,
NodeMap<Block> currentNodeMap,
LocationIdentity constrainingLocation,
BlockMap<ArrayList<FloatingReadNode>> watchListMap,
BlockMap<List<Node>> latestBlockToNodesMap,
NodeBitMap visited,
boolean immutableGraph) {
Block latestBlock = null;
assert currentNode.hasUsages();
for (Node usage : currentNode.usages()) {
if (immutableGraph && !visited.contains(usage)) {
/*
* Normally, dead nodes are deleted by the scheduler before we reach this point.
* Only when the scheduler is asked to not modify a graph, we can see dead nodes
* here.
*/
continue;
}
latestBlock = calcBlockForUsage(currentNode, usage, latestBlock, currentNodeMap);
}
if (strategy == SchedulingStrategy.FINAL_SCHEDULE
|| strategy == SchedulingStrategy.LATEST_OUT_OF_LOOPS) {
while (latestBlock.getLoopDepth() > earliestBlock.getLoopDepth()
&& latestBlock != earliestBlock.getDominator()) {
latestBlock = latestBlock.getDominator();
}
}
if (latestBlock != earliestBlock
&& latestBlock != earliestBlock.getDominator()
&& constrainingLocation != null) {
latestBlock = checkKillsBetween(earliestBlock, latestBlock, constrainingLocation);
}
selectLatestBlock(
currentNode,
earliestBlock,
latestBlock,
currentNodeMap,
watchListMap,
constrainingLocation,
latestBlockToNodesMap);
}
private static void processStack(
ControlFlowGraph cfg,
BlockMap<List<Node>> blockToNodes,
NodeMap<Block> nodeToBlock,
NodeBitMap visited,
BitSet floatingReads,
NodeStack stack) {
Block startBlock = cfg.getStartBlock();
while (!stack.isEmpty()) {
Node current = stack.peek();
if (visited.checkAndMarkInc(current)) {
// Push inputs and predecessor.
Node predecessor = current.predecessor();
if (predecessor != null) {
stack.push(predecessor);
}
if (current instanceof PhiNode) {
processStackPhi(stack, (PhiNode) current);
} else if (current instanceof ProxyNode) {
processStackProxy(stack, (ProxyNode) current);
} else if (current instanceof FrameState) {
processStackFrameState(stack, current);
} else {
current.pushInputs(stack);
}
} else {
stack.pop();
if (nodeToBlock.get(current) == null) {
Block curBlock = cfg.blockFor(current);
if (curBlock == null) {
assert current.predecessor() == null && !(current instanceof FixedNode)
: "The assignment of blocks to fixed nodes is already done when constructing the cfg.";
Block earliest = startBlock;
for (Node input : current.inputs()) {
Block inputEarliest = nodeToBlock.get(input);
if (inputEarliest == null) {
assert current instanceof FrameState
&& input instanceof StateSplit
&& ((StateSplit) input).stateAfter() == current
: current;
} else {
assert inputEarliest != null;
if (inputEarliest.getEndNode() == input) {
// This is the last node of the block.
if (current instanceof FrameState
&& input instanceof StateSplit
&& ((StateSplit) input).stateAfter() == current) {
// Keep regular inputEarliest.
} else if (input instanceof ControlSplitNode) {
inputEarliest =
nodeToBlock.get(((ControlSplitNode) input).getPrimarySuccessor());
} else {
assert inputEarliest.getSuccessorCount() == 1;
assert !(input instanceof AbstractEndNode);
// Keep regular inputEarliest
}
}
if (earliest.getDominatorDepth() < inputEarliest.getDominatorDepth()) {
earliest = inputEarliest;
}
}
}
curBlock = earliest;
}
assert curBlock != null;
addNode(blockToNodes, curBlock, current);
nodeToBlock.set(current, curBlock);
if (current instanceof FloatingReadNode) {
FloatingReadNode floatingReadNode = (FloatingReadNode) current;
if (curBlock.canKill(floatingReadNode.getLocationIdentity())) {
floatingReads.set(curBlock.getId());
}
}
}
}
}
}
private static void resortEarliestWithinBlock(
Block b,
BlockMap<List<Node>> blockToNodes,
NodeMap<Block> nodeToBlock,
NodeBitMap unprocessed) {
ArrayList<FloatingReadNode> watchList = new ArrayList<>();
List<Node> oldList = blockToNodes.get(b);
AbstractBeginNode beginNode = b.getBeginNode();
for (Node n : oldList) {
if (n instanceof FloatingReadNode) {
FloatingReadNode floatingReadNode = (FloatingReadNode) n;
LocationIdentity locationIdentity = floatingReadNode.getLocationIdentity();
MemoryNode lastLocationAccess = floatingReadNode.getLastLocationAccess();
if (locationIdentity.isMutable() && lastLocationAccess != null) {
ValueNode lastAccessLocation = lastLocationAccess.asNode();
if (nodeToBlock.get(lastAccessLocation) == b
&& lastAccessLocation != beginNode
&& !(lastAccessLocation instanceof MemoryPhiNode)) {
// This node's last access location is within this block. Add to watch
// list when processing the last access location.
} else {
watchList.add(floatingReadNode);
}
}
}
}
ArrayList<Node> newList = new ArrayList<>(oldList.size());
assert oldList.get(0) == beginNode;
unprocessed.clear(beginNode);
newList.add(beginNode);
for (int i = 1; i < oldList.size(); ++i) {
Node n = oldList.get(i);
if (unprocessed.isMarked(n)) {
if (n instanceof MemoryNode) {
if (n instanceof MemoryCheckpoint) {
assert n instanceof FixedNode;
if (watchList.size() > 0) {
// Check whether we need to commit reads from the watch list.
checkWatchList(b, nodeToBlock, unprocessed, newList, watchList, n);
}
}
// Add potential dependent reads to the watch list.
for (Node usage : n.usages()) {
if (usage instanceof FloatingReadNode) {
FloatingReadNode floatingReadNode = (FloatingReadNode) usage;
if (nodeToBlock.get(floatingReadNode) == b
&& floatingReadNode.getLastLocationAccess() == n
&& !(n instanceof MemoryPhiNode)) {
watchList.add(floatingReadNode);
}
}
}
}
assert unprocessed.isMarked(n);
unprocessed.clear(n);
newList.add(n);
} else {
// This node was pulled up.
assert !(n instanceof FixedNode) : n;
}
}
for (Node n : newList) {
unprocessed.mark(n);
}
assert newList.size() == oldList.size();
blockToNodes.put(b, newList);
}
private void scheduleEarliestIterative(
BlockMap<List<Node>> blockToNodes,
NodeMap<Block> nodeToBlock,
NodeBitMap visited,
StructuredGraph graph,
boolean immutableGraph) {
BitSet floatingReads = new BitSet(cfg.getBlocks().length);
// Add begin nodes as the first entry and set the block for phi nodes.
for (Block b : cfg.getBlocks()) {
AbstractBeginNode beginNode = b.getBeginNode();
ArrayList<Node> nodes = new ArrayList<>();
nodeToBlock.set(beginNode, b);
nodes.add(beginNode);
blockToNodes.put(b, nodes);
if (beginNode instanceof AbstractMergeNode) {
AbstractMergeNode mergeNode = (AbstractMergeNode) beginNode;
for (PhiNode phi : mergeNode.phis()) {
nodeToBlock.set(phi, b);
}
} else if (beginNode instanceof LoopExitNode) {
LoopExitNode loopExitNode = (LoopExitNode) beginNode;
for (ProxyNode proxy : loopExitNode.proxies()) {
nodeToBlock.set(proxy, b);
}
}
}
NodeStack stack = new NodeStack();
// Start analysis with control flow ends.
Block[] reversePostOrder = cfg.reversePostOrder();
for (int j = reversePostOrder.length - 1; j >= 0; --j) {
Block b = reversePostOrder[j];
FixedNode endNode = b.getEndNode();
if (isFixedEnd(endNode)) {
stack.push(endNode);
nodeToBlock.set(endNode, b);
}
}
processStack(cfg, blockToNodes, nodeToBlock, visited, floatingReads, stack);
// Visit back input edges of loop phis.
boolean changed;
boolean unmarkedPhi;
do {
changed = false;
unmarkedPhi = false;
for (LoopBeginNode loopBegin : graph.getNodes(LoopBeginNode.TYPE)) {
for (PhiNode phi : loopBegin.phis()) {
if (visited.isMarked(phi)) {
for (int i = 0; i < loopBegin.getLoopEndCount(); ++i) {
Node node = phi.valueAt(i + loopBegin.forwardEndCount());
if (node != null && !visited.isMarked(node)) {
changed = true;
stack.push(node);
processStack(cfg, blockToNodes, nodeToBlock, visited, floatingReads, stack);
}
}
} else {
unmarkedPhi = true;
}
}
}
/*
* the processing of one loop phi could have marked a previously checked loop phi,
* therefore this needs to be iterative.
*/
} while (unmarkedPhi && changed);
// Check for dead nodes.
if (!immutableGraph && visited.getCounter() < graph.getNodeCount()) {
for (Node n : graph.getNodes()) {
if (!visited.isMarked(n)) {
n.clearInputs();
n.markDeleted();
}
}
}
// Add end nodes as the last nodes in each block.
for (Block b : cfg.getBlocks()) {
FixedNode endNode = b.getEndNode();
if (isFixedEnd(endNode)) {
if (endNode != b.getBeginNode()) {
addNode(blockToNodes, b, endNode);
}
}
}
if (!floatingReads.isEmpty()) {
for (Block b : cfg.getBlocks()) {
if (floatingReads.get(b.getId())) {
resortEarliestWithinBlock(b, blockToNodes, nodeToBlock, visited);
}
}
}
assert MemoryScheduleVerification.check(cfg.getStartBlock(), blockToNodes);
}
private static void sortNodesLatestWithinBlock(
Block b,
BlockMap<List<Node>> earliestBlockToNodesMap,
BlockMap<List<Node>> latestBlockToNodesMap,
NodeMap<Block> nodeMap,
BlockMap<ArrayList<FloatingReadNode>> watchListMap,
NodeBitMap unprocessed) {
List<Node> earliestSorting = earliestBlockToNodesMap.get(b);
ArrayList<Node> result = new ArrayList<>(earliestSorting.size());
ArrayList<FloatingReadNode> watchList = null;
if (watchListMap != null) {
watchList = watchListMap.get(b);
assert watchList == null || !b.getKillLocations().isEmpty();
}
AbstractBeginNode beginNode = b.getBeginNode();
if (beginNode instanceof LoopExitNode) {
LoopExitNode loopExitNode = (LoopExitNode) beginNode;
for (ProxyNode proxy : loopExitNode.proxies()) {
unprocessed.clear(proxy);
ValueNode value = proxy.value();
if (value != null && nodeMap.get(value) == b) {
sortIntoList(value, b, result, nodeMap, unprocessed, null);
}
}
}
FixedNode endNode = b.getEndNode();
FixedNode fixedEndNode = null;
if (isFixedEnd(endNode)) {
// Only if the end node is either a control split or an end node, we need to force
// it to be the last node in the schedule.
fixedEndNode = endNode;
}
for (Node n : earliestSorting) {
if (n != fixedEndNode) {
if (n instanceof FixedNode) {
assert nodeMap.get(n) == b;
checkWatchList(b, nodeMap, unprocessed, result, watchList, n);
sortIntoList(n, b, result, nodeMap, unprocessed, null);
} else if (nodeMap.get(n) == b && n instanceof FloatingReadNode) {
FloatingReadNode floatingReadNode = (FloatingReadNode) n;
LocationIdentity location = floatingReadNode.getLocationIdentity();
if (b.canKill(location)) {
// This read can be killed in this block, add to watch list.
if (watchList == null) {
watchList = new ArrayList<>();
}
watchList.add(floatingReadNode);
}
}
}
}
for (Node n : latestBlockToNodesMap.get(b)) {
assert nodeMap.get(n) == b : n;
assert !(n instanceof FixedNode);
if (unprocessed.isMarked(n)) {
sortIntoList(n, b, result, nodeMap, unprocessed, fixedEndNode);
}
}
if (endNode != null && unprocessed.isMarked(endNode)) {
sortIntoList(endNode, b, result, nodeMap, unprocessed, null);
}
latestBlockToNodesMap.put(b, result);
}
@SuppressFBWarnings(
value = "RCN_REDUNDANT_NULLCHECK_WOULD_HAVE_BEEN_A_NPE",
justification = "false positive found by findbugs")
private BlockMap<ArrayList<FloatingReadNode>> calcLatestBlocks(
SchedulingStrategy strategy,
NodeMap<Block> currentNodeMap,
BlockMap<List<Node>> earliestBlockToNodesMap,
NodeBitMap visited,
BlockMap<List<Node>> latestBlockToNodesMap,
boolean immutableGraph) {
BlockMap<ArrayList<FloatingReadNode>> watchListMap = new BlockMap<>(cfg);
Block[] reversePostOrder = cfg.reversePostOrder();
for (int j = reversePostOrder.length - 1; j >= 0; --j) {
Block currentBlock = reversePostOrder[j];
List<Node> blockToNodes = earliestBlockToNodesMap.get(currentBlock);
LocationSet killed = null;
int previousIndex = blockToNodes.size();
for (int i = blockToNodes.size() - 1; i >= 0; --i) {
Node currentNode = blockToNodes.get(i);
assert currentNodeMap.get(currentNode) == currentBlock;
assert !(currentNode instanceof PhiNode) && !(currentNode instanceof ProxyNode);
assert visited.isMarked(currentNode);
if (currentNode instanceof FixedNode) {
// For these nodes, the earliest is at the same time the latest block.
} else {
Block latestBlock = null;
LocationIdentity constrainingLocation = null;
if (currentNode instanceof FloatingReadNode) {
// We are scheduling a floating read node => check memory
// anti-dependencies.
FloatingReadNode floatingReadNode = (FloatingReadNode) currentNode;
LocationIdentity location = floatingReadNode.getLocationIdentity();
if (location.isMutable()) {
// Location can be killed.
constrainingLocation = location;
if (currentBlock.canKill(location)) {
if (killed == null) {
killed = new LocationSet();
}
fillKillSet(killed, blockToNodes.subList(i + 1, previousIndex));
previousIndex = i;
if (killed.contains(location)) {
// Earliest block kills location => we need to stay within
// earliest block.
latestBlock = currentBlock;
}
}
}
}
if (latestBlock == null) {
// We are not constraint within earliest block => calculate optimized
// schedule.
calcLatestBlock(
currentBlock,
strategy,
currentNode,
currentNodeMap,
constrainingLocation,
watchListMap,
latestBlockToNodesMap,
visited,
immutableGraph);
} else {
selectLatestBlock(
currentNode,
currentBlock,
latestBlock,
currentNodeMap,
watchListMap,
constrainingLocation,
latestBlockToNodesMap);
}
}
}
}
return watchListMap;
}
