/** Finds shortcuts, does not change the underlying graph. */
void findShortcuts(ShortcutHandler sch) {
long tmpDegreeCounter = 0;
EdgeIterator incomingEdges = vehicleInExplorer.setBaseNode(sch.getNode());
// collect outgoing nodes (goal-nodes) only once
while (incomingEdges.next()) {
int u_fromNode = incomingEdges.getAdjNode();
// accept only uncontracted nodes
if (g.getLevel(u_fromNode) != 0) continue;
double v_u_weight = incomingEdges.getDistance();
int skippedEdge1 = incomingEdges.getEdge();
int incomingEdgeOrigCount = getOrigEdgeCount(skippedEdge1);
// collect outgoing nodes (goal-nodes) only once
EdgeIterator outgoingEdges = vehicleOutExplorer.setBaseNode(sch.getNode());
// force fresh maps etc as this cannot be determined by from node alone (e.g. same from node
// but different avoidNode)
algo.clear();
tmpDegreeCounter++;
while (outgoingEdges.next()) {
int w_toNode = outgoingEdges.getAdjNode();
// add only uncontracted nodes
if (g.getLevel(w_toNode) != 0 || u_fromNode == w_toNode) {
continue;
}
// Limit weight as ferries or forbidden edges can increase local search too much.
// If we decrease the correct weight we only explore less and introduce more shortcuts.
// I.e. no change to accuracy is made.
double existingDirectWeight = v_u_weight + outgoingEdges.getDistance();
algo.setLimitWeight(existingDirectWeight)
.setLimitVisitedNodes((int) meanDegree * 100)
.setEdgeFilter(levelEdgeFilter.setAvoidNode(sch.getNode()));
dijkstraSW.start();
dijkstraCount++;
int endNode = algo.findEndNode(u_fromNode, w_toNode);
dijkstraSW.stop();
// compare end node as the limit could force dijkstra to finish earlier
if (endNode == w_toNode && algo.getWeight(endNode) <= existingDirectWeight)
// FOUND witness path, so do not add shortcut
continue;
sch.foundShortcut(
u_fromNode,
w_toNode,
existingDirectWeight,
outgoingEdges,
skippedEdge1,
incomingEdgeOrigCount);
}
}
if (sch instanceof AddShortcutHandler) {
// sliding mean value when using "*2" => slower changes
meanDegree = (meanDegree * 2 + tmpDegreeCounter) / 3;
// meanDegree = (meanDegree + tmpDegreeCounter) / 2;
}
}
private Set<List<Integer>> distributions(int bound, List<GrammarProductionElement> elems) {
distroTimer.start();
int sumSize = elems.size();
List<Integer> lowerBounds = lowerBounds(elems, bound);
List<Integer> upperBounds = upperBounds(elems, bound, lowerBounds);
Set<List<Integer>> result = distributor.distribute2(sumSize, bound, lowerBounds, upperBounds);
distroTimer.stop();
return result;
}
public void run() {
StopWatch st = null;
if (GPResourceHome.perfProfile.PROFILE) {
st = new StopWatch(false, false, true);
st.start();
}
StopWatch ct = new StopWatch();
int numWorkers = 0;
try {
if (GPResourceHome.commonState == null) {
GPResourceHome.load_common();
}
} catch (Exception e) {
e.printStackTrace();
}
ct.start();
while (true) {
if (GPResourceHome.perfProfile.PROFILE) {
st.stop();
GPResourceHome.perfProfile.addSample("notification_engine_worker", st.getElapsedTimeMX());
st.reset();
st.start();
}
GPResource gpResource = null;
try {
// logger.debug("gpResource.notificationWorkerQ.remove()...");
gpResource = (GPResource) GPResourceHome.commonState.getNextResource();
// gpResource =
// (GPResource)GPResourceHome.commonState.notificationWorkerQ.remove();//GPResourceHome.commonState.notificationQ.waitUntilNotEmpty();
boolean sentOK = false;
if (gpResource != null
&& gpResource.taskQ != null
&& gpResource.taskPendQ != null
&& (gpResource.taskQ.size() + gpResource.taskPendQ.size()) > 0)
// if (gpResource != null)
{
Task task = null;
// if (gpResource.DATA_AWARE_SCHEDULER && !gpResource.MAX_CACHE_HIT)
if (gpResource.DATA_AWARE_SCHEDULER) {
// synchronized(gpResource)
// {
// task = (Task)gpResource.taskQ.remove();
task = (Task) gpResource.taskQ.removeNoWait();
if (task == null) {
task = (Task) gpResource.taskPendQ.remove();
}
logger.debug(
"GPService:NotificationEngine:sendNotification(): removed task "
+ task.getExecutable().getId()
+ " from gpResource.taskQ... gpResource.taskQ.size() = "
+ gpResource.taskQ.size());
gpResource.taskPendQ.insert(task);
logger.debug(
"GPService:NotificationEngine:sendNotification(): inserted task "
+ task.getExecutable().getId()
+ " into gpResource.taskPendQ... gpResource.taskPendQ.size() = "
+ gpResource.taskPendQ.size());
// }
}
logger.debug(
"GPService:NotificationEngine:sendNotification(): key = "
+ String.valueOf(gpResource.resourceKey)
+ " ...");
// if (gpResourceHome.sendNotification(gpResource.resourceKey))
boolean sendNotificationTest = false;
if (gpResourceHome.useTCPCore) {
sendNotificationTest = gpResourceHome.sendNotificationTCPCore(gpResource);
}
// else if (gpResource.DATA_AWARE_SCHEDULER && !gpResource.MAX_CACHE_HIT)
else if (gpResource.DATA_AWARE_SCHEDULER) {
sendNotificationTest = gpResourceHome.sendNotification(gpResource, task);
} /*
else if (gpResourceHome.localFork)
{
sendNotificationTest = gpResourceHome.sendNotificationLocal(gpResource);
} */ else {
sendNotificationTest = gpResourceHome.sendNotification(gpResource);
}
if (sendNotificationTest) {
setNotificationsSent();
logger.debug(
"Notification for key "
+ String.valueOf(gpResource.resourceKey)
+ " sent successfully!");
// if (gpResource.DATA_AWARE_SCHEDULER)
// {
// gpResource.taskPendQ.insert(task);
// logger.debug("GPService:NotificationEngine:sendNotification(): inserted task into
// gpResource.taskPendQ... gpResource.taskPendQ.size() = " +
// gpResource.taskPendQ.size());
// }
// insert back in to try again later... this is a hack to ensure that we never run out
// of notification messages :)
// if (!gpResourceHome.useTCPCore)
// if (gpResource != null && gpResource.taskQ != null && (gpResource.taskQ.size() +
// gpResource.taskPendQ.size()) > 0 && (gpResource.taskQ.size() +
// gpResource.taskPendQ.size()) <= GPResourceHome.maxNumNotificationWorkerThreads*2)
// {
// logger.debug("there is still work to be collected and we have dipped below the
// number of notification threads, insert back into the notification queue...");
// GPResourceHome.commonState.notificationWorkerQ.insert(gpResource);
// }
} else {
logger.debug(
"Notification for key " + String.valueOf(gpResource.resourceKey) + " failed!");
// if (gpResource.DATA_AWARE_SCHEDULER && !gpResource.MAX_CACHE_HIT)
if (gpResource.DATA_AWARE_SCHEDULER) {
// synchronized(gpResource)
// {
// gpResource.taskPendQ.removeTask(task);
// should insert back at the front... fix this
// gpResource.taskQ.insertFront(task);
// }
// logger.debug("GPService:NotificationEngine:sendNotification(): inserted task " +
// task.getExecutable().getId() + " into gpResource.taskQ... gpResource.taskQ.size() =
// " + gpResource.taskQ.size());
}
// insert back in to try again later...
// if ((gpResource.taskQ.size() + gpResource.taskPendQ.size()) > 0)
// {
// logger.debug("there is still work to be collected, insert back into the
// notification queue...");
// GPResourceHome.commonState.notificationWorkerQ.insert(gpResource);
// used for throtling notifications...
/*
try
{
System.currentTimeMillis();
Thread.sleep(20);
}
catch (Exception sss)
{
} */
// }
}
}
if (MAX_NOT_PER_SEC > 0 && getNotificationsSent() >= MAX_NOT_PER_SEC) {
long timeLeft = NOT_TIME_QUANTA - ct.getElapsedTime();
if (timeLeft > 20)
try {
logger.debug(
getNotificationsSent()
+ " notifications sent in "
+ ct.getElapsedTime()
+ " ms, sleeping for "
+ timeLeft
+ " ms");
// System.currentTimeMillis();
Thread.sleep(timeLeft);
} catch (Exception sss) {
}
}
if (ct.getElapsedTime() >= NOT_TIME_QUANTA) {
if (getNotificationsSent() > 0) {
logger.debug(
"***NotificationEngineWorker(): "
+ getNotificationsSent()
+ " notifications sent to workers in "
+ ct.getElapsedTime()
+ " ms, maximum allowed notifications "
+ MAX_NOT_PER_SEC
+ " per "
+ NOT_TIME_QUANTA
+ " ms...");
}
ct.reset();
resetNotificationsSent();
ct.start();
}
// if (GPResourceHome.commonState.resourceQ.size() > 0 &&
// GPResourceHome.commonState.getWaitingTasks() <= 0)
if (GPResourceHome.commonState.getWaitingTasks() <= 0) {
try {
Thread.sleep(100);
} catch (Exception e) {
if (logger.isDebugEnabled()) e.printStackTrace();
}
}
} catch (Exception eee) {
logger.debug("Error in NotificationEngineWorker thread: " + eee);
if (logger.isDebugEnabled()) eee.printStackTrace();
// continue;
// if (gpResource != null && gpResource.taskQ != null && gpResource.taskQ.size() +
// gpResource.taskPendQ.size() > 0)
// if (gpResource != null && gpResource.taskQ != null)
// {
// logger.debug("there is still work to be collected, insert back into the notification
// queue...");
// GPResourceHome.commonState.notificationWorkerQ.insert(gpResource);
// }
}
}
}
void contractNodes() {
meanDegree = g.getAllEdges().getMaxId() / g.getNodes();
int level = 1;
counter = 0;
int initSize = sortedNodes.getSize();
int logSize =
(int) Math.round(Math.max(10, sortedNodes.getSize() / 100 * logMessagesPercentage));
if (logMessagesPercentage == 0) logSize = Integer.MAX_VALUE;
// preparation takes longer but queries are slightly faster with preparation
// => enable it but call not so often
boolean periodicUpdate = true;
StopWatch periodSW = new StopWatch();
int updateCounter = 0;
int periodicUpdatesCount =
Math.max(10, sortedNodes.getSize() / 100 * periodicUpdatesPercentage);
if (periodicUpdatesPercentage == 0) periodicUpdate = false;
// disable as preparation is slower and query time does not benefit
int lastNodesLazyUpdates =
lastNodesLazyUpdatePercentage == 0
? 0
: sortedNodes.getSize() / 100 * lastNodesLazyUpdatePercentage;
StopWatch lazySW = new StopWatch();
// Recompute priority of uncontracted neighbors.
// Without neighborupdates preparation is faster but we need them
// to slightly improve query time. Also if not applied too often it decreases the shortcut
// number.
boolean neighborUpdate = true;
if (neighborUpdatePercentage == 0) neighborUpdate = false;
StopWatch neighborSW = new StopWatch();
LevelGraphStorage lg = ((LevelGraphStorage) g);
while (!sortedNodes.isEmpty()) {
// periodically update priorities of ALL nodes
if (periodicUpdate && counter > 0 && counter % periodicUpdatesCount == 0) {
periodSW.start();
sortedNodes.clear();
int len = g.getNodes();
for (int node = 0; node < len; node++) {
if (g.getLevel(node) != 0) continue;
int priority = oldPriorities[node] = calculatePriority(node);
sortedNodes.insert(node, priority);
}
periodSW.stop();
updateCounter++;
}
if (counter % logSize == 0) {
// TODO necessary?
System.gc();
logger.info(
Helper.nf(counter)
+ ", updates:"
+ updateCounter
+ ", nodes: "
+ Helper.nf(sortedNodes.getSize())
+ ", shortcuts:"
+ Helper.nf(newShortcuts)
+ ", dijkstras:"
+ Helper.nf(dijkstraCount)
+ ", t(dijk):"
+ (int) dijkstraSW.getSeconds()
+ ", t(period):"
+ (int) periodSW.getSeconds()
+ ", t(lazy):"
+ (int) lazySW.getSeconds()
+ ", t(neighbor):"
+ (int) neighborSW.getSeconds()
+ ", meanDegree:"
+ (long) meanDegree
+ ", algo:"
+ algo.getMemoryUsageAsString()
+ ", "
+ Helper.getMemInfo());
dijkstraSW = new StopWatch();
periodSW = new StopWatch();
lazySW = new StopWatch();
neighborSW = new StopWatch();
}
counter++;
int polledNode = sortedNodes.pollKey();
if (sortedNodes.getSize() < lastNodesLazyUpdates) {
lazySW.start();
int priority = oldPriorities[polledNode] = calculatePriority(polledNode);
if (!sortedNodes.isEmpty() && priority > sortedNodes.peekValue()) {
// current node got more important => insert as new value and contract it later
sortedNodes.insert(polledNode, priority);
lazySW.stop();
continue;
}
lazySW.stop();
}
// contract!
newShortcuts += addShortcuts(polledNode);
g.setLevel(polledNode, level);
level++;
EdgeSkipIterator iter = vehicleAllExplorer.setBaseNode(polledNode);
while (iter.next()) {
int nn = iter.getAdjNode();
if (g.getLevel(nn) != 0)
// already contracted no update necessary
continue;
if (neighborUpdate && rand.nextInt(100) < neighborUpdatePercentage) {
neighborSW.start();
int oldPrio = oldPriorities[nn];
int priority = oldPriorities[nn] = calculatePriority(nn);
if (priority != oldPrio) sortedNodes.update(nn, oldPrio, priority);
neighborSW.stop();
}
if (removesHigher2LowerEdges) lg.disconnect(vehicleAllTmpExplorer, iter);
}
}
// Preparation works only once so we can release temporary data.
// The preparation object itself has to be intact to create the algorithm.
close();
logger.info(
"took:"
+ (int) allSW.stop().getSeconds()
+ ", new shortcuts: "
+ newShortcuts
+ ", "
+ prepareWeighting
+ ", "
+ prepareEncoder
+ ", removeHigher2LowerEdges:"
+ removesHigher2LowerEdges
+ ", dijkstras:"
+ dijkstraCount
+ ", t(dijk):"
+ (int) dijkstraSW.getSeconds()
+ ", t(period):"
+ (int) periodSW.getSeconds()
+ ", t(lazy):"
+ (int) lazySW.getSeconds()
+ ", t(neighbor):"
+ (int) neighborSW.getSeconds()
+ ", meanDegree:"
+ (long) meanDegree
+ ", initSize:"
+ initSize
+ ", periodic:"
+ periodicUpdatesPercentage
+ ", lazy:"
+ lastNodesLazyUpdatePercentage
+ ", neighbor:"
+ neighborUpdatePercentage);
}