public String routeToString(RoutingHop[] route) { if (route == null) return ""; long timeBase = -1L; StringBuilder sb = new StringBuilder(); sb.append("["); for (RoutingHop hop : route) { if (timeBase < 0L) { timeBase = hop.getTime(); } sb.append("\n "); sb.append(hop.getIDAddressPair()); sb.append(" ("); sb.append(hop.getTime() - timeBase); sb.append(")"); } sb.append("\n]"); return sb.toString(); }
private boolean forwardOrReturnResult( Message msg, RoutingContext[] lastRoutingContexts, final IDAddressPair[][] nextHopCands) { IDAddressPair lastHop = (IDAddressPair) msg.getSource(); int routingID = ((AbstractRecRouteMessage) msg).routingID; final ID[] targets = ((AbstractRecRouteMessage) msg).target; RoutingContext[] routingContexts = ((AbstractRecRouteMessage) msg).cxt; int numResponsibleNodeCands = ((AbstractRecRouteMessage) msg).numRespNodeCands; IDAddressPair initiator = ((AbstractRecRouteMessage) msg).initiator; int ttl = ((AbstractRecRouteMessage) msg).ttl; RoutingHop[] route = ((AbstractRecRouteMessage) msg).route; IDAddressPair[] blackList = ((AbstractRecRouteMessage) msg).blackList; int callbackTag = -1; Serializable[][] callbackArgs = null; if (msg instanceof RecRouteInvokeMessage) { callbackTag = ((RecRouteInvokeMessage) msg).callbackTag; callbackArgs = ((RecRouteInvokeMessage) msg).callbackArgs; } // { // StringBuilder sb = new StringBuilder(); // sb.append("forRetResult called:\n"); // for (ID id: targets) sb.append(" " + id.toString(-1)); // sb.append("\n"); // sb.append(" on " + getSelfIDAddressPair().getAddress() + "\n"); // sb.append(" msg: " + msg.getName() + "\n"); // sb.append(" route:"); // for (RoutingHop h: route) sb.append(" " + h.toString(-1)); // sb.append("\n"); // System.out.print(sb.toString()); // } boolean ttlExpired = false; boolean succeed = true; Message newMsg; boolean[] forwarded = new boolean[targets.length]; for (int i = 0; i < forwarded.length; i++) forwarded[i] = false; Set<IDAddressPair> blackListSet = new HashSet<IDAddressPair>(); if (blackList != null) { for (IDAddressPair a : blackList) { blackListSet.add(a); } } // add this node itself to the resulting route RoutingHop[] lastRoute = route; route = new RoutingHop[lastRoute.length + 1]; System.arraycopy(lastRoute, 0, route, 0, lastRoute.length); route[route.length - 1] = RoutingHop.newInstance(getSelfIDAddressPair()); // TTL check if (ttl < 0) { StringBuilder sb = new StringBuilder(); sb.append("TTL expired (target"); for (ID t : targets) { sb.append(" ").append(t.toString(-1)); } sb.append("):"); for (RoutingHop h : route) { if (h == null) break; sb.append(" "); sb.append(h.getIDAddressPair().toString(-1)); } logger.log(Level.WARNING, sb.toString(), new Throwable()); ttlExpired = true; if (!(msg instanceof RecRouteJoinMessage)) { // allow joining node to succeed succeed = false; } } IDAddressPair[] nextHops = new IDAddressPair[targets.length]; forward: while (true) { if (ttlExpired) break; do { // ... } while (false) Set<IDAddressPair> contactSet = new HashSet<IDAddressPair>(); boolean allContactsAreNull = true; boolean aContactIsNull = false; for (int i = 0; i < targets.length; i++) { if (nextHopCands[i] == null || nextHopCands[i].length <= 0) { nextHops[i] = null; } nextHops[i] = nextHopCands[i][0]; if (nextHops[i] == null) continue; if (blackListSet.contains(nextHops[i].getAddress())) { // next hop is in the black list nextHops[i] = null; System.arraycopy(nextHopCands[i], 1, nextHopCands[i], 0, nextHopCands[i].length - 1); continue; } if (msg instanceof RecRouteJoinMessage && nextHops[i].getAddress().equals(initiator.getAddress())) { // next hop is initiator of routing nextHops[i] = null; System.arraycopy(nextHopCands[i], 1, nextHopCands[i], 0, nextHopCands[i].length - 1); i--; logger.log( Level.WARNING, "Next hop is the joining node " + initiator.getAddress() + ". RoutingAlgorithm#touch() has been called too early?"); continue; } // StringBuilder sb = new StringBuilder(); // sb.append("judge to terminate[" + i + "]: " + targets[i].toString(-1) + "\n"); // sb.append(" on: " + this.getSelfIDAddressPair().toString(-1) + ":\n"); // sb.append(" nextHop: " + nextHops[i].toString(-1) + "\n"); // sb.append(" " + // nextHops[i].getAddress().equals(this.getSelfIDAddressPair().getAddress()) + "\n"); // sb.append(" context: " + (routingContexts[i] != null ? // routingContexts[i].toString(-1) : "null") + "\n"); // sb.append(" lastContext: " + (lastRoutingContexts[i] != null ? // lastRoutingContexts[i].toString(-1) : "null") + "\n"); // if (routingContexts[i] != null && lastRoutingContexts[i] != null) // sb.append(" " + routingContexts[i].equals(lastRoutingContexts[i]) + "\n"); if (nextHops[i].getAddress().equals(this.getSelfIDAddressPair().getAddress()) && (routingContexts[i] == null || routingContexts[i].equals(lastRoutingContexts[i]))) { // next hop is this node itself nextHops[i] = null; // terminates routing // sb.append(" terminate.\n"); } // System.out.print(sb.toString()); if (nextHops[i] != null) { contactSet.add(nextHops[i]); allContactsAreNull = false; } else { contactSet.add(null); aContactIsNull = true; } } if (allContactsAreNull) { // this node is the responsible node break forward; } // fork if (contactSet.size() > 1 || aContactIsNull) { // System.out.println("fork on " + getSelfIDAddressPair().getAddress()); Set<Forwarder> forkedForwarder = new HashSet<Forwarder>(); List<Integer> contactIndexList = new ArrayList<Integer>(); for (IDAddressPair c : contactSet) { contactIndexList.clear(); if (c == null) { for (int i = 0; i < targets.length; i++) if (nextHops[i] == null) contactIndexList.add(i); } else { for (int i = 0; i < targets.length; i++) if (c.equals(nextHops[i])) contactIndexList.add(i); } int nTgts = contactIndexList.size(); final ID[] forkedTarget = new ID[nTgts]; final RoutingContext[] forkedRoutingContext = new RoutingContext[nTgts]; final RoutingContext[] forkedLastRoutingContext = new RoutingContext[nTgts]; final IDAddressPair[][] forkedNextHopCands = new IDAddressPair[nTgts][]; for (int i = 0; i < nTgts; i++) { int index = contactIndexList.get(i); forkedTarget[i] = targets[index]; forkedRoutingContext[i] = routingContexts[index]; forkedLastRoutingContext[i] = lastRoutingContexts[index]; forkedNextHopCands[i] = nextHopCands[index]; } Serializable[][] forkedCallbackArgs = null; if (callbackArgs != null) { forkedCallbackArgs = new Serializable[nTgts][]; for (int i = 0; i < nTgts; i++) { int index = contactIndexList.get(i); forkedCallbackArgs[i] = callbackArgs[index]; } } RoutingHop[] copiedRoute = new RoutingHop[route.length]; System.arraycopy(route, 0, copiedRoute, 0, route.length); Forwarder f = new Forwarder( RecursiveRoutingDriver.getRecRouteMessage( msg.getClass(), routingID, forkedTarget, forkedRoutingContext, numResponsibleNodeCands, initiator, ttl, copiedRoute, blackList, callbackTag, forkedCallbackArgs), forkedLastRoutingContext, forkedNextHopCands); forkedForwarder.add(f); } // execute boolean ret = true; if (config.getUseThreadPool()) { Set<Future<Boolean>> fSet = new HashSet<Future<Boolean>>(); Forwarder firstForwarder = null; ExecutorService ex = SingletonThreadPoolExecutors.getThreadPool( ExecutorBlockingMode.CONCURRENT_REJECTING, Thread.currentThread().isDaemon()); for (Forwarder forwarder : forkedForwarder) { if (firstForwarder == null) { firstForwarder = forwarder; continue; } try { Future<Boolean> f = ex.submit((Callable<Boolean>) forwarder); fSet.add(f); } catch (RejectedExecutionException e) { // invoke directly if rejected // Note that this is required to avoid deadlocks ret &= forwarder.call(); } } ret &= firstForwarder.call(); // direct invocation for (Future<Boolean> f : fSet) { try { ret &= f.get(); } catch (Exception e) { /*ignore*/ } } } else { Set<Thread> tSet = new HashSet<Thread>(); for (Runnable r : forkedForwarder) { Thread t = new Thread(r); t.setName("Forwarder"); t.setDaemon(Thread.currentThread().isDaemon()); tSet.add(t); t.start(); } for (Thread t : tSet) { try { t.join(); } catch (InterruptedException e) { /*ignore*/ } } for (Forwarder f : forkedForwarder) { ret &= f.getResult(); } } return ret; } // if (contactSet.size() > 1 || aContactIsNull) { // fork // System.out.println("forward or reply on " + getSelfIDAddressPair().getAddress()); IDAddressPair nextHop = nextHops[0]; // assert: all nextHops[i].getIDAddressPair() is the same value // System.out.println("On " + getSelfIDAddressPair().getAddress() + ", nextHop: " + // nextHop); // System.out.println(" target: " + target[0]); // prepare a Message newMsg = RecursiveRoutingDriver.getRecRouteMessage( msg.getClass(), routingID, targets, routingContexts, numResponsibleNodeCands, initiator, ttl - 1, route, blackList, callbackTag, callbackArgs); try { Message ack = sender.sendAndReceive(nextHop.getAddress(), newMsg); // throws IOException // System.out.println("On " + getSelfIDAddressPair().getAddress() + ", forwarded " + // Tag.getNameByNumber(oldMsgTag) + " from " + getSelfIDAddressPair().getAddress() + " to // " + nextHop.getAddress()); // fill ID of nextHop for (int i = 0; i < targets.length; i++) { if (nextHops[i].getID() == null) { // this is the case in the first iteration of joining nextHops[i].setID(((IDAddressPair) ack.getSource()).getID()); } } // notify the routing algorithm if (algorithm != null) { algorithm.touch((IDAddressPair) ack.getSource()); } if (ack instanceof RecAckMessage) { for (int i = 0; i < forwarded.length; i++) forwarded[i] = true; break forward; } else { logger.log(Level.SEVERE, "Received message is not REC_ACK."); } } catch (IOException e) { // System.out.println(" failed."); // sending failure and try the next node logger.log( Level.WARNING, "Failed to forward a request to " + nextHop.getAddress() + " on " + getSelfIDAddressPair().getAddress(), e); } // fail to send/receive if (nextHop.getID() != null) { // nextHop.getID() is null when joining super.fail(nextHop); if (blackList != null) { IDAddressPair[] oldBlackList = blackList; blackList = new IDAddressPair[oldBlackList.length + 1]; System.arraycopy(oldBlackList, 0, blackList, 0, oldBlackList.length); } else { blackList = new IDAddressPair[1]; } blackList[blackList.length - 1] = nextHop; blackListSet.add(nextHop); logger.log( Level.INFO, nextHop.getAddress() + " is added to blacklist on " + this.getSelfIDAddressPair().getAddress()); } } while (false); // shift nextHopCands[i] shiftNextHopCands: for (int i = 0; i < targets.length; i++) { if (nextHopCands[i] == null) continue; System.arraycopy(nextHopCands[i], 1, nextHopCands[i], 0, nextHopCands[i].length - 1); for (int j = 0; j < nextHopCands[i].length; j++) { if (nextHopCands[i][j] != null) continue shiftNextHopCands; } nextHopCands[i] = null; } } // forward: while (true) // notify the routing algorithm if (lastHop != null) algorithm.touch(lastHop); // source of message // this is an additional call to touch() compared with iterative lookup if (!this.getSelfIDAddressPair().equals(initiator)) algorithm.touch(initiator); // initiator of message // message dependent processes Serializable[] callbackResult = new Serializable[targets.length]; if (msg instanceof RecRouteInvokeMessage) { // invoke callbacks for (int i = 0; i < targets.length; i++) { callbackResult[i] = invokeCallbacks(targets[i], callbackTag, callbackArgs[i], lastHop, !forwarded[i]); if (callbackResult[i] != null) { logger.log(Level.INFO, "A callback returned non-null object: " + callbackResult[i]); } } } else if (msg instanceof RecRouteJoinMessage) { final IDAddressPair copiedJoiningNode = initiator; final IDAddressPair copiedLastHop = lastHop; final boolean[] copiedForwarded = new boolean[forwarded.length]; System.arraycopy(forwarded, 0, copiedForwarded, 0, copiedForwarded.length); Runnable r = new Runnable() { public void run() { for (int i = 0; i < targets.length; i++) { algorithm.join(copiedJoiningNode, copiedLastHop, !copiedForwarded[i]); } } }; try { if (config.getUseThreadPool()) { ExecutorService ex = SingletonThreadPoolExecutors.getThreadPool( ExecutorBlockingMode.CONCURRENT_NON_BLOCKING, Thread.currentThread().isDaemon()); ex.submit(r); } else { Thread t = new Thread(r); t.setName("Message type specific processes"); t.setDaemon(Thread.currentThread().isDaemon()); t.start(); } } catch (OutOfMemoryError e) { logger.log(Level.SEVERE, "# of threads: " + Thread.activeCount(), e); // Thread[] tarray = new Thread[Thread.activeCount()]; // Thread.enumerate(tarray); // for (Thread t: tarray) System.out.println("Th: " + t.getName()); // System.out.flush(); throw e; } } // reports the routing result to the initiator List<Integer> notForwardedIndexList = new ArrayList<Integer>(); for (int i = 0; i < targets.length; i++) { if (!forwarded[i]) notForwardedIndexList.add(i); } if (!notForwardedIndexList.isEmpty()) { // get candidates for the responsible node ID[] partOfTarget = new ID[notForwardedIndexList.size()]; RoutingResult[] partOfResult = new RoutingResult[notForwardedIndexList.size()]; Serializable[] partOfCallbackResult = new Serializable[notForwardedIndexList.size()]; for (int i = 0; i < notForwardedIndexList.size(); i++) { // target partOfTarget[i] = targets[notForwardedIndexList.get(i)]; // routing result IDAddressPair[] respCands = algorithm.responsibleNodeCandidates(partOfTarget[i], numResponsibleNodeCands); if (msg instanceof RecRouteJoinMessage && initiator.equals(respCands[0])) { // remove initiator from the first place on the responsible node candidates list IDAddressPair[] orig = respCands; respCands = new IDAddressPair[respCands.length - 1]; System.arraycopy(orig, 1, respCands, 0, respCands.length); } partOfResult[i] = new RoutingResult(route, respCands); // callback result partOfCallbackResult[i] = callbackResult[notForwardedIndexList.get(i)]; } // this node is the destination, or failed to send Message repMsg = new RecResultMessage( routingID, succeed, partOfTarget, partOfResult, blackList, partOfCallbackResult); try { sender.send(initiator.getAddress(), repMsg); // System.out.println("replied from " + getSelfIDAddressPair().getAddress() + " to " + // initiator.getAddress() // + " for " + targets[0].toString(-1) + ".."); for (int i : notForwardedIndexList) { forwarded[i] = true; } } catch (IOException e) { // sending failure logger.log( Level.WARNING, "Failed to report to the initiator: " + initiator.getAddress() + " on " + getSelfIDAddressPair().getAddress()); super.fail(initiator); } } // if (!notForwardedIndexList.isEmpty()) boolean ret = true; for (boolean b : forwarded) ret &= b; return ret; }
private RoutingResult[] route0( Class<? extends Message> msgClass, ID[] target, RoutingContext[] routingContexts, int numResponsibleNodeCands, Serializable[][] resultingCallbackResult, int callbackTag, Serializable[][] callbackArgs, MessagingAddress joinInitialContact) { IDAddressPair[][] nextHopCands = new IDAddressPair[target.length][]; IDAddressPair[] blackList = null; if (numResponsibleNodeCands < 1) numResponsibleNodeCands = 1; int routingID = Thread.currentThread().hashCode(); RoutingContext[] lastRoutingContexts = new RoutingContext[target.length]; if (routingContexts == null) routingContexts = new RoutingContext[target.length]; if (!msgClass.equals(RecRouteJoinMessage.class)) { for (int i = 0; i < target.length; i++) { if (routingContexts[i] == null) routingContexts[i] = algorithm.initialRoutingContext(target[i]); } } // notify messaging visualizer if (!msgClass.equals(RecRouteJoinMessage.class)) { MessagingReporter msgReporter = receiver.getMessagingReporter(); msgReporter.notifyStatCollectorOfEmphasizeNode(this.getSelfIDAddressPair().getID()); msgReporter.notifyStatCollectorOfMarkedID(target, 0); } // forward if (msgClass.equals(RecRouteJoinMessage.class)) { for (int i = 0; i < target.length; i++) { nextHopCands[i] = new IDAddressPair[1]; nextHopCands[i][0] = IDAddressPair.getIDAddressPair(null, joinInitialContact); } } else { // REC_ROUTE_NONE || REC_ROUTE_INVOKE for (int i = 0; i < target.length; i++) { nextHopCands[i] = algorithm.nextHopCandidates( target[i], null, false, this.config.getNumOfNextHopCandidatesRequested(), routingContexts[i]); } } // put a null Message as a marker Message nullMsg = new NullMessage(); synchronized (this.routeResultMsgTable) { for (int i = 0; i < target.length; i++) { this.routeResultMsgTable.put(target[i].hashCode() ^ routingID, nullMsg); } } Message msg = RecursiveRoutingDriver.getRecRouteMessage( msgClass, routingID, target, routingContexts, numResponsibleNodeCands, this.getSelfIDAddressPair(), config.getTTL(), new RoutingHop[0], blackList, callbackTag, callbackArgs); forwardOrReturnResult(msg, lastRoutingContexts, nextHopCands); // wait for REC_RESULT messages RoutingResult[] ret = new RoutingResult[target.length]; Set<Integer> failedIndexSet = new HashSet<Integer>(); long sleepLimit = Timer.currentTimeMillis() + config.getRoutingTimeout(); waitForResults: while (true) { Message resultMsg = null; RoutingResult[] result; retrieveMessage: while (true) { // peek a received message synchronized (this.routeResultMsgTable) { for (int i = 0; i < target.length; i++) { if (ret[i] == null) resultMsg = this.routeResultMsgTable.get(target[i].hashCode() ^ routingID); if (resultMsg != null && !(resultMsg instanceof NullMessage)) { break retrieveMessage; } resultMsg = null; } } // sleep long sleepPeriod = sleepLimit - Timer.currentTimeMillis(); if (sleepPeriod <= 0L) { // clean up result message table synchronized (this.routeResultMsgTable) { for (ID id : target) this.routeResultMsgTable.remove(id.hashCode() ^ routingID); } break waitForResults; } try { synchronized (nullMsg) { nullMsg.wait(sleepPeriod); } } catch (InterruptedException e) { sleepLimit = Timer.currentTimeMillis(); } } // retrieveMessage: while (true) int rtID = ((RecResultMessage) resultMsg).routingID; boolean succeed = ((RecResultMessage) resultMsg).succeed; ID[] tgt = ((RecResultMessage) resultMsg).target; result = ((RecResultMessage) resultMsg).routingRes; Serializable[] callbackResult = ((RecResultMessage) resultMsg).callbackResult; if (rtID != routingID) continue waitForResults; // prepare RoutingResult and callback results for (int i = 0; i < tgt.length; i++) { boolean match = false; for (int j = 0; j < target.length; j++) { if (tgt[i].equals(target[j])) { match = true; ret[j] = result[i]; synchronized (this.routeResultMsgTable) { this.routeResultMsgTable.remove(target[j].hashCode() ^ routingID); } if (!succeed) failedIndexSet.add(j); if (resultingCallbackResult != null && resultingCallbackResult[j] != null) { resultingCallbackResult[j][0] = callbackResult[i]; } } } if (!match) { logger.log( Level.WARNING, "Received REC_RESULT message is not for an expected target: " + tgt[i]); } } // notify the routing algorithm of nodes on the route for (RoutingResult res : result) { RoutingHop[] route = res.getRoute(); IDAddressPair selfIDAddress = this.getSelfIDAddressPair(); for (RoutingHop h : route) { IDAddressPair p = h.getIDAddressPair(); if (p == null || selfIDAddress.equals(p)) continue; algorithm.touch(p); } } // break if filled boolean filled = true; for (int i = 0; i < target.length; i++) { if (ret[i] == null) filled = false; } if (filled) break; } // waitForResults: while (true) Set<ID> noResultTarget = new HashSet<ID>(); for (int i = 0; i < target.length; i++) { if (ret[i] == null) { noResultTarget.add(target[i]); } } if (!noResultTarget.isEmpty()) { StringBuilder sb = new StringBuilder(); sb.append("Could not receive a REC_RESULT message for the target"); for (ID id : noResultTarget) { sb.append(" ").append(id); } logger.log(Level.WARNING, sb.toString()); } for (int index : failedIndexSet) { ret[index] = null; } return ret; }