/** Search was totally successful */ private void succeed() { if (_log.shouldLog(Log.INFO)) _log.info( getJobId() + ": Succeeded search for key " + _state.getTarget() + " after querying " + _state.getAttempted().size()); if (_log.shouldLog(Log.DEBUG)) _log.debug(getJobId() + ": State of successful search: " + _state); if (_keepStats) { long time = getContext().clock().now() - _state.getWhenStarted(); getContext().statManager().addRateData("netDb.successTime", time, 0); getContext() .statManager() .addRateData("netDb.successPeers", _state.getAttempted().size(), time); } if (_onSuccess != null) getContext().jobQueue().addJob(_onSuccess); _facade.searchComplete(_state.getTarget()); handleDeferred(true); resend(); }
/** Search totally failed */ protected void fail() { if (isLocal()) { if (_log.shouldLog(Log.ERROR)) _log.error( getJobId() + ": why did we fail if the target is local?: " + _state.getTarget().toBase64(), new Exception("failure cause")); succeed(); return; } if (_log.shouldLog(Log.INFO)) _log.info(getJobId() + ": Failed search for key " + _state.getTarget()); if (_log.shouldLog(Log.DEBUG)) _log.debug(getJobId() + ": State of failed search: " + _state); long time = getContext().clock().now() - _state.getWhenStarted(); int attempted = _state.getAttempted().size(); getContext().statManager().addRateData("netDb.failedAttemptedPeers", attempted, time); if (_keepStats) { getContext().statManager().addRateData("netDb.failedTime", time, 0); // _facade.fail(_state.getTarget()); } if (_onFailure != null) getContext().jobQueue().addJob(_onFailure); _facade.searchComplete(_state.getTarget()); handleDeferred(false); }
public int addDeferred(Job onFind, Job onFail, long expiration, boolean isLease) { Search search = new Search(onFind, onFail, expiration, isLease); boolean ok = true; int deferred = 0; synchronized (_deferredSearches) { if (_deferredCleared) ok = false; else _deferredSearches.add(search); deferred = _deferredSearches.size(); } if (!ok) { // race between adding deferred and search completing if (_log.shouldLog(Log.WARN)) _log.warn( "Race deferred before searchCompleting? our onFind=" + _onSuccess + " new one: " + onFind); // the following /shouldn't/ be necessary, but it doesnt hurt _facade.searchComplete(_state.getTarget()); _facade.search( _state.getTarget(), onFind, onFail, expiration - getContext().clock().now(), isLease); return 0; } else { return deferred; } }
/** * Build the database search message * * @param replyTunnelId tunnel to receive replies through * @param replyGateway gateway for the reply tunnel * @param expiration when the search should stop */ protected DatabaseLookupMessage buildMessage( TunnelId replyTunnelId, Hash replyGateway, long expiration) { DatabaseLookupMessage msg = new DatabaseLookupMessage(getContext(), true); msg.setSearchKey(_state.getTarget()); // msg.setFrom(replyGateway.getIdentity().getHash()); msg.setFrom(replyGateway); msg.setDontIncludePeers(_state.getClosestAttempted(MAX_CLOSEST)); msg.setMessageExpiration(expiration); msg.setReplyTunnel(replyTunnelId); return msg; }
protected LoadServiceResource tryResourceFromJarURL( SearchState state, String baseName, SuffixType suffixType) { // if a jar or file URL, return load service resource directly without further searching LoadServiceResource foundResource = null; if (baseName.startsWith("jar:")) { for (String suffix : suffixType.getSuffixes()) { String namePlusSuffix = baseName + suffix; try { URL url = new URL(namePlusSuffix); debugLogTry("resourceFromJarURL", url.toString()); if (url.openStream() != null) { foundResource = new LoadServiceResource(url, namePlusSuffix); debugLogFound(foundResource); } } catch (FileNotFoundException e) { } catch (MalformedURLException e) { throw runtime.newIOErrorFromException(e); } catch (IOException e) { throw runtime.newIOErrorFromException(e); } if (foundResource != null) { state.loadName = resolveLoadName(foundResource, namePlusSuffix); break; // end suffix iteration } } } else if (baseName.startsWith("file:") && baseName.indexOf("!/") != -1) { for (String suffix : suffixType.getSuffixes()) { String namePlusSuffix = baseName + suffix; try { String jarFile = namePlusSuffix.substring(5, namePlusSuffix.indexOf("!/")); JarFile file = new JarFile(jarFile); String filename = namePlusSuffix.substring(namePlusSuffix.indexOf("!/") + 2); String canonicalFilename = canonicalizePath(filename); debugLogTry("resourceFromJarURL", canonicalFilename.toString()); if (file.getJarEntry(canonicalFilename) != null) { foundResource = new LoadServiceResource( new URL("jar:file:" + jarFile + "!/" + canonicalFilename), namePlusSuffix); debugLogFound(foundResource); } } catch (Exception e) { } if (foundResource != null) { state.loadName = resolveLoadName(foundResource, namePlusSuffix); break; // end suffix iteration } } } return foundResource; }
public SearchState findFileForLoad(String file) throws AlreadyLoaded { SearchState state = new SearchState(file); state.prepareRequireSearch(file); for (LoadSearcher searcher : searchers) { if (searcher.shouldTrySearch(state)) { searcher.trySearch(state); } else { continue; } } return state; }
public void trySearch(SearchState state) { // This code exploits the fact that all .jar files will be found for the JarredScript feature. // This is where the basic extension mechanism gets fixed Library oldLibrary = state.library; // Create package name, by splitting on / and joining all but the last elements with a ".", // and downcasing them. String[] all = state.searchFile.split("/"); StringBuilder finName = new StringBuilder(); for (int i = 0, j = (all.length - 1); i < j; i++) { finName.append(all[i].toLowerCase()).append("."); } try { // Make the class name look nice, by splitting on _ and capitalize each segment, then // joining // the, together without anything separating them, and last put on "Service" at the end. String[] last = all[all.length - 1].split("_"); for (int i = 0, j = last.length; i < j; i++) { finName.append(Character.toUpperCase(last[i].charAt(0))).append(last[i].substring(1)); } finName.append("Service"); // We don't want a package name beginning with dots, so we remove them String className = finName.toString().replaceAll("^\\.*", ""); // If there is a jar-file with the required name, we add this to the class path. if (state.library instanceof JarredScript) { // It's _really_ expensive to check that the class actually exists in the Jar, so // we don't do that now. runtime .getJRubyClassLoader() .addURL(((JarredScript) state.library).getResource().getURL()); } // quietly try to load the class Class theClass = runtime.getJavaSupport().loadJavaClassQuiet(className); state.library = new ClassExtensionLibrary(theClass); } catch (Exception ee) { state.library = null; runtime.getGlobalVariables().set("$!", runtime.getNil()); } // If there was a good library before, we go back to that if (state.library == null && oldLibrary != null) { state.library = oldLibrary; } }
/** Send a search to the given peer */ protected void sendSearch(RouterInfo router) { if (router.getIdentity().equals(getContext().router().getRouterInfo().getIdentity())) { // don't search ourselves if (_log.shouldLog(Log.ERROR)) _log.error(getJobId() + ": Dont send search to ourselves - why did we try?"); return; } else { if (_log.shouldLog(Log.INFO)) _log.info( getJobId() + ": Send search to " + router.getIdentity().getHash().toBase64() + " for " + _state.getTarget().toBase64() + " w/ timeout " + getPerPeerTimeoutMs(router.getIdentity().calculateHash())); } getContext().statManager().addRateData("netDb.searchMessageCount", 1, 0); // if (_isLease || true) // always send searches out tunnels sendLeaseSearch(router); // else // sendRouterSearch(router); }
public void trySearch(SearchState state) throws RaiseException { // no library or extension found, try to load directly as a class Script script; String className = buildClassName(state.searchFile); int lastSlashIndex = className.lastIndexOf('/'); if (lastSlashIndex > -1 && lastSlashIndex < className.length() - 1 && !Character.isJavaIdentifierStart(className.charAt(lastSlashIndex + 1))) { if (lastSlashIndex == -1) { className = "_" + className; } else { className = className.substring(0, lastSlashIndex + 1) + "_" + className.substring(lastSlashIndex + 1); } } className = className.replace('/', '.'); try { Class scriptClass = Class.forName(className); script = (Script) scriptClass.newInstance(); } catch (Exception cnfe) { throw runtime.newLoadError("no such file to load -- " + state.searchFile); } state.library = new ScriptClassLibrary(script); }
protected LoadServiceResource tryResourceFromCWD( SearchState state, String baseName, SuffixType suffixType) throws RaiseException { LoadServiceResource foundResource = null; for (String suffix : suffixType.getSuffixes()) { String namePlusSuffix = baseName + suffix; // check current directory; if file exists, retrieve URL and return resource try { JRubyFile file = JRubyFile.create( runtime.getCurrentDirectory(), RubyFile.expandUserPath(runtime.getCurrentContext(), namePlusSuffix)); debugLogTry("resourceFromCWD", file.toString()); if (file.isFile() && file.isAbsolute() && file.canRead()) { boolean absolute = true; String s = namePlusSuffix; if (!namePlusSuffix.startsWith("./")) { s = "./" + s; } foundResource = new LoadServiceResource(file, s, absolute); debugLogFound(foundResource); state.loadName = resolveLoadName(foundResource, namePlusSuffix); break; } } catch (IllegalArgumentException illArgEx) { } catch (SecurityException secEx) { } } return foundResource; }
public double getGlobalCost(SearchState state) { int[] permutations = state.getIArray(); int[][] costMatrix = dataStorager.getCostMatrix(); int totalLength = costMatrix[permutations[permutations.length - 1]][permutations[0]]; for (int i = 1; i < permutations.length; i++) { totalLength += costMatrix[permutations[i - 1]][permutations[i]]; } return totalLength; }
protected Library findLibraryBySearchState(SearchState state) { Library library = super.findLibraryBySearchState(state); if (library == null) { library = findLibraryWithClassloaders(state, state.searchFile, state.suffixType); if (library != null) { state.library = library; } } return library; }
protected Library findLibraryWithClassloaders( SearchState state, String baseName, SuffixType suffixType) { for (String suffix : suffixType.getSuffixes()) { String file = baseName + suffix; LoadServiceResource resource = findFileInClasspath(file); if (resource != null) { state.loadName = resolveLoadName(resource, file); return createLibrary(state, resource); } } return null; }
protected Library findBuiltinLibrary(SearchState state, String baseName, SuffixType suffixType) { for (String suffix : suffixType.getSuffixes()) { String namePlusSuffix = baseName + suffix; debugLogTry("builtinLib", namePlusSuffix); if (builtinLibraries.containsKey(namePlusSuffix)) { state.loadName = namePlusSuffix; Library lib = builtinLibraries.get(namePlusSuffix); debugLogFound("builtinLib", namePlusSuffix); return lib; } } return null; }
/** Send the next search, or stop if its completed */ protected void searchNext() { if (_state.completed()) { if (_log.shouldLog(Log.DEBUG)) _log.debug(getJobId() + ": Already completed"); return; } if (_log.shouldLog(Log.INFO)) _log.info(getJobId() + ": Searching: " + _state); if (isLocal()) { if (_log.shouldLog(Log.INFO)) _log.info(getJobId() + ": Key found locally"); _state.complete(true); succeed(); } else if (isExpired()) { if (_log.shouldLog(Log.INFO)) _log.info(getJobId() + ": Key search expired"); _state.complete(true); fail(); } else if (_state.getAttempted().size() > MAX_PEERS_QUERIED) { if (_log.shouldLog(Log.INFO)) _log.info(getJobId() + ": Too many peers quried"); _state.complete(true); fail(); } else { // _log.debug("Continuing search"); continueSearch(); } }
public void load(String file, boolean wrap) { if (!runtime.getProfile().allowLoad(file)) { throw runtime.newLoadError("No such file to load -- " + file); } SearchState state = new SearchState(file); state.prepareLoadSearch(file); Library library = findBuiltinLibrary(state, state.searchFile, state.suffixType); if (library == null) library = findLibraryWithoutCWD(state, state.searchFile, state.suffixType); if (library == null) { library = findLibraryWithClassloaders(state, state.searchFile, state.suffixType); if (library == null) { throw runtime.newLoadError("No such file to load -- " + file); } } try { library.load(runtime, wrap); } catch (IOException e) { if (runtime.getDebug().isTrue()) e.printStackTrace(runtime.getErr()); throw newLoadErrorFromThrowable(runtime, file, e); } }
protected LoadServiceResource tryResourceFromHome( SearchState state, String baseName, SuffixType suffixType) throws RaiseException { LoadServiceResource foundResource = null; RubyHash env = (RubyHash) runtime.getObject().fastGetConstant("ENV"); RubyString env_home = runtime.newString("HOME"); if (env.has_key_p(env_home).isFalse()) { return null; } String home = env.op_aref(runtime.getCurrentContext(), env_home).toString(); String path = baseName.substring(2); for (String suffix : suffixType.getSuffixes()) { String namePlusSuffix = path + suffix; // check home directory; if file exists, retrieve URL and return resource try { JRubyFile file = JRubyFile.create( home, RubyFile.expandUserPath(runtime.getCurrentContext(), namePlusSuffix)); debugLogTry("resourceFromHome", file.toString()); if (file.isFile() && file.isAbsolute() && file.canRead()) { boolean absolute = true; String s = "~/" + namePlusSuffix; foundResource = new LoadServiceResource(file, s, absolute); debugLogFound(foundResource); state.loadName = resolveLoadName(foundResource, s); break; } } catch (IllegalArgumentException illArgEx) { } catch (SecurityException secEx) { } } return foundResource; }
/** * After we get the data we were searching for, rebroadcast it to the peers we would query first * if we were to search for it again (healing the network). */ private void resend() { DatabaseEntry ds = _facade.lookupLeaseSetLocally(_state.getTarget()); if (ds == null) { if (SHOULD_RESEND_ROUTERINFO) { ds = _facade.lookupRouterInfoLocally(_state.getTarget()); if (ds != null) _facade.sendStore( _state.getTarget(), ds, null, null, RESEND_TIMEOUT, _state.getSuccessful()); } } else { Set sendTo = _state.getRepliedPeers(); // _state.getFailed(); sendTo.addAll(_state.getPending()); int numSent = 0; for (Iterator iter = sendTo.iterator(); iter.hasNext(); ) { Hash peer = (Hash) iter.next(); RouterInfo peerInfo = _facade.lookupRouterInfoLocally(peer); if (peerInfo == null) continue; if (resend(peerInfo, (LeaseSet) ds)) numSent++; if (numSent >= MAX_LEASE_RESEND) break; } getContext().statManager().addRateData("netDb.republishQuantity", numSent, numSent); } }
boolean wasAttempted(Hash peer) { return _state.wasAttempted(peer); }
public void trySearch(SearchState state) { state.library = findLibraryWithoutCWD(state, state.searchFile, state.suffixType); }
/** * ***** always send through the lease protected DatabaseLookupMessage buildMessage(long * expiration) { DatabaseLookupMessage msg = new DatabaseLookupMessage(getContext(), true); * msg.setSearchKey(_state.getTarget()); msg.setFrom(getContext().routerHash()); * msg.setDontIncludePeers(_state.getClosestAttempted(MAX_CLOSEST)); * msg.setMessageExpiration(expiration); msg.setReplyTunnel(null); return msg; } ******* */ void replyFound(DatabaseSearchReplyMessage message, Hash peer) { long duration = _state.replyFound(peer); // this processing can take a while, so split 'er up getContext().jobQueue().addJob(new SearchReplyJob(getContext(), this, message, peer, duration)); }
public void trySearch(SearchState state) { state.library = findLibraryWithClassloaders(state, state.searchFile, state.suffixType); }
/** * Send a series of searches to the next available peers as selected by the routing table, but * making sure no more than SEARCH_BREDTH are outstanding at any time */ protected void continueSearch() { if (_state.completed()) { if (_log.shouldLog(Log.DEBUG)) _log.debug(getJobId() + ": Search already completed", new Exception("already completed")); return; } int toCheck = getBredth() - _state.getPending().size(); if (toCheck <= 0) { // too many already pending if (_log.shouldLog(Log.INFO)) _log.info( getJobId() + ": Too many searches already pending (pending: " + _state.getPending().size() + " max: " + getBredth() + ")"); requeuePending(); return; } int sent = 0; Set attempted = _state.getAttempted(); while (sent <= 0) { // boolean onlyFloodfill = onlyQueryFloodfillPeers(getContext()); boolean onlyFloodfill = true; if (_floodfillPeersExhausted && onlyFloodfill && _state.getPending().isEmpty()) { if (_log.shouldLog(Log.WARN)) _log.warn( getJobId() + ": no non-floodfill peers left, and no more pending. Searched: " + _state.getAttempted().size() + " failed: " + _state.getFailed().size()); fail(); return; } List closestHashes = getClosestRouters(_state.getTarget(), toCheck, attempted); if ((closestHashes == null) || (closestHashes.isEmpty())) { if (_state.getPending().isEmpty()) { // we tried to find some peers, but there weren't any and no one else is going to answer if (_log.shouldLog(Log.INFO)) _log.info( getJobId() + ": No peers left, and none pending! Already searched: " + _state.getAttempted().size() + " failed: " + _state.getFailed().size()); fail(); } else { // no more to try, but we might get data or close peers from some outstanding requests if (_log.shouldLog(Log.INFO)) _log.info( getJobId() + ": No peers left, but some are pending! Pending: " + _state.getPending().size() + " attempted: " + _state.getAttempted().size() + " failed: " + _state.getFailed().size()); requeuePending(); } return; } else { attempted.addAll(closestHashes); for (Iterator iter = closestHashes.iterator(); iter.hasNext(); ) { Hash peer = (Hash) iter.next(); DatabaseEntry ds = _facade.getDataStore().get(peer); if (ds == null) { if (_log.shouldLog(Log.INFO)) _log.info( "Next closest peer " + peer + " was only recently referred to us, sending a search for them"); getContext().netDb().lookupRouterInfo(peer, null, null, _timeoutMs); } else if (!(ds.getType() == DatabaseEntry.KEY_TYPE_ROUTERINFO)) { if (_log.shouldLog(Log.WARN)) _log.warn( getJobId() + ": Error selecting closest hash that wasnt a router! " + peer + " : " + ds.getClass().getName()); _state.replyTimeout(peer); } else { RouterInfo ri = (RouterInfo) ds; if (!FloodfillNetworkDatabaseFacade.isFloodfill(ri)) { _floodfillPeersExhausted = true; if (onlyFloodfill) continue; } if (ri.isHidden()) { // || // allow querying shitlisted, since its indirect // getContext().shitlist().isShitlisted(peer)) { // dont bother } else { _state.addPending(peer); sendSearch((RouterInfo) ds); sent++; } } } /* if (sent <= 0) { // the (potentially) last peers being searched for could not be, // er, searched for, so lets retry ASAP (causing either another // peer to be selected, or the whole search to fail) if (_log.shouldLog(Log.INFO)) _log.info(getJobId() + ": No new peer queued up, so we are going to requeue " + "ourselves in our search for " + _state.getTarget().toBase64()); requeuePending(0); } */ } } }
/** True if the data is already locally stored */ private boolean isLocal() { return _facade.getDataStore().isKnown(_state.getTarget()); }
public void runJob() { if (_startedOn <= 0) _startedOn = getContext().clock().now(); if (_log.shouldLog(Log.INFO)) _log.info(getJobId() + ": Searching for " + _state.getTarget()); // , getAddedBy()); searchNext(); }
protected LoadServiceResource tryResourceFromLoadPathOrURL( SearchState state, String baseName, SuffixType suffixType) { LoadServiceResource foundResource = null; // if it's a ./ baseName, use CWD logic if (baseName.startsWith("./")) { foundResource = tryResourceFromCWD(state, baseName, suffixType); if (foundResource != null) { state.loadName = resolveLoadName(foundResource, foundResource.getName()); return foundResource; } } // if it's a ~/ baseName use HOME logic if (baseName.startsWith("~/")) { foundResource = tryResourceFromHome(state, baseName, suffixType); if (foundResource != null) { state.loadName = resolveLoadName(foundResource, foundResource.getName()); return foundResource; } } // if given path is absolute, just try it as-is (with extensions) and no load path if (new File(baseName).isAbsolute() || baseName.startsWith("../")) { for (String suffix : suffixType.getSuffixes()) { String namePlusSuffix = baseName + suffix; foundResource = tryResourceAsIs(namePlusSuffix); if (foundResource != null) { state.loadName = resolveLoadName(foundResource, namePlusSuffix); return foundResource; } } return null; } Outer: for (int i = 0; i < loadPath.size(); i++) { // TODO this is really inefficient, and potentially a problem everytime anyone require's // something. // we should try to make LoadPath a special array object. RubyString entryString = loadPath.eltInternal(i).convertToString(); String loadPathEntry = entryString.asJavaString(); if (loadPathEntry.equals(".") || loadPathEntry.equals("")) { foundResource = tryResourceFromCWD(state, baseName, suffixType); if (foundResource != null) { String ss = foundResource.getName(); if (ss.startsWith("./")) { ss = ss.substring(2); } state.loadName = resolveLoadName(foundResource, ss); break Outer; } } else { boolean looksLikeJarURL = loadPathLooksLikeJarURL(loadPathEntry); for (String suffix : suffixType.getSuffixes()) { String namePlusSuffix = baseName + suffix; if (looksLikeJarURL) { foundResource = tryResourceFromJarURLWithLoadPath(namePlusSuffix, loadPathEntry); } else if (namePlusSuffix.startsWith("./")) { throw runtime.newLoadError(""); } else { foundResource = tryResourceFromLoadPath(namePlusSuffix, loadPathEntry); } if (foundResource != null) { String ss = namePlusSuffix; if (ss.startsWith("./")) { ss = ss.substring(2); } state.loadName = resolveLoadName(foundResource, ss); break Outer; // end suffix iteration } } } } return foundResource; }
protected LoadServiceResource tryResourceFromLoadPathOrURL( SearchState state, String baseName, SuffixType suffixType) { LoadServiceResource foundResource = null; // if it's a ./ baseName, use CWD logic if (baseName.startsWith("./")) { foundResource = tryResourceFromCWD(state, baseName, suffixType); if (foundResource != null) { state.loadName = foundResource.getName(); return foundResource; } } // if given path is absolute, just try it as-is (with extensions) and no load path if (new File(baseName).isAbsolute() || baseName.startsWith("../")) { for (String suffix : suffixType.getSuffixes()) { String namePlusSuffix = baseName + suffix; foundResource = tryResourceAsIs(namePlusSuffix); if (foundResource != null) { state.loadName = namePlusSuffix; return foundResource; } } return null; } Outer: for (Iterator pathIter = loadPath.getList().iterator(); pathIter.hasNext(); ) { // TODO this is really inefficient, and potentially a problem everytime anyone require's // something. // we should try to make LoadPath a special array object. String loadPathEntry = ((IRubyObject) pathIter.next()).toString(); if (loadPathEntry.equals(".") || loadPathEntry.equals("")) { foundResource = tryResourceFromCWD(state, baseName, suffixType); if (foundResource != null) { String ss = foundResource.getName(); if (ss.startsWith("./")) { ss = ss.substring(2); } state.loadName = ss; break Outer; } } else { for (String suffix : suffixType.getSuffixes()) { String namePlusSuffix = baseName + suffix; if (loadPathLooksLikeJarURL(loadPathEntry)) { foundResource = tryResourceFromJarURLWithLoadPath(namePlusSuffix, loadPathEntry); } else { foundResource = tryResourceFromLoadPath(namePlusSuffix, loadPathEntry); } if (foundResource != null) { String ss = namePlusSuffix; if (ss.startsWith("./")) { ss = ss.substring(2); } state.loadName = ss; break Outer; // end suffix iteration } } } } return foundResource; }