public void testSimplePushDownload() throws Exception { int successfulPushes = ((AtomicInteger) ((Map) statsTracker.inspect()).get("push connect success")).intValue(); LOG.info("-Testing non-swarmed push download"); GUID guid = new GUID(); AlternateLocation pushLoc = alternateLocationFactory.create( guid.toHexString() + ";127.0.0.1:" + PPORT_1, TestFile.hash()); ((PushAltLoc) pushLoc).updateProxies(true); RemoteFileDesc rfd = newRFDPush(guid, PPORT_1, 1); assertTrue(rfd.getAddress() instanceof PushEndpoint); RemoteFileDesc[] rfds = {rfd}; TestUploader uploader = injector.getInstance(TestUploader.class); uploader.start("push uploader"); testUDPAcceptorFactoryImpl.createTestUDPAcceptor( PPORT_1, networkManager.getPort(), savedFile.getName(), uploader, guid, _currentTestName); tGeneric(rfds); assertEquals( successfulPushes + 1, ((AtomicInteger) ((Map) statsTracker.inspect()).get("push connect success")).intValue()); }
/** Returns this Response as a RemoteFileDesc. */ public RemoteFileDesc toRemoteFileDesc(HostData data) { if (cachedRFD != null && cachedRFD.getPort() == data.getPort() && cachedRFD.getHost().equals(data.getIP())) return cachedRFD; else { RemoteFileDesc rfd = new RemoteFileDesc( data.getIP(), data.getPort(), getIndex(), getName(), (int) getSize(), data.getClientGUID(), data.getSpeed(), data.isChatEnabled(), data.getQuality(), data.isBrowseHostEnabled(), getDocument(), getUrns(), data.isReplyToMulticastQuery(), data.isFirewalled(), data.getVendorCode(), System.currentTimeMillis(), data.getPushProxies(), getCreateTime(), data.getFWTVersionSupported()); cachedRFD = rfd; return rfd; } }
public synchronized RemoteFileDesc getBest() throws NoSuchElementException { if (!hasMore()) return null; RemoteFileDesc ret; // try a verified host if (!verifiedHosts.isEmpty()) { LOG.debug("getting a verified host"); ret = (RemoteFileDesc) verifiedHosts.first(); verifiedHosts.remove(ret); } else { LOG.debug("getting a non-verified host"); // use the legacy ranking logic to select a non-verified host Iterator dual = new DualIterator(testedLocations.iterator(), newHosts.iterator()); ret = LegacyRanker.getBest(dual); newHosts.remove(ret); testedLocations.remove(ret); if (ret.needsPush()) { for (Iterator iter = ret.getPushProxies().iterator(); iter.hasNext(); ) pingedHosts.remove(iter.next()); } else pingedHosts.remove(ret); } pingNewHosts(); if (LOG.isDebugEnabled()) LOG.debug("the best host we came up with is " + ret + " " + ret.getPushAddr()); return ret; }
private boolean addInternal(RemoteFileDesc host) { // initialize the sha1 if we don't have one if (sha1 == null) { if (host.getSHA1Urn() != null) sha1 = host.getSHA1Urn(); else // BUGFIX: We can't discard sources w/out a SHA1 when we dont' have // a SHA1 for the download, or else it won't be possible to download a // file from a query hit without a SHA1, if we can received UDP pings return testedLocations.add(host); // we can't do anything yet } // do not allow duplicate hosts if (running && knowsAboutHost(host)) return false; if (LOG.isDebugEnabled()) LOG.debug("adding new host " + host + " " + host.getPushAddr()); boolean ret = false; // don't bother ranking multicasts if (host.isReplyToMulticast()) ret = verifiedHosts.add(host); else ret = newHosts.add(host); // rank // make sure that if we were stopped, we return true ret = ret | !running; // initialize the guid if we don't have one if (myGUID == null && meshHandler != null) { myGUID = new GUID(GUID.makeGuid()); RouterService.getMessageRouter().registerMessageListener(myGUID.bytes(), this); } return ret; }
/** * tests that a download from a push location becomes an alternate location. * * <p>It creates a push uploader from which we must create a PushLoc. After a while, two open * uploaders join the swarm -one which is interested in receiving push locs and one which isn't. * The interested one should receive the push loc, the other one should not. */ public void testPusherBecomesPushLocAndSentToInterested() throws Exception { LOG.info("-Testing push download creating a push location..."); final int RATE = 200; testUploaders[0].setRate(RATE); testUploaders[0].setInterestedInFalts(true); testUploaders[0].stopAfter(600000); testUploaders[1].setRate(RATE); testUploaders[1].setInterestedInFalts(false); testUploaders[1].stopAfter(300000); TestUploader pusher = injector.getInstance(TestUploader.class); pusher.start("push uploader"); pusher.setRate(RATE); pusher.stopAfter(200000); GUID guid = new GUID(); AlternateLocation pushLoc = alternateLocationFactory.create( guid.toHexString() + ";127.0.0.2:" + PPORT_1, TestFile.hash()); RemoteFileDesc pushRFD = newRFDPush(guid, PPORT_1, 1, 2); PushEndpoint pushEndpoint = (PushEndpoint) pushRFD.getAddress(); assertEquals(0, pushEndpoint.getFWTVersion()); RemoteFileDesc openRFD1 = newRFDWithURN(PORTS[0], TestFile.hash().toString(), false); RemoteFileDesc openRFD2 = newRFDWithURN(PORTS[1], TestFile.hash().toString(), false); RemoteFileDesc[] now = {pushRFD}; HashSet<RemoteFileDesc> later = new HashSet<RemoteFileDesc>(); later.add(openRFD1); later.add(openRFD2); testUDPAcceptorFactoryImpl.createTestUDPAcceptor( PPORT_1, networkManager.getPort(), savedFile.getName(), pusher, guid, _currentTestName); ManagedDownloader download = (ManagedDownloader) downloadServices.download(now, RemoteFileDesc.EMPTY_LIST, null, false); Thread.sleep(1000); download.addDownload(later, false); waitForComplete(); assertGreaterThan("u1 did no work", 100000, testUploaders[0].getAmountUploaded()); assertGreaterThan("u2 did no work", 100000, testUploaders[1].getAmountUploaded()); assertLessThan("u2 did too much work", 550 * 1024, testUploaders[1].getAmountUploaded()); assertGreaterThan("pusher did no work", 100 * 1024, pusher.getAmountUploaded()); List alc = testUploaders[0].getIncomingGoodAltLocs(); assertTrue("interested uploader did not get pushloc", alc.contains(pushLoc)); alc = testUploaders[1].getIncomingGoodAltLocs(); assertFalse("not interested uploader got pushloc", alc.contains(pushLoc)); alc = pusher.getIncomingGoodAltLocs(); assertFalse("not interested uploader got pushloc", alc.contains(pushLoc)); }
/** Informs the Ranker that a host has replied with a HeadPing */ public void processMessage(Message m, ReplyHandler handler) { MeshHandler mesh; RemoteFileDesc rfd; Collection alts = null; // this -> meshHandler NOT ok synchronized (this) { if (!running) return; if (!(m instanceof HeadPong)) return; HeadPong pong = (HeadPong) m; if (!pingedHosts.containsKey(handler)) return; rfd = (RemoteFileDesc) pingedHosts.remove(handler); testedLocations.remove(rfd); if (LOG.isDebugEnabled()) { LOG.debug( "received a pong " + pong + " from " + handler + " for rfd " + rfd + " with PE " + rfd.getPushAddr()); } // older push proxies do not route but respond directly, we want to get responses // from other push proxies if (!pong.hasFile() && !pong.isGGEPPong() && rfd.needsPush()) return; // if the pong is firewalled, remove the other proxies from the // pinged set if (pong.isFirewalled()) { for (Iterator iter = rfd.getPushProxies().iterator(); iter.hasNext(); ) pingedHosts.remove(iter.next()); } mesh = meshHandler; if (pong.hasFile()) { // update the rfd with information from the pong pong.updateRFD(rfd); // if the remote host is busy, re-add him for later ranking if (rfd.isBusy()) newHosts.add(rfd); else verifiedHosts.add(rfd); alts = pong.getAllLocsRFD(rfd); } } // if the pong didn't have the file, drop it // otherwise add any altlocs the pong had to our known hosts if (alts == null) mesh.informMesh(rfd, false); else mesh.addPossibleSources(alts); }
public int compare(Object a, Object b) { RemoteFileDesc rfd1 = (RemoteFileDesc) a; RemoteFileDesc rfd2 = (RemoteFileDesc) b; if (rfd1.isFromAlternateLocation() != rfd2.isFromAlternateLocation()) { if (rfd1.isFromAlternateLocation()) return 1; else return -1; } return 0; }
/** pings a bunch of hosts if necessary */ private void pingNewHosts() { // if we have reached our desired # of altlocs, don't ping if (isCancelled()) return; // if we don't have anybody to ping, don't ping if (!hasNonBusy()) return; // if we haven't found a single RFD with URN, don't ping anybody if (sha1 == null) return; // if its not time to ping yet, don't ping // use the same interval as workers for now long now = System.currentTimeMillis(); if (now - lastPingTime < DownloadSettings.WORKER_INTERVAL.getValue()) return; // create a ping for the non-firewalled hosts HeadPing ping = new HeadPing(myGUID, sha1, getPingFlags()); // prepare a batch of hosts to ping int batch = DownloadSettings.PING_BATCH.getValue(); List toSend = new ArrayList(batch); int sent = 0; for (Iterator iter = newHosts.iterator(); iter.hasNext() && sent < batch; ) { RemoteFileDesc rfd = (RemoteFileDesc) iter.next(); if (rfd.isBusy(now)) continue; iter.remove(); if (rfd.needsPush()) { if (rfd.getPushProxies().size() > 0 && rfd.getSHA1Urn() != null) pingProxies(rfd); } else { pingedHosts.put(rfd, rfd); toSend.add(rfd); } testedLocations.add(rfd); sent++; } if (LOG.isDebugEnabled()) { LOG.debug( "\nverified hosts " + verifiedHosts.size() + "\npingedHosts " + pingedHosts.values().size() + "\nnewHosts " + newHosts.size() + "\npinging hosts: " + sent); } pinger.rank(toSend, null, this, ping); lastPingTime = now; }
/** * Creates a new LimeWire result. * * @param sha1 The hash of the file. * @param rfd The base descriptor for the file. * @param hd The host data. * @param ipPorts The endpoints for the file. * @throws URISyntaxException If there's an error creating any associated URIs. */ public LimeWireResult( final URI sha1, final RemoteFileDesc rfd, final GUID queryGUID, final Set<? extends IpPort> ipPorts) throws URISyntaxException { super( rfd.getFileName(), sha1, createThumbnailUrl(rfd.getFileName()), -1, -1, rfd.getSize(), "limewire", sha1, null); this.m_queryGUID = queryGUID; this.m_alts.addAll(ipPorts); this.m_rfds.add(rfd); }
/** * Constructs a new RemoteFileDesc exactly like the other one, but with a different remote host. * * <p>It is okay to use the same internal structures for URNs because the Set is immutable. */ public RemoteFileDesc(RemoteFileDesc rfd, IpPort ep) { this( ep.getAddress(), // host ep.getPort(), // port COPY_INDEX, // index (unknown) rfd.getFileName(), // filename rfd.getSize(), // filesize DataUtils.EMPTY_GUID, // client GUID 0, // speed false, // chat capable 2, // quality false, // browse hostable rfd.getUrns(), // urns false, // reply to MCast false, // is firewalled AlternateLocation.ALT_VENDOR, // vendor System.currentTimeMillis(), // timestamp Collections.EMPTY_SET, // push proxies rfd.getCreationTime(), // creation time 0); // firewalled transfer }
/* * Returns true if both rfd "have the same content". Currently * rfd1~=rfd2 iff either of the following conditions hold: * * <ul> * <li>Both files have the same hash, i.e., * rfd1.getSHA1Urn().equals(rfd2.getSHA1Urn(). Note that this (almost) * always means that rfd1.getSize()==rfd2.getSize(), though rfd1 and * rfd2 may have different names. * <li>Both files have the same name and size and don't have conflicting * hashes, i.e., rfd1.getName().equals(rfd2.getName()) && * rfd1.getSize()==rfd2.getSize() && (rfd1.getSHA1Urn()==null || * rfd2.getSHA1Urn()==null || * rfd1.getSHA1Urn().equals(rfd2.getSHA1Urn())). * </ul> * Note that the second condition allows risky resumes, i.e., resumes when * one (or both) of the files doesn't have a hash. * * @see getFile */ static boolean same(RemoteFileDesc rfd1, RemoteFileDesc rfd2) { return same( rfd1.getFileName(), rfd1.getSize(), rfd1.getSHA1Urn(), rfd2.getFileName(), rfd2.getSize(), rfd2.getSHA1Urn()); }
/** * Constructs a new RemoteFileDesc exactly like the other one, but with a different push proxy * host. Will be handy when processing head pongs. */ public RemoteFileDesc(RemoteFileDesc rfd, PushEndpoint pe) { this( rfd.getHost(), // host - ignored rfd.getPort(), // port -ignored COPY_INDEX, // index (unknown) rfd.getFileName(), // filename rfd.getSize(), // filesize rfd.getSpeed(), // speed false, // chat capable rfd.getQuality(), // quality false, // browse hostable rfd.getUrns(), // urns false, // reply to MCast true, // is firewalled AlternateLocation.ALT_VENDOR, // vendor System.currentTimeMillis(), // timestamp rfd.getCreationTime(), // creation time pe); }
/** schedules a push ping to each proxy of the given host */ private void pingProxies(RemoteFileDesc rfd) { if (RouterService.acceptedIncomingConnection() || (RouterService.getUdpService().canDoFWT() && rfd.supportsFWTransfer())) { HeadPing pushPing = new HeadPing( myGUID, rfd.getSHA1Urn(), new GUID(rfd.getPushAddr().getClientGUID()), getPingFlags()); for (Iterator iter = rfd.getPushProxies().iterator(); iter.hasNext(); ) pingedHosts.put(iter.next(), rfd); if (LOG.isDebugEnabled()) LOG.debug("pinging push location " + rfd.getPushAddr()); pinger.rank(rfd.getPushProxies(), null, this, pushPing); } }
/** * Same as getFile(String, urn, int), except taking the values from the RFD. getFile(rfd) == * getFile(rfd.getFileName(), rfd.getSHA1Urn(), rfd.getSize()); */ public synchronized File getFile(RemoteFileDesc rfd) throws IOException { return getFile(rfd.getFileName(), rfd.getSHA1Urn(), rfd.getSize()); }
// problem is: // old code ranked as follows: GUID:C72D25808E87DE738FD57BCF51F5FF00, address: 1.1.1.1:6346, // proxies:{ /127.0.0.2:10002 // }] // // GUID:C72D25808E87DE738FD57BCF51F5FF00, address: 127.0.0.1:7498, // proxies:{ /1.2.3.4:5 // /6.7.8.9:10 // // new code picks: // GUID:7ECD430C3F5E93BB76B5CF3706C40700, address: 127.0.0.1:6346, // proxies:{ /127.0.0.2:10002 // // GUID:7ECD430C3F5E93BB76B5CF3706C40700, address: 127.0.0.1:7498, // proxies:{ /1.2.3.4:5 // /6.7.8.9:10 // // GUID:7ECD430C3F5E93BB76B5CF3706C40700, address: 127.0.0.1:7498, // proxies:{ /1.2.3.4:5 // /6.7.8.9:10 // /127.0.0.2:10002 // }] public void testPushLocUpdatesStatus() throws Exception { int successfulPushes = ((AtomicInteger) ((Map) statsTracker.inspect()).get("push connect success")).intValue(); LOG.info("testing that a push loc updates its status"); final int RATE = 100; final int FWTPort = 7498; NetworkManagerStub uploaderNetworkManager = new NetworkManagerStub(); uploaderNetworkManager.setAcceptedIncomingConnection(false); uploaderNetworkManager.setCanDoFWT(true); udpService.setReceiveSolicited(true); testUploaders[0].setRate(RATE); testUploaders[0].stopAfter(900000); testUploaders[0].setInterestedInFalts(true); TestUploader pusher2 = new TestUploader(uploaderNetworkManager); pusher2.start("firewalled pusher"); pusher2.setRate(RATE); pusher2.stopAfter(200000); pusher2.setFirewalled(true); pusher2.setProxiesString("1.2.3.4:5,6.7.8.9:10"); pusher2.setInterestedInFalts(true); pusher2.setFWTPort(FWTPort); GUID guid = new GUID(); // register proxies for GUID, this will add 127.0.0.2:10002 to proxies PushEndpoint cachedPE = pushEndpointFactory.createPushEndpoint( guid.bytes(), new IpPortSet(new IpPortImpl("127.0.0.2", PPORT_2))); PushEndpointCache cache = injector.getInstance(PushEndpointCache.class); GUID retGuid = cache.updateProxiesFor(guid, cachedPE, true); assertSame(retGuid, guid); assertEquals(1, cachedPE.getProxies().size()); RemoteFileDesc openRFD = newRFDWithURN(PORTS[0], false); RemoteFileDesc pushRFD2 = newRFDPush(guid, PPORT_2, 1, 2); PushEndpoint pushEndpoint = (PushEndpoint) pushRFD2.getAddress(); assertEquals(0, pushEndpoint.getFWTVersion()); testUDPAcceptorFactoryImpl.createTestUDPAcceptor( PPORT_2, networkManager.getPort(), savedFile.getName(), pusher2, guid, _currentTestName); // start download with rfd that needs udp push request ManagedDownloader download = (ManagedDownloader) downloadServices.download( new RemoteFileDesc[] {pushRFD2}, RemoteFileDesc.EMPTY_LIST, null, false); Thread.sleep(2000); LOG.debug("adding regular downloader"); // also download from uploader1, so it gets the proxy headers from pusher2 download.addDownload(openRFD, false); waitForComplete(); List<AlternateLocation> alc = testUploaders[0].getIncomingGoodAltLocs(); assertEquals(1, alc.size()); PushAltLoc pushLoc = (PushAltLoc) alc.iterator().next(); assertEquals(RUDPUtils.VERSION, pushLoc.supportsFWTVersion()); RemoteFileDesc readRFD = pushLoc.createRemoteFileDesc(1, remoteFileDescFactory); pushEndpoint = (PushEndpoint) readRFD.getAddress(); assertTrue(pushEndpoint.getFWTVersion() > 0); assertEquals(pushEndpoint.getPort(), FWTPort); Set<IpPort> expectedProxies = new IpPortSet(new IpPortImpl("1.2.3.4:5"), new IpPortImpl("6.7.8.9:10")); assertEquals( "expected: " + expectedProxies + ", actual: " + pushEndpoint.getProxies(), expectedProxies.size(), pushEndpoint.getProxies().size()); assertTrue(expectedProxies.containsAll(pushEndpoint.getProxies())); assertEquals( successfulPushes + 1, ((AtomicInteger) ((Map) statsTracker.inspect()).get("push connect success")).intValue()); }
@Override public String toString() { return remoteFileDesc.toString(); }
public boolean isFromAlternateLocation() { return remoteFileDesc.isFromAlternateLocation(); }
public Credentials getCredentials() { return remoteFileDesc.getCredentials(); }
public Address getAddress() { return remoteFileDesc.getAddress(); }
public boolean isReplyToMulticast() { return remoteFileDesc.isReplyToMulticast(); }
public URN getSHA1Urn() { return remoteFileDesc.getSHA1Urn(); }
public int compare(Object a, Object b) { RemoteFileDesc pongA = (RemoteFileDesc) a; RemoteFileDesc pongB = (RemoteFileDesc) b; // Multicasts are best if (pongA.isReplyToMulticast() != pongB.isReplyToMulticast()) { if (pongA.isReplyToMulticast()) return -1; else return 1; } // HeadPongs with highest number of free slots get the highest priority if (pongA.getQueueStatus() > pongB.getQueueStatus()) return 1; else if (pongA.getQueueStatus() < pongB.getQueueStatus()) return -1; // Within the same queue rank, firewalled hosts get priority if (pongA.needsPush() != pongB.needsPush()) { if (pongA.needsPush()) return -1; else return 1; } // Within the same queue/fwall, partial hosts get priority if (pongA.isPartialSource() != pongB.isPartialSource()) { if (pongA.isPartialSource()) return -1; else return 1; } // the two pongs seem completely the same return pongA.hashCode() - pongB.hashCode(); }