public void testSimplePushDownload() throws Exception {
int successfulPushes =
((AtomicInteger) ((Map) statsTracker.inspect()).get("push connect success")).intValue();
LOG.info("-Testing non-swarmed push download");
GUID guid = new GUID();
AlternateLocation pushLoc =
alternateLocationFactory.create(
guid.toHexString() + ";127.0.0.1:" + PPORT_1, TestFile.hash());
((PushAltLoc) pushLoc).updateProxies(true);
RemoteFileDesc rfd = newRFDPush(guid, PPORT_1, 1);
assertTrue(rfd.getAddress() instanceof PushEndpoint);
RemoteFileDesc[] rfds = {rfd};
TestUploader uploader = injector.getInstance(TestUploader.class);
uploader.start("push uploader");
testUDPAcceptorFactoryImpl.createTestUDPAcceptor(
PPORT_1, networkManager.getPort(), savedFile.getName(), uploader, guid, _currentTestName);
tGeneric(rfds);
assertEquals(
successfulPushes + 1,
((AtomicInteger) ((Map) statsTracker.inspect()).get("push connect success")).intValue());
}
/** Returns this Response as a RemoteFileDesc. */
public RemoteFileDesc toRemoteFileDesc(HostData data) {
if (cachedRFD != null
&& cachedRFD.getPort() == data.getPort()
&& cachedRFD.getHost().equals(data.getIP())) return cachedRFD;
else {
RemoteFileDesc rfd =
new RemoteFileDesc(
data.getIP(),
data.getPort(),
getIndex(),
getName(),
(int) getSize(),
data.getClientGUID(),
data.getSpeed(),
data.isChatEnabled(),
data.getQuality(),
data.isBrowseHostEnabled(),
getDocument(),
getUrns(),
data.isReplyToMulticastQuery(),
data.isFirewalled(),
data.getVendorCode(),
System.currentTimeMillis(),
data.getPushProxies(),
getCreateTime(),
data.getFWTVersionSupported());
cachedRFD = rfd;
return rfd;
}
}
public synchronized RemoteFileDesc getBest() throws NoSuchElementException {
if (!hasMore()) return null;
RemoteFileDesc ret;
// try a verified host
if (!verifiedHosts.isEmpty()) {
LOG.debug("getting a verified host");
ret = (RemoteFileDesc) verifiedHosts.first();
verifiedHosts.remove(ret);
} else {
LOG.debug("getting a non-verified host");
// use the legacy ranking logic to select a non-verified host
Iterator dual = new DualIterator(testedLocations.iterator(), newHosts.iterator());
ret = LegacyRanker.getBest(dual);
newHosts.remove(ret);
testedLocations.remove(ret);
if (ret.needsPush()) {
for (Iterator iter = ret.getPushProxies().iterator(); iter.hasNext(); )
pingedHosts.remove(iter.next());
} else pingedHosts.remove(ret);
}
pingNewHosts();
if (LOG.isDebugEnabled())
LOG.debug("the best host we came up with is " + ret + " " + ret.getPushAddr());
return ret;
}
private boolean addInternal(RemoteFileDesc host) {
// initialize the sha1 if we don't have one
if (sha1 == null) {
if (host.getSHA1Urn() != null) sha1 = host.getSHA1Urn();
else // BUGFIX: We can't discard sources w/out a SHA1 when we dont' have
// a SHA1 for the download, or else it won't be possible to download a
// file from a query hit without a SHA1, if we can received UDP pings
return testedLocations.add(host); // we can't do anything yet
}
// do not allow duplicate hosts
if (running && knowsAboutHost(host)) return false;
if (LOG.isDebugEnabled()) LOG.debug("adding new host " + host + " " + host.getPushAddr());
boolean ret = false;
// don't bother ranking multicasts
if (host.isReplyToMulticast()) ret = verifiedHosts.add(host);
else ret = newHosts.add(host); // rank
// make sure that if we were stopped, we return true
ret = ret | !running;
// initialize the guid if we don't have one
if (myGUID == null && meshHandler != null) {
myGUID = new GUID(GUID.makeGuid());
RouterService.getMessageRouter().registerMessageListener(myGUID.bytes(), this);
}
return ret;
}
/**
* tests that a download from a push location becomes an alternate location.
*
* <p>It creates a push uploader from which we must create a PushLoc. After a while, two open
* uploaders join the swarm -one which is interested in receiving push locs and one which isn't.
* The interested one should receive the push loc, the other one should not.
*/
public void testPusherBecomesPushLocAndSentToInterested() throws Exception {
LOG.info("-Testing push download creating a push location...");
final int RATE = 200;
testUploaders[0].setRate(RATE);
testUploaders[0].setInterestedInFalts(true);
testUploaders[0].stopAfter(600000);
testUploaders[1].setRate(RATE);
testUploaders[1].setInterestedInFalts(false);
testUploaders[1].stopAfter(300000);
TestUploader pusher = injector.getInstance(TestUploader.class);
pusher.start("push uploader");
pusher.setRate(RATE);
pusher.stopAfter(200000);
GUID guid = new GUID();
AlternateLocation pushLoc =
alternateLocationFactory.create(
guid.toHexString() + ";127.0.0.2:" + PPORT_1, TestFile.hash());
RemoteFileDesc pushRFD = newRFDPush(guid, PPORT_1, 1, 2);
PushEndpoint pushEndpoint = (PushEndpoint) pushRFD.getAddress();
assertEquals(0, pushEndpoint.getFWTVersion());
RemoteFileDesc openRFD1 = newRFDWithURN(PORTS[0], TestFile.hash().toString(), false);
RemoteFileDesc openRFD2 = newRFDWithURN(PORTS[1], TestFile.hash().toString(), false);
RemoteFileDesc[] now = {pushRFD};
HashSet<RemoteFileDesc> later = new HashSet<RemoteFileDesc>();
later.add(openRFD1);
later.add(openRFD2);
testUDPAcceptorFactoryImpl.createTestUDPAcceptor(
PPORT_1, networkManager.getPort(), savedFile.getName(), pusher, guid, _currentTestName);
ManagedDownloader download =
(ManagedDownloader) downloadServices.download(now, RemoteFileDesc.EMPTY_LIST, null, false);
Thread.sleep(1000);
download.addDownload(later, false);
waitForComplete();
assertGreaterThan("u1 did no work", 100000, testUploaders[0].getAmountUploaded());
assertGreaterThan("u2 did no work", 100000, testUploaders[1].getAmountUploaded());
assertLessThan("u2 did too much work", 550 * 1024, testUploaders[1].getAmountUploaded());
assertGreaterThan("pusher did no work", 100 * 1024, pusher.getAmountUploaded());
List alc = testUploaders[0].getIncomingGoodAltLocs();
assertTrue("interested uploader did not get pushloc", alc.contains(pushLoc));
alc = testUploaders[1].getIncomingGoodAltLocs();
assertFalse("not interested uploader got pushloc", alc.contains(pushLoc));
alc = pusher.getIncomingGoodAltLocs();
assertFalse("not interested uploader got pushloc", alc.contains(pushLoc));
}
/** Informs the Ranker that a host has replied with a HeadPing */
public void processMessage(Message m, ReplyHandler handler) {
MeshHandler mesh;
RemoteFileDesc rfd;
Collection alts = null;
// this -> meshHandler NOT ok
synchronized (this) {
if (!running) return;
if (!(m instanceof HeadPong)) return;
HeadPong pong = (HeadPong) m;
if (!pingedHosts.containsKey(handler)) return;
rfd = (RemoteFileDesc) pingedHosts.remove(handler);
testedLocations.remove(rfd);
if (LOG.isDebugEnabled()) {
LOG.debug(
"received a pong "
+ pong
+ " from "
+ handler
+ " for rfd "
+ rfd
+ " with PE "
+ rfd.getPushAddr());
}
// older push proxies do not route but respond directly, we want to get responses
// from other push proxies
if (!pong.hasFile() && !pong.isGGEPPong() && rfd.needsPush()) return;
// if the pong is firewalled, remove the other proxies from the
// pinged set
if (pong.isFirewalled()) {
for (Iterator iter = rfd.getPushProxies().iterator(); iter.hasNext(); )
pingedHosts.remove(iter.next());
}
mesh = meshHandler;
if (pong.hasFile()) {
// update the rfd with information from the pong
pong.updateRFD(rfd);
// if the remote host is busy, re-add him for later ranking
if (rfd.isBusy()) newHosts.add(rfd);
else verifiedHosts.add(rfd);
alts = pong.getAllLocsRFD(rfd);
}
}
// if the pong didn't have the file, drop it
// otherwise add any altlocs the pong had to our known hosts
if (alts == null) mesh.informMesh(rfd, false);
else mesh.addPossibleSources(alts);
}
public int compare(Object a, Object b) {
RemoteFileDesc rfd1 = (RemoteFileDesc) a;
RemoteFileDesc rfd2 = (RemoteFileDesc) b;
if (rfd1.isFromAlternateLocation() != rfd2.isFromAlternateLocation()) {
if (rfd1.isFromAlternateLocation()) return 1;
else return -1;
}
return 0;
}
/** pings a bunch of hosts if necessary */
private void pingNewHosts() {
// if we have reached our desired # of altlocs, don't ping
if (isCancelled()) return;
// if we don't have anybody to ping, don't ping
if (!hasNonBusy()) return;
// if we haven't found a single RFD with URN, don't ping anybody
if (sha1 == null) return;
// if its not time to ping yet, don't ping
// use the same interval as workers for now
long now = System.currentTimeMillis();
if (now - lastPingTime < DownloadSettings.WORKER_INTERVAL.getValue()) return;
// create a ping for the non-firewalled hosts
HeadPing ping = new HeadPing(myGUID, sha1, getPingFlags());
// prepare a batch of hosts to ping
int batch = DownloadSettings.PING_BATCH.getValue();
List toSend = new ArrayList(batch);
int sent = 0;
for (Iterator iter = newHosts.iterator(); iter.hasNext() && sent < batch; ) {
RemoteFileDesc rfd = (RemoteFileDesc) iter.next();
if (rfd.isBusy(now)) continue;
iter.remove();
if (rfd.needsPush()) {
if (rfd.getPushProxies().size() > 0 && rfd.getSHA1Urn() != null) pingProxies(rfd);
} else {
pingedHosts.put(rfd, rfd);
toSend.add(rfd);
}
testedLocations.add(rfd);
sent++;
}
if (LOG.isDebugEnabled()) {
LOG.debug(
"\nverified hosts "
+ verifiedHosts.size()
+ "\npingedHosts "
+ pingedHosts.values().size()
+ "\nnewHosts "
+ newHosts.size()
+ "\npinging hosts: "
+ sent);
}
pinger.rank(toSend, null, this, ping);
lastPingTime = now;
}
/**
* Creates a new LimeWire result.
*
* @param sha1 The hash of the file.
* @param rfd The base descriptor for the file.
* @param hd The host data.
* @param ipPorts The endpoints for the file.
* @throws URISyntaxException If there's an error creating any associated URIs.
*/
public LimeWireResult(
final URI sha1,
final RemoteFileDesc rfd,
final GUID queryGUID,
final Set<? extends IpPort> ipPorts)
throws URISyntaxException {
super(
rfd.getFileName(),
sha1,
createThumbnailUrl(rfd.getFileName()),
-1,
-1,
rfd.getSize(),
"limewire",
sha1,
null);
this.m_queryGUID = queryGUID;
this.m_alts.addAll(ipPorts);
this.m_rfds.add(rfd);
}
/**
* Constructs a new RemoteFileDesc exactly like the other one, but with a different remote host.
*
* <p>It is okay to use the same internal structures for URNs because the Set is immutable.
*/
public RemoteFileDesc(RemoteFileDesc rfd, IpPort ep) {
this(
ep.getAddress(), // host
ep.getPort(), // port
COPY_INDEX, // index (unknown)
rfd.getFileName(), // filename
rfd.getSize(), // filesize
DataUtils.EMPTY_GUID, // client GUID
0, // speed
false, // chat capable
2, // quality
false, // browse hostable
rfd.getUrns(), // urns
false, // reply to MCast
false, // is firewalled
AlternateLocation.ALT_VENDOR, // vendor
System.currentTimeMillis(), // timestamp
Collections.EMPTY_SET, // push proxies
rfd.getCreationTime(), // creation time
0); // firewalled transfer
}
/*
* Returns true if both rfd "have the same content". Currently
* rfd1~=rfd2 iff either of the following conditions hold:
*
* <ul>
* <li>Both files have the same hash, i.e.,
* rfd1.getSHA1Urn().equals(rfd2.getSHA1Urn(). Note that this (almost)
* always means that rfd1.getSize()==rfd2.getSize(), though rfd1 and
* rfd2 may have different names.
* <li>Both files have the same name and size and don't have conflicting
* hashes, i.e., rfd1.getName().equals(rfd2.getName()) &&
* rfd1.getSize()==rfd2.getSize() && (rfd1.getSHA1Urn()==null ||
* rfd2.getSHA1Urn()==null ||
* rfd1.getSHA1Urn().equals(rfd2.getSHA1Urn())).
* </ul>
* Note that the second condition allows risky resumes, i.e., resumes when
* one (or both) of the files doesn't have a hash.
*
* @see getFile
*/
static boolean same(RemoteFileDesc rfd1, RemoteFileDesc rfd2) {
return same(
rfd1.getFileName(),
rfd1.getSize(),
rfd1.getSHA1Urn(),
rfd2.getFileName(),
rfd2.getSize(),
rfd2.getSHA1Urn());
}
/**
* Constructs a new RemoteFileDesc exactly like the other one, but with a different push proxy
* host. Will be handy when processing head pongs.
*/
public RemoteFileDesc(RemoteFileDesc rfd, PushEndpoint pe) {
this(
rfd.getHost(), // host - ignored
rfd.getPort(), // port -ignored
COPY_INDEX, // index (unknown)
rfd.getFileName(), // filename
rfd.getSize(), // filesize
rfd.getSpeed(), // speed
false, // chat capable
rfd.getQuality(), // quality
false, // browse hostable
rfd.getUrns(), // urns
false, // reply to MCast
true, // is firewalled
AlternateLocation.ALT_VENDOR, // vendor
System.currentTimeMillis(), // timestamp
rfd.getCreationTime(), // creation time
pe);
}
/** schedules a push ping to each proxy of the given host */
private void pingProxies(RemoteFileDesc rfd) {
if (RouterService.acceptedIncomingConnection()
|| (RouterService.getUdpService().canDoFWT() && rfd.supportsFWTransfer())) {
HeadPing pushPing =
new HeadPing(
myGUID,
rfd.getSHA1Urn(),
new GUID(rfd.getPushAddr().getClientGUID()),
getPingFlags());
for (Iterator iter = rfd.getPushProxies().iterator(); iter.hasNext(); )
pingedHosts.put(iter.next(), rfd);
if (LOG.isDebugEnabled()) LOG.debug("pinging push location " + rfd.getPushAddr());
pinger.rank(rfd.getPushProxies(), null, this, pushPing);
}
}
/**
* Same as getFile(String, urn, int), except taking the values from the RFD. getFile(rfd) ==
* getFile(rfd.getFileName(), rfd.getSHA1Urn(), rfd.getSize());
*/
public synchronized File getFile(RemoteFileDesc rfd) throws IOException {
return getFile(rfd.getFileName(), rfd.getSHA1Urn(), rfd.getSize());
}
// problem is:
// old code ranked as follows: GUID:C72D25808E87DE738FD57BCF51F5FF00, address: 1.1.1.1:6346,
// proxies:{ /127.0.0.2:10002
// }]
//
// GUID:C72D25808E87DE738FD57BCF51F5FF00, address: 127.0.0.1:7498,
// proxies:{ /1.2.3.4:5
// /6.7.8.9:10
//
// new code picks:
// GUID:7ECD430C3F5E93BB76B5CF3706C40700, address: 127.0.0.1:6346,
// proxies:{ /127.0.0.2:10002
//
// GUID:7ECD430C3F5E93BB76B5CF3706C40700, address: 127.0.0.1:7498,
// proxies:{ /1.2.3.4:5
// /6.7.8.9:10
//
// GUID:7ECD430C3F5E93BB76B5CF3706C40700, address: 127.0.0.1:7498,
// proxies:{ /1.2.3.4:5
// /6.7.8.9:10
// /127.0.0.2:10002
// }]
public void testPushLocUpdatesStatus() throws Exception {
int successfulPushes =
((AtomicInteger) ((Map) statsTracker.inspect()).get("push connect success")).intValue();
LOG.info("testing that a push loc updates its status");
final int RATE = 100;
final int FWTPort = 7498;
NetworkManagerStub uploaderNetworkManager = new NetworkManagerStub();
uploaderNetworkManager.setAcceptedIncomingConnection(false);
uploaderNetworkManager.setCanDoFWT(true);
udpService.setReceiveSolicited(true);
testUploaders[0].setRate(RATE);
testUploaders[0].stopAfter(900000);
testUploaders[0].setInterestedInFalts(true);
TestUploader pusher2 = new TestUploader(uploaderNetworkManager);
pusher2.start("firewalled pusher");
pusher2.setRate(RATE);
pusher2.stopAfter(200000);
pusher2.setFirewalled(true);
pusher2.setProxiesString("1.2.3.4:5,6.7.8.9:10");
pusher2.setInterestedInFalts(true);
pusher2.setFWTPort(FWTPort);
GUID guid = new GUID();
// register proxies for GUID, this will add 127.0.0.2:10002 to proxies
PushEndpoint cachedPE =
pushEndpointFactory.createPushEndpoint(
guid.bytes(), new IpPortSet(new IpPortImpl("127.0.0.2", PPORT_2)));
PushEndpointCache cache = injector.getInstance(PushEndpointCache.class);
GUID retGuid = cache.updateProxiesFor(guid, cachedPE, true);
assertSame(retGuid, guid);
assertEquals(1, cachedPE.getProxies().size());
RemoteFileDesc openRFD = newRFDWithURN(PORTS[0], false);
RemoteFileDesc pushRFD2 = newRFDPush(guid, PPORT_2, 1, 2);
PushEndpoint pushEndpoint = (PushEndpoint) pushRFD2.getAddress();
assertEquals(0, pushEndpoint.getFWTVersion());
testUDPAcceptorFactoryImpl.createTestUDPAcceptor(
PPORT_2, networkManager.getPort(), savedFile.getName(), pusher2, guid, _currentTestName);
// start download with rfd that needs udp push request
ManagedDownloader download =
(ManagedDownloader)
downloadServices.download(
new RemoteFileDesc[] {pushRFD2}, RemoteFileDesc.EMPTY_LIST, null, false);
Thread.sleep(2000);
LOG.debug("adding regular downloader");
// also download from uploader1, so it gets the proxy headers from pusher2
download.addDownload(openRFD, false);
waitForComplete();
List<AlternateLocation> alc = testUploaders[0].getIncomingGoodAltLocs();
assertEquals(1, alc.size());
PushAltLoc pushLoc = (PushAltLoc) alc.iterator().next();
assertEquals(RUDPUtils.VERSION, pushLoc.supportsFWTVersion());
RemoteFileDesc readRFD = pushLoc.createRemoteFileDesc(1, remoteFileDescFactory);
pushEndpoint = (PushEndpoint) readRFD.getAddress();
assertTrue(pushEndpoint.getFWTVersion() > 0);
assertEquals(pushEndpoint.getPort(), FWTPort);
Set<IpPort> expectedProxies =
new IpPortSet(new IpPortImpl("1.2.3.4:5"), new IpPortImpl("6.7.8.9:10"));
assertEquals(
"expected: " + expectedProxies + ", actual: " + pushEndpoint.getProxies(),
expectedProxies.size(),
pushEndpoint.getProxies().size());
assertTrue(expectedProxies.containsAll(pushEndpoint.getProxies()));
assertEquals(
successfulPushes + 1,
((AtomicInteger) ((Map) statsTracker.inspect()).get("push connect success")).intValue());
}
@Override
public String toString() {
return remoteFileDesc.toString();
}
public boolean isFromAlternateLocation() {
return remoteFileDesc.isFromAlternateLocation();
}
public Credentials getCredentials() {
return remoteFileDesc.getCredentials();
}
public Address getAddress() {
return remoteFileDesc.getAddress();
}
public boolean isReplyToMulticast() {
return remoteFileDesc.isReplyToMulticast();
}
public URN getSHA1Urn() {
return remoteFileDesc.getSHA1Urn();
}
public int compare(Object a, Object b) {
RemoteFileDesc pongA = (RemoteFileDesc) a;
RemoteFileDesc pongB = (RemoteFileDesc) b;
// Multicasts are best
if (pongA.isReplyToMulticast() != pongB.isReplyToMulticast()) {
if (pongA.isReplyToMulticast()) return -1;
else return 1;
}
// HeadPongs with highest number of free slots get the highest priority
if (pongA.getQueueStatus() > pongB.getQueueStatus()) return 1;
else if (pongA.getQueueStatus() < pongB.getQueueStatus()) return -1;
// Within the same queue rank, firewalled hosts get priority
if (pongA.needsPush() != pongB.needsPush()) {
if (pongA.needsPush()) return -1;
else return 1;
}
// Within the same queue/fwall, partial hosts get priority
if (pongA.isPartialSource() != pongB.isPartialSource()) {
if (pongA.isPartialSource()) return -1;
else return 1;
}
// the two pongs seem completely the same
return pongA.hashCode() - pongB.hashCode();
}