/**
* A non-authoritative hint that a specific edition *might* exist. At the moment, we just fetch
* the block. We do not fetch the contents, and it is possible that USKFetcher's are also fetching
* the block. FIXME would it be more efficient to pass it along to a USKFetcher?
*
* @param context
* @throws MalformedURLException If the uri passed in is not a USK.
*/
public void hintUpdate(FreenetURI uri, ClientContext context, short priority)
throws MalformedURLException {
if (uri.getSuggestedEdition() < lookupLatestSlot(USK.create(uri))) {
if (logMINOR)
Logger.minor(
this,
"Ignoring hint because edition is "
+ uri.getSuggestedEdition()
+ " but latest is "
+ lookupLatestSlot(USK.create(uri)));
return;
}
uri = uri.sskForUSK();
if (logMINOR) Logger.minor(this, "Doing hint fetch for " + uri);
final ClientGetter get =
new ClientGetter(
new NullClientCallback(),
uri,
new FetchContext(backgroundFetchContext, FetchContext.IDENTICAL_MASK, false, null),
priority,
rcBulk,
new NullBucket(),
null,
null);
try {
get.start(null, context);
} catch (FetchException e) {
if (logMINOR) Logger.minor(this, "Cannot start hint fetch for " + uri + " : " + e, e);
// Ignore
}
}
void updateSlot(final USK origUSK, final long number, final ClientContext context) {
if (logMINOR) Logger.minor(this, "Updating (slot) " + origUSK.getURI() + " : " + number);
USK clear = origUSK.clearCopy();
final USKCallback[] callbacks;
synchronized (this) {
Long l = latestSlotByClearUSK.get(clear);
if (logMINOR) Logger.minor(this, "Old slot: " + l);
if ((l == null) || (number > l.longValue())) {
l = Long.valueOf(number);
latestSlotByClearUSK.put(clear, l);
if (logMINOR) Logger.minor(this, "Put " + number);
} else return;
callbacks = subscribersByClearUSK.get(clear);
if (temporaryBackgroundFetchersPrefetch.containsKey(clear)) {
temporaryBackgroundFetchersPrefetch.put(clear, System.currentTimeMillis());
schedulePrefetchChecker();
}
}
if (callbacks != null) {
// Run off-thread, because of locking, and because client callbacks may take some time
final USK usk = origUSK.copy(number);
for (final USKCallback callback : callbacks)
context.mainExecutor.execute(
new Runnable() {
@Override
public void run() {
callback.onFoundEdition(
number, usk, null, // non-persistent
context, false, (short) -1, null, false, false);
}
},
"USKManager callback executor for " + callback);
}
}
void updateKnownGood(final USK origUSK, final long number, final ClientContext context) {
if (logMINOR) Logger.minor(this, "Updating (known good) " + origUSK.getURI() + " : " + number);
USK clear = origUSK.clearCopy();
final USKCallback[] callbacks;
boolean newSlot = false;
synchronized (this) {
Long l = latestKnownGoodByClearUSK.get(clear);
if (logMINOR) Logger.minor(this, "Old known good: " + l);
if ((l == null) || (number > l.longValue())) {
l = Long.valueOf(number);
latestKnownGoodByClearUSK.put(clear, l);
if (logMINOR) Logger.minor(this, "Put " + number);
} else return; // If it's in KnownGood, it will also be in Slot
l = latestSlotByClearUSK.get(clear);
if (logMINOR) Logger.minor(this, "Old slot: " + l);
if ((l == null) || (number > l.longValue())) {
l = Long.valueOf(number);
latestSlotByClearUSK.put(clear, l);
if (logMINOR) Logger.minor(this, "Put " + number);
newSlot = true;
}
callbacks = subscribersByClearUSK.get(clear);
}
if (callbacks != null) {
// Run off-thread, because of locking, and because client callbacks may take some time
final USK usk = origUSK.copy(number);
final boolean newSlotToo = newSlot;
for (final USKCallback callback : callbacks)
context.mainExecutor.execute(
new Runnable() {
@Override
public void run() {
callback.onFoundEdition(
number,
usk,
null, // non-persistent
context,
false,
(short) -1,
null,
true,
newSlotToo);
}
},
"USKManager callback executor for " + callback);
}
}
public void checkUSK(
FreenetURI uri, boolean persistent, ObjectContainer container, boolean isMetadata) {
try {
if (persistent) container.activate(uri, 5);
FreenetURI uu;
if (uri.isSSK() && uri.isSSKForUSK()) {
uu = uri.setMetaString(null).uskForSSK();
} else if (uri.isUSK()) {
uu = uri;
} else {
return;
}
USK usk = USK.create(uu);
if (!isMetadata) context.uskManager.updateKnownGood(usk, uu.getSuggestedEdition(), context);
else
// We don't know whether the metadata is fetchable.
// FIXME add a callback so if the rest of the request completes we updateKnownGood().
context.uskManager.updateSlot(usk, uu.getSuggestedEdition(), context);
} catch (MalformedURLException e) {
Logger.error(this, "Caught " + e, e);
} catch (Throwable t) {
// Don't let the USK hint cause us to not succeed on the block.
Logger.error(this, "Caught " + t, t);
}
}
public void onFinished(USKFetcher fetcher, boolean ignoreError) {
USK orig = fetcher.getOriginalUSK();
USK clear = orig.clearCopy();
synchronized (this) {
if (backgroundFetchersByClearUSK.get(clear) == fetcher) {
backgroundFetchersByClearUSK.remove(clear);
if (!ignoreError) {
// This shouldn't happen, it's a sanity check: the only way we get cancelled is from
// USKManager, which removes us before calling cancel().
Logger.error(
this,
"onCancelled for " + fetcher + " - was still registered, how did this happen??",
new Exception("debug"));
}
}
if (temporaryBackgroundFetchersLRU.get(clear) == fetcher) {
temporaryBackgroundFetchersLRU.removeKey(clear);
temporaryBackgroundFetchersPrefetch.remove(clear);
}
}
}
/**
* A non-authoritative hint that a specific edition *might* exist. At the moment, we just fetch
* the block. We do not fetch the contents, and it is possible that USKFetcher's are also fetching
* the block. FIXME would it be more efficient to pass it along to a USKFetcher?
*
* @param usk
* @param edition
* @param context
*/
public void hintUpdate(USK usk, long edition, ClientContext context) {
if (edition < lookupLatestSlot(usk)) return;
FreenetURI uri = usk.copy(edition).getURI().sskForUSK();
final ClientGetter get =
new ClientGetter(
new NullClientCallback(),
uri,
new FetchContext(backgroundFetchContext, FetchContext.IDENTICAL_MASK, false, null),
RequestStarter.UPDATE_PRIORITY_CLASS,
rcBulk,
new NullBucket(),
null,
null);
try {
get.start(null, context);
} catch (FetchException e) {
// Ignore
}
}
public void unsubscribe(USK origUSK, USKCallback cb) {
USKFetcher toCancel = null;
synchronized (this) {
USK clear = origUSK.clearCopy();
USKCallback[] callbacks = subscribersByClearUSK.get(clear);
if (callbacks
== null) { // maybe we should throw something ? shall we allow multiple unsubscriptions ?
if (logMINOR) Logger.minor(this, "No longer subscribed");
return;
}
int j = 0;
for (USKCallback c : callbacks) {
if ((c != null) && (c != cb)) {
callbacks[j++] = c;
}
}
USKCallback[] newCallbacks = Arrays.copyOf(callbacks, j);
if (newCallbacks.length > 0) subscribersByClearUSK.put(clear, newCallbacks);
else {
subscribersByClearUSK.remove(clear);
}
USKFetcher f = backgroundFetchersByClearUSK.get(clear);
if (f != null) {
f.removeSubscriber(cb, context);
if (!f.hasSubscribers()) {
toCancel = f;
backgroundFetchersByClearUSK.remove(clear);
}
}
// Temporary background fetchers run once and then die.
// They do not care about callbacks.
}
if (toCancel != null) {
toCancel.cancel(null, context);
} else {
if (logMINOR) Logger.minor(this, "Not found unsubscribing: " + cb + " for " + origUSK);
}
}
/**
* Subscribe to a given USK. Callback will be notified when it is updated. Note that this does not
* imply that the USK will be checked on a regular basis, unless runBackgroundFetch=true.
*/
public void subscribe(
USK origUSK,
USKCallback cb,
boolean runBackgroundFetch,
boolean ignoreUSKDatehints,
RequestClient client) {
if (logMINOR) Logger.minor(this, "Subscribing to " + origUSK + " for " + cb);
if (client.persistent())
throw new UnsupportedOperationException("USKManager subscriptions cannot be persistent");
USKFetcher sched = null;
long ed = origUSK.suggestedEdition;
if (ed < 0) {
Logger.error(this, "Subscribing to USK with negative edition number: " + ed);
ed = -ed;
}
long curEd;
curEd = lookupLatestSlot(origUSK);
long goodEd;
goodEd = lookupKnownGood(origUSK);
synchronized (this) {
USK clear = origUSK.clearCopy();
USKCallback[] callbacks = subscribersByClearUSK.get(clear);
if (callbacks == null) {
callbacks = new USKCallback[] {cb};
} else {
boolean mustAdd = true;
for (USKCallback callback : callbacks) {
if (callback == cb) {
// Already subscribed.
// But it may still be waiting for the callback.
if (!(curEd > ed || goodEd > ed)) return;
mustAdd = false;
}
}
if (mustAdd) {
callbacks = Arrays.copyOf(callbacks, callbacks.length + 1);
callbacks[callbacks.length - 1] = cb;
}
}
subscribersByClearUSK.put(clear, callbacks);
if (runBackgroundFetch) {
USKFetcher f = backgroundFetchersByClearUSK.get(clear);
if (f == null) {
f =
new USKFetcher(
origUSK,
this,
ignoreUSKDatehints ? backgroundFetchContextIgnoreDBR : backgroundFetchContext,
new USKFetcherWrapper(origUSK, RequestStarter.UPDATE_PRIORITY_CLASS, client),
3,
true,
false,
false);
sched = f;
backgroundFetchersByClearUSK.put(clear, f);
}
f.addSubscriber(cb, origUSK.suggestedEdition);
}
}
if (goodEd > ed)
cb.onFoundEdition(
goodEd, origUSK.copy(curEd), null, context, false, (short) -1, null, true, curEd > ed);
else if (curEd > ed)
cb.onFoundEdition(
curEd, origUSK.copy(curEd), null, context, false, (short) -1, null, false, false);
final USKFetcher fetcher = sched;
if (fetcher != null) {
executor.execute(
new Runnable() {
@Override
public void run() {
if (logMINOR) Logger.minor(this, "Starting " + fetcher);
fetcher.schedule(null, context);
}
},
"USKManager.schedule for " + fetcher);
}
}
@Override
public void run() {
if (logDEBUG) Logger.debug(this, "Running prefetch checker...");
ArrayList<USK> toFetch = null;
long now = System.currentTimeMillis();
boolean empty = true;
synchronized (USKManager.this) {
for (Map.Entry<USK, Long> entry : temporaryBackgroundFetchersPrefetch.entrySet()) {
empty = false;
if (entry.getValue() > 0 && now - entry.getValue() >= PREFETCH_DELAY) {
if (toFetch == null) toFetch = new ArrayList<USK>();
USK clear = entry.getKey();
long l = lookupLatestSlot(clear);
if (lookupKnownGood(clear) < l) toFetch.add(clear.copy(l));
entry.setValue(-1L); // Reset counter until new data comes in
} else {
if (logMINOR)
Logger.minor(
this, "Not prefetching: " + entry.getKey() + " : " + entry.getValue());
}
}
}
if (toFetch == null) return;
for (final USK key : toFetch) {
final long l = key.suggestedEdition;
if (logMINOR)
Logger.minor(
this, "Prefetching content for background fetch for edition " + l + " on " + key);
FetchContext fctx =
new FetchContext(realFetchContext, FetchContext.IDENTICAL_MASK, false, null);
final ClientGetter get =
new ClientGetter(
new ClientGetCallback() {
@Override
public void onFailure(
FetchException e, ClientGetter state, ObjectContainer container) {
if (e.newURI != null) {
if (logMINOR)
Logger.minor(this, "Prefetch succeeded with redirect for " + key);
updateKnownGood(key, l, context);
return;
} else {
if (logMINOR)
Logger.minor(this, "Prefetch failed later: " + e + " for " + key, e);
// Ignore
}
}
@Override
public void onSuccess(
FetchResult result, ClientGetter state, ObjectContainer container) {
if (logMINOR) Logger.minor(this, "Prefetch succeeded for " + key);
result.asBucket().free();
updateKnownGood(key, l, context);
}
@Override
public void onMajorProgress(ObjectContainer container) {
// Ignore
}
},
key.getURI().sskForUSK() /* FIXME add getSSKURI() */,
fctx,
RequestStarter.UPDATE_PRIORITY_CLASS,
rcBulk,
new NullBucket(),
null,
null);
try {
get.start(null, context);
} catch (FetchException e) {
if (logMINOR) Logger.minor(this, "Prefetch failed: " + e, e);
// Ignore
}
}
if (!empty) schedulePrefetchChecker();
}
public void startTemporaryBackgroundFetcher(
USK usk,
ClientContext context,
final FetchContext fctx,
boolean prefetchContent,
boolean realTimeFlag) {
final USK clear = usk.clearCopy();
USKFetcher sched = null;
ArrayList<USKFetcher> toCancel = null;
synchronized (this) {
// int x = 0;
// for(USK key: backgroundFetchersByClearUSK.keySet()) {
// System.err.println("Fetcher "+x+": "+key);
// x++;
// }
USKFetcher f = temporaryBackgroundFetchersLRU.get(clear);
if (f == null) {
f =
new USKFetcher(
usk,
this,
fctx.ignoreUSKDatehints ? backgroundFetchContextIgnoreDBR : backgroundFetchContext,
new USKFetcherWrapper(
usk, RequestStarter.UPDATE_PRIORITY_CLASS, realTimeFlag ? rcRT : rcBulk),
3,
false,
false,
false);
sched = f;
temporaryBackgroundFetchersLRU.push(clear, f);
} else {
f.addHintEdition(usk.suggestedEdition);
}
if (prefetchContent) {
long fetchTime = -1;
// If nothing in 60 seconds, try fetching the last known slot.
long slot = lookupLatestSlot(clear);
long good = lookupKnownGood(clear);
if (slot > -1 && good != slot) fetchTime = System.currentTimeMillis();
temporaryBackgroundFetchersPrefetch.put(clear, fetchTime);
if (logMINOR) Logger.minor(this, "Prefetch: set " + fetchTime + " for " + clear);
schedulePrefetchChecker();
}
temporaryBackgroundFetchersLRU.push(clear, f);
while (temporaryBackgroundFetchersLRU.size() > NodeClientCore.getMaxBackgroundUSKFetchers()) {
USKFetcher fetcher = temporaryBackgroundFetchersLRU.popValue();
temporaryBackgroundFetchersPrefetch.remove(fetcher.getOriginalUSK().clearCopy());
if (!fetcher.hasSubscribers()) {
if (toCancel == null) toCancel = new ArrayList<USKFetcher>(2);
toCancel.add(fetcher);
} else {
if (logMINOR)
Logger.minor(
this,
"Allowing temporary background fetcher to continue as it has subscribers... "
+ fetcher);
}
}
}
final ArrayList<USKFetcher> cancelled = toCancel;
final USKFetcher scheduleMe = sched;
// This is just a prefetching method. so it should not unnecessarily delay the parent, nor
// should it take important locks.
// So we should do the actual schedule/cancels off-thread.
// However, the above is done on-thread because a lot of the time it will already be running.
if (cancelled != null || sched != null) {
executor.execute(
new Runnable() {
@Override
public void run() {
if (cancelled != null) {
for (int i = 0; i < cancelled.size(); i++) {
USKFetcher fetcher = cancelled.get(i);
fetcher.cancel(null, USKManager.this.context);
}
}
if (scheduleMe != null) scheduleMe.schedule(null, USKManager.this.context);
}
});
}
}
/**
* Look up the latest SSK slot, whether the data it links to has been successfully fetched or not,
* of the given USK.
*
* @return The latest known edition number, or -1.
*/
public synchronized long lookupLatestSlot(USK usk) {
Long l = latestSlotByClearUSK.get(usk.clearCopy());
if (l != null) return l.longValue();
else return -1;
}