Example #1
0
 /**
  * A non-authoritative hint that a specific edition *might* exist. At the moment, we just fetch
  * the block. We do not fetch the contents, and it is possible that USKFetcher's are also fetching
  * the block. FIXME would it be more efficient to pass it along to a USKFetcher?
  *
  * @param context
  * @throws MalformedURLException If the uri passed in is not a USK.
  */
 public void hintUpdate(FreenetURI uri, ClientContext context, short priority)
     throws MalformedURLException {
   if (uri.getSuggestedEdition() < lookupLatestSlot(USK.create(uri))) {
     if (logMINOR)
       Logger.minor(
           this,
           "Ignoring hint because edition is "
               + uri.getSuggestedEdition()
               + " but latest is "
               + lookupLatestSlot(USK.create(uri)));
     return;
   }
   uri = uri.sskForUSK();
   if (logMINOR) Logger.minor(this, "Doing hint fetch for " + uri);
   final ClientGetter get =
       new ClientGetter(
           new NullClientCallback(),
           uri,
           new FetchContext(backgroundFetchContext, FetchContext.IDENTICAL_MASK, false, null),
           priority,
           rcBulk,
           new NullBucket(),
           null,
           null);
   try {
     get.start(null, context);
   } catch (FetchException e) {
     if (logMINOR) Logger.minor(this, "Cannot start hint fetch for " + uri + " : " + e, e);
     // Ignore
   }
 }
Example #2
0
  void updateSlot(final USK origUSK, final long number, final ClientContext context) {
    if (logMINOR) Logger.minor(this, "Updating (slot) " + origUSK.getURI() + " : " + number);
    USK clear = origUSK.clearCopy();
    final USKCallback[] callbacks;
    synchronized (this) {
      Long l = latestSlotByClearUSK.get(clear);
      if (logMINOR) Logger.minor(this, "Old slot: " + l);
      if ((l == null) || (number > l.longValue())) {
        l = Long.valueOf(number);
        latestSlotByClearUSK.put(clear, l);
        if (logMINOR) Logger.minor(this, "Put " + number);
      } else return;

      callbacks = subscribersByClearUSK.get(clear);
      if (temporaryBackgroundFetchersPrefetch.containsKey(clear)) {
        temporaryBackgroundFetchersPrefetch.put(clear, System.currentTimeMillis());
        schedulePrefetchChecker();
      }
    }
    if (callbacks != null) {
      // Run off-thread, because of locking, and because client callbacks may take some time
      final USK usk = origUSK.copy(number);
      for (final USKCallback callback : callbacks)
        context.mainExecutor.execute(
            new Runnable() {
              @Override
              public void run() {
                callback.onFoundEdition(
                    number, usk, null, // non-persistent
                    context, false, (short) -1, null, false, false);
              }
            },
            "USKManager callback executor for " + callback);
    }
  }
Example #3
0
  void updateKnownGood(final USK origUSK, final long number, final ClientContext context) {
    if (logMINOR) Logger.minor(this, "Updating (known good) " + origUSK.getURI() + " : " + number);
    USK clear = origUSK.clearCopy();
    final USKCallback[] callbacks;
    boolean newSlot = false;
    synchronized (this) {
      Long l = latestKnownGoodByClearUSK.get(clear);
      if (logMINOR) Logger.minor(this, "Old known good: " + l);
      if ((l == null) || (number > l.longValue())) {
        l = Long.valueOf(number);
        latestKnownGoodByClearUSK.put(clear, l);
        if (logMINOR) Logger.minor(this, "Put " + number);
      } else return; // If it's in KnownGood, it will also be in Slot

      l = latestSlotByClearUSK.get(clear);
      if (logMINOR) Logger.minor(this, "Old slot: " + l);
      if ((l == null) || (number > l.longValue())) {
        l = Long.valueOf(number);
        latestSlotByClearUSK.put(clear, l);
        if (logMINOR) Logger.minor(this, "Put " + number);
        newSlot = true;
      }

      callbacks = subscribersByClearUSK.get(clear);
    }
    if (callbacks != null) {
      // Run off-thread, because of locking, and because client callbacks may take some time
      final USK usk = origUSK.copy(number);
      final boolean newSlotToo = newSlot;
      for (final USKCallback callback : callbacks)
        context.mainExecutor.execute(
            new Runnable() {
              @Override
              public void run() {
                callback.onFoundEdition(
                    number,
                    usk,
                    null, // non-persistent
                    context,
                    false,
                    (short) -1,
                    null,
                    true,
                    newSlotToo);
              }
            },
            "USKManager callback executor for " + callback);
    }
  }
Example #4
0
 public void checkUSK(
     FreenetURI uri, boolean persistent, ObjectContainer container, boolean isMetadata) {
   try {
     if (persistent) container.activate(uri, 5);
     FreenetURI uu;
     if (uri.isSSK() && uri.isSSKForUSK()) {
       uu = uri.setMetaString(null).uskForSSK();
     } else if (uri.isUSK()) {
       uu = uri;
     } else {
       return;
     }
     USK usk = USK.create(uu);
     if (!isMetadata) context.uskManager.updateKnownGood(usk, uu.getSuggestedEdition(), context);
     else
       // We don't know whether the metadata is fetchable.
       // FIXME add a callback so if the rest of the request completes we updateKnownGood().
       context.uskManager.updateSlot(usk, uu.getSuggestedEdition(), context);
   } catch (MalformedURLException e) {
     Logger.error(this, "Caught " + e, e);
   } catch (Throwable t) {
     // Don't let the USK hint cause us to not succeed on the block.
     Logger.error(this, "Caught " + t, t);
   }
 }
Example #5
0
 public void onFinished(USKFetcher fetcher, boolean ignoreError) {
   USK orig = fetcher.getOriginalUSK();
   USK clear = orig.clearCopy();
   synchronized (this) {
     if (backgroundFetchersByClearUSK.get(clear) == fetcher) {
       backgroundFetchersByClearUSK.remove(clear);
       if (!ignoreError) {
         // This shouldn't happen, it's a sanity check: the only way we get cancelled is from
         // USKManager, which removes us before calling cancel().
         Logger.error(
             this,
             "onCancelled for " + fetcher + " - was still registered, how did this happen??",
             new Exception("debug"));
       }
     }
     if (temporaryBackgroundFetchersLRU.get(clear) == fetcher) {
       temporaryBackgroundFetchersLRU.removeKey(clear);
       temporaryBackgroundFetchersPrefetch.remove(clear);
     }
   }
 }
Example #6
0
 /**
  * A non-authoritative hint that a specific edition *might* exist. At the moment, we just fetch
  * the block. We do not fetch the contents, and it is possible that USKFetcher's are also fetching
  * the block. FIXME would it be more efficient to pass it along to a USKFetcher?
  *
  * @param usk
  * @param edition
  * @param context
  */
 public void hintUpdate(USK usk, long edition, ClientContext context) {
   if (edition < lookupLatestSlot(usk)) return;
   FreenetURI uri = usk.copy(edition).getURI().sskForUSK();
   final ClientGetter get =
       new ClientGetter(
           new NullClientCallback(),
           uri,
           new FetchContext(backgroundFetchContext, FetchContext.IDENTICAL_MASK, false, null),
           RequestStarter.UPDATE_PRIORITY_CLASS,
           rcBulk,
           new NullBucket(),
           null,
           null);
   try {
     get.start(null, context);
   } catch (FetchException e) {
     // Ignore
   }
 }
Example #7
0
 public void unsubscribe(USK origUSK, USKCallback cb) {
   USKFetcher toCancel = null;
   synchronized (this) {
     USK clear = origUSK.clearCopy();
     USKCallback[] callbacks = subscribersByClearUSK.get(clear);
     if (callbacks
         == null) { // maybe we should throw something ? shall we allow multiple unsubscriptions ?
       if (logMINOR) Logger.minor(this, "No longer subscribed");
       return;
     }
     int j = 0;
     for (USKCallback c : callbacks) {
       if ((c != null) && (c != cb)) {
         callbacks[j++] = c;
       }
     }
     USKCallback[] newCallbacks = Arrays.copyOf(callbacks, j);
     if (newCallbacks.length > 0) subscribersByClearUSK.put(clear, newCallbacks);
     else {
       subscribersByClearUSK.remove(clear);
     }
     USKFetcher f = backgroundFetchersByClearUSK.get(clear);
     if (f != null) {
       f.removeSubscriber(cb, context);
       if (!f.hasSubscribers()) {
         toCancel = f;
         backgroundFetchersByClearUSK.remove(clear);
       }
     }
     // Temporary background fetchers run once and then die.
     // They do not care about callbacks.
   }
   if (toCancel != null) {
     toCancel.cancel(null, context);
   } else {
     if (logMINOR) Logger.minor(this, "Not found unsubscribing: " + cb + " for " + origUSK);
   }
 }
Example #8
0
 /**
  * Subscribe to a given USK. Callback will be notified when it is updated. Note that this does not
  * imply that the USK will be checked on a regular basis, unless runBackgroundFetch=true.
  */
 public void subscribe(
     USK origUSK,
     USKCallback cb,
     boolean runBackgroundFetch,
     boolean ignoreUSKDatehints,
     RequestClient client) {
   if (logMINOR) Logger.minor(this, "Subscribing to " + origUSK + " for " + cb);
   if (client.persistent())
     throw new UnsupportedOperationException("USKManager subscriptions cannot be persistent");
   USKFetcher sched = null;
   long ed = origUSK.suggestedEdition;
   if (ed < 0) {
     Logger.error(this, "Subscribing to USK with negative edition number: " + ed);
     ed = -ed;
   }
   long curEd;
   curEd = lookupLatestSlot(origUSK);
   long goodEd;
   goodEd = lookupKnownGood(origUSK);
   synchronized (this) {
     USK clear = origUSK.clearCopy();
     USKCallback[] callbacks = subscribersByClearUSK.get(clear);
     if (callbacks == null) {
       callbacks = new USKCallback[] {cb};
     } else {
       boolean mustAdd = true;
       for (USKCallback callback : callbacks) {
         if (callback == cb) {
           // Already subscribed.
           // But it may still be waiting for the callback.
           if (!(curEd > ed || goodEd > ed)) return;
           mustAdd = false;
         }
       }
       if (mustAdd) {
         callbacks = Arrays.copyOf(callbacks, callbacks.length + 1);
         callbacks[callbacks.length - 1] = cb;
       }
     }
     subscribersByClearUSK.put(clear, callbacks);
     if (runBackgroundFetch) {
       USKFetcher f = backgroundFetchersByClearUSK.get(clear);
       if (f == null) {
         f =
             new USKFetcher(
                 origUSK,
                 this,
                 ignoreUSKDatehints ? backgroundFetchContextIgnoreDBR : backgroundFetchContext,
                 new USKFetcherWrapper(origUSK, RequestStarter.UPDATE_PRIORITY_CLASS, client),
                 3,
                 true,
                 false,
                 false);
         sched = f;
         backgroundFetchersByClearUSK.put(clear, f);
       }
       f.addSubscriber(cb, origUSK.suggestedEdition);
     }
   }
   if (goodEd > ed)
     cb.onFoundEdition(
         goodEd, origUSK.copy(curEd), null, context, false, (short) -1, null, true, curEd > ed);
   else if (curEd > ed)
     cb.onFoundEdition(
         curEd, origUSK.copy(curEd), null, context, false, (short) -1, null, false, false);
   final USKFetcher fetcher = sched;
   if (fetcher != null) {
     executor.execute(
         new Runnable() {
           @Override
           public void run() {
             if (logMINOR) Logger.minor(this, "Starting " + fetcher);
             fetcher.schedule(null, context);
           }
         },
         "USKManager.schedule for " + fetcher);
   }
 }
Example #9
0
        @Override
        public void run() {
          if (logDEBUG) Logger.debug(this, "Running prefetch checker...");
          ArrayList<USK> toFetch = null;
          long now = System.currentTimeMillis();
          boolean empty = true;
          synchronized (USKManager.this) {
            for (Map.Entry<USK, Long> entry : temporaryBackgroundFetchersPrefetch.entrySet()) {
              empty = false;
              if (entry.getValue() > 0 && now - entry.getValue() >= PREFETCH_DELAY) {
                if (toFetch == null) toFetch = new ArrayList<USK>();
                USK clear = entry.getKey();
                long l = lookupLatestSlot(clear);
                if (lookupKnownGood(clear) < l) toFetch.add(clear.copy(l));
                entry.setValue(-1L); // Reset counter until new data comes in
              } else {
                if (logMINOR)
                  Logger.minor(
                      this, "Not prefetching: " + entry.getKey() + " : " + entry.getValue());
              }
            }
          }
          if (toFetch == null) return;
          for (final USK key : toFetch) {
            final long l = key.suggestedEdition;
            if (logMINOR)
              Logger.minor(
                  this, "Prefetching content for background fetch for edition " + l + " on " + key);
            FetchContext fctx =
                new FetchContext(realFetchContext, FetchContext.IDENTICAL_MASK, false, null);
            final ClientGetter get =
                new ClientGetter(
                    new ClientGetCallback() {

                      @Override
                      public void onFailure(
                          FetchException e, ClientGetter state, ObjectContainer container) {
                        if (e.newURI != null) {
                          if (logMINOR)
                            Logger.minor(this, "Prefetch succeeded with redirect for " + key);
                          updateKnownGood(key, l, context);
                          return;
                        } else {
                          if (logMINOR)
                            Logger.minor(this, "Prefetch failed later: " + e + " for " + key, e);
                          // Ignore
                        }
                      }

                      @Override
                      public void onSuccess(
                          FetchResult result, ClientGetter state, ObjectContainer container) {
                        if (logMINOR) Logger.minor(this, "Prefetch succeeded for " + key);
                        result.asBucket().free();
                        updateKnownGood(key, l, context);
                      }

                      @Override
                      public void onMajorProgress(ObjectContainer container) {
                        // Ignore
                      }
                    },
                    key.getURI().sskForUSK() /* FIXME add getSSKURI() */,
                    fctx,
                    RequestStarter.UPDATE_PRIORITY_CLASS,
                    rcBulk,
                    new NullBucket(),
                    null,
                    null);
            try {
              get.start(null, context);
            } catch (FetchException e) {
              if (logMINOR) Logger.minor(this, "Prefetch failed: " + e, e);
              // Ignore
            }
          }
          if (!empty) schedulePrefetchChecker();
        }
Example #10
0
  public void startTemporaryBackgroundFetcher(
      USK usk,
      ClientContext context,
      final FetchContext fctx,
      boolean prefetchContent,
      boolean realTimeFlag) {
    final USK clear = usk.clearCopy();
    USKFetcher sched = null;
    ArrayList<USKFetcher> toCancel = null;
    synchronized (this) {
      //			int x = 0;
      //			for(USK key: backgroundFetchersByClearUSK.keySet()) {
      //				System.err.println("Fetcher "+x+": "+key);
      //				x++;
      //			}
      USKFetcher f = temporaryBackgroundFetchersLRU.get(clear);
      if (f == null) {
        f =
            new USKFetcher(
                usk,
                this,
                fctx.ignoreUSKDatehints ? backgroundFetchContextIgnoreDBR : backgroundFetchContext,
                new USKFetcherWrapper(
                    usk, RequestStarter.UPDATE_PRIORITY_CLASS, realTimeFlag ? rcRT : rcBulk),
                3,
                false,
                false,
                false);
        sched = f;
        temporaryBackgroundFetchersLRU.push(clear, f);
      } else {
        f.addHintEdition(usk.suggestedEdition);
      }
      if (prefetchContent) {
        long fetchTime = -1;
        // If nothing in 60 seconds, try fetching the last known slot.
        long slot = lookupLatestSlot(clear);
        long good = lookupKnownGood(clear);
        if (slot > -1 && good != slot) fetchTime = System.currentTimeMillis();
        temporaryBackgroundFetchersPrefetch.put(clear, fetchTime);
        if (logMINOR) Logger.minor(this, "Prefetch: set " + fetchTime + " for " + clear);
        schedulePrefetchChecker();
      }
      temporaryBackgroundFetchersLRU.push(clear, f);
      while (temporaryBackgroundFetchersLRU.size() > NodeClientCore.getMaxBackgroundUSKFetchers()) {
        USKFetcher fetcher = temporaryBackgroundFetchersLRU.popValue();
        temporaryBackgroundFetchersPrefetch.remove(fetcher.getOriginalUSK().clearCopy());
        if (!fetcher.hasSubscribers()) {
          if (toCancel == null) toCancel = new ArrayList<USKFetcher>(2);
          toCancel.add(fetcher);
        } else {
          if (logMINOR)
            Logger.minor(
                this,
                "Allowing temporary background fetcher to continue as it has subscribers... "
                    + fetcher);
        }
      }
    }
    final ArrayList<USKFetcher> cancelled = toCancel;
    final USKFetcher scheduleMe = sched;
    // This is just a prefetching method. so it should not unnecessarily delay the parent, nor
    // should it take important locks.
    // So we should do the actual schedule/cancels off-thread.
    // However, the above is done on-thread because a lot of the time it will already be running.
    if (cancelled != null || sched != null) {
      executor.execute(
          new Runnable() {

            @Override
            public void run() {
              if (cancelled != null) {
                for (int i = 0; i < cancelled.size(); i++) {
                  USKFetcher fetcher = cancelled.get(i);
                  fetcher.cancel(null, USKManager.this.context);
                }
              }
              if (scheduleMe != null) scheduleMe.schedule(null, USKManager.this.context);
            }
          });
    }
  }
Example #11
0
 /**
  * Look up the latest SSK slot, whether the data it links to has been successfully fetched or not,
  * of the given USK.
  *
  * @return The latest known edition number, or -1.
  */
 public synchronized long lookupLatestSlot(USK usk) {
   Long l = latestSlotByClearUSK.get(usk.clearCopy());
   if (l != null) return l.longValue();
   else return -1;
 }