public MyRequestThrottle(
     int rtt, String string, SimpleFieldSet fs, int size, boolean realTime) {
   roundTripTime =
       new BootstrappingDecayingRunningAverage(
           rtt, 10, MINUTES.toMillis(5), 10, fs == null ? null : fs.subset("RoundTripTime"));
   this.size = size;
   this.realTime = realTime;
 }
Ejemplo n.º 2
0
  /**
   * Create a ClientGet from a request serialized to a SimpleFieldSet. Can throw, and does minimal
   * verification, as is dealing with data supposedly serialized out by the node.
   *
   * @throws IOException
   * @throws FetchException
   */
  public ClientGet(SimpleFieldSet fs, FCPClient client2, FCPServer server)
      throws IOException, FetchException {
    super(fs, client2);

    returnType = ClientGetMessage.parseValidReturnType(fs.get("ReturnType"));
    String f = fs.get("Filename");
    if (f != null) targetFile = new File(f);
    else targetFile = null;
    f = fs.get("TempFilename");
    if (f != null) tempFile = new File(f);
    else tempFile = null;
    boolean ignoreDS = Fields.stringToBool(fs.get("IgnoreDS"), false);
    boolean dsOnly = Fields.stringToBool(fs.get("DSOnly"), false);
    int maxRetries = Integer.parseInt(fs.get("MaxRetries"));
    fctx = new FetchContext(server.defaultFetchContext, FetchContext.IDENTICAL_MASK, false, null);
    fctx.eventProducer.addEventListener(this);
    // ignoreDS
    fctx.localRequestOnly = dsOnly;
    fctx.ignoreStore = ignoreDS;
    fctx.maxNonSplitfileRetries = maxRetries;
    fctx.maxSplitfileBlockRetries = maxRetries;
    binaryBlob = Fields.stringToBool(fs.get("BinaryBlob"), false);
    succeeded = Fields.stringToBool(fs.get("Succeeded"), false);
    if (finished) {
      if (succeeded) {
        foundDataLength = Long.parseLong(fs.get("FoundDataLength"));
        foundDataMimeType = fs.get("FoundDataMimeType");
        SimpleFieldSet fs1 = fs.subset("PostFetchProtocolError");
        if (fs1 != null) postFetchProtocolErrorMessage = new ProtocolErrorMessage(fs1);
      } else {
        getFailedMessage = new GetFailedMessage(fs.subset("GetFailed"), false);
      }
    }
    Bucket ret = null;
    if (returnType == ClientGetMessage.RETURN_TYPE_DISK) {
      if (succeeded) {
        ret = new FileBucket(targetFile, false, true, false, false, false);
      } else {
        ret = new FileBucket(tempFile, false, true, false, false, false);
      }
    } else if (returnType == ClientGetMessage.RETURN_TYPE_NONE) {
      ret = new NullBucket();
    } else if (returnType == ClientGetMessage.RETURN_TYPE_DIRECT) {
      try {
        ret =
            SerializableToFieldSetBucketUtil.create(
                fs.subset("ReturnBucket"),
                server.core.random,
                server.core.persistentTempBucketFactory);
        if (ret == null) throw new CannotCreateFromFieldSetException("ret == null");
      } catch (CannotCreateFromFieldSetException e) {
        Logger.error(this, "Cannot read: " + this + " : " + e, e);
        try {
          // Create a new temp bucket
          if (persistenceType == PERSIST_FOREVER)
            ret = server.core.persistentTempBucketFactory.makeBucket(fctx.maxOutputLength);
          else ret = server.core.tempBucketFactory.makeBucket(fctx.maxOutputLength);
        } catch (IOException e1) {
          Logger.error(this, "Cannot create bucket for temp storage: " + e, e);
          getter = null;
          returnBucket = null;
          throw new FetchException(FetchException.BUCKET_ERROR, e);
        }
      }
    } else {
      throw new IllegalArgumentException();
    }
    if (succeeded) {
      if (foundDataLength < ret.size()) {
        Logger.error(this, "Failing " + identifier + " because lost data");
        succeeded = false;
      }
    }
    if (ret == null)
      Logger.error(
          this, "Impossible: ret = null in SFS constructor for " + this, new Exception("debug"));
    returnBucket = ret;

    String[] allowed = fs.getAll("AllowedMIMETypes");
    if (allowed != null) {
      fctx.allowedMIMETypes = new HashSet<String>();
      for (String a : allowed) fctx.allowedMIMETypes.add(a);
    }

    getter =
        new ClientGetter(
            this,
            uri,
            fctx,
            priorityClass,
            lowLevelClient,
            binaryBlob ? new NullBucket() : returnBucket,
            binaryBlob ? returnBucket : null);

    if (finished && succeeded)
      allDataPending =
          new AllDataMessage(
              returnBucket,
              identifier,
              global,
              startupTime,
              completionTime,
              this.foundDataMimeType);
  }
  RequestStarterGroup(
      Node node,
      NodeClientCore core,
      int portNumber,
      RandomSource random,
      Config config,
      SimpleFieldSet fs,
      ClientContext ctx,
      long dbHandle)
      throws InvalidConfigValueException {
    SubConfig schedulerConfig = config.createSubConfig("node.scheduler");
    this.stats = core.nodeStats;

    throttleWindowBulk =
        new ThrottleWindowManager(2.0, fs == null ? null : fs.subset("ThrottleWindow"), node);
    throttleWindowRT =
        new ThrottleWindowManager(2.0, fs == null ? null : fs.subset("ThrottleWindowRT"), node);

    throttleWindowCHK =
        new ThrottleWindowManager(2.0, fs == null ? null : fs.subset("ThrottleWindowCHK"), node);
    throttleWindowSSK =
        new ThrottleWindowManager(2.0, fs == null ? null : fs.subset("ThrottleWindowSSK"), node);
    throttleWindowInsert =
        new ThrottleWindowManager(2.0, fs == null ? null : fs.subset("ThrottleWindowInsert"), node);
    throttleWindowRequest =
        new ThrottleWindowManager(
            2.0, fs == null ? null : fs.subset("ThrottleWindowRequest"), node);
    chkRequestThrottleBulk =
        new MyRequestThrottle(
            5000, "CHK Request", fs == null ? null : fs.subset("CHKRequestThrottle"), 32768, false);
    chkRequestThrottleRT =
        new MyRequestThrottle(
            5000,
            "CHK Request (RT)",
            fs == null ? null : fs.subset("CHKRequestThrottleRT"),
            32768,
            true);
    chkRequestStarterBulk =
        new RequestStarter(
            core,
            chkRequestThrottleBulk,
            "CHK Request starter (" + portNumber + ')',
            stats.requestOutputThrottle,
            stats.requestInputThrottle,
            stats.localChkFetchBytesSentAverage,
            stats.localChkFetchBytesReceivedAverage,
            false,
            false,
            false);
    chkRequestStarterRT =
        new RequestStarter(
            core,
            chkRequestThrottleRT,
            "CHK Request starter (" + portNumber + ')',
            stats.requestOutputThrottle,
            stats.requestInputThrottle,
            stats.localChkFetchBytesSentAverage,
            stats.localChkFetchBytesReceivedAverage,
            false,
            false,
            true);
    chkFetchSchedulerBulk =
        new ClientRequestScheduler(
            false, false, false, random, chkRequestStarterBulk, node, core, "CHKrequester", ctx);
    chkFetchSchedulerRT =
        new ClientRequestScheduler(
            false, false, true, random, chkRequestStarterRT, node, core, "CHKrequester", ctx);
    chkRequestStarterBulk.setScheduler(chkFetchSchedulerBulk);
    chkRequestStarterRT.setScheduler(chkFetchSchedulerRT);

    registerSchedulerConfig(
        schedulerConfig, "CHKrequester", chkFetchSchedulerBulk, chkFetchSchedulerRT, false, false);

    // insertThrottle = new ChainedRequestThrottle(10000, 2.0F, requestThrottle);
    // FIXME reenable the above
    chkInsertThrottleBulk =
        new MyRequestThrottle(
            20000, "CHK Insert", fs == null ? null : fs.subset("CHKInsertThrottle"), 32768, false);
    chkInsertThrottleRT =
        new MyRequestThrottle(
            20000,
            "CHK Insert (RT)",
            fs == null ? null : fs.subset("CHKInsertThrottleRT"),
            32768,
            true);
    chkInsertStarterBulk =
        new RequestStarter(
            core,
            chkInsertThrottleBulk,
            "CHK Insert starter (" + portNumber + ')',
            stats.requestOutputThrottle,
            stats.requestInputThrottle,
            stats.localChkInsertBytesSentAverage,
            stats.localChkInsertBytesReceivedAverage,
            true,
            false,
            false);
    chkInsertStarterRT =
        new RequestStarter(
            core,
            chkInsertThrottleRT,
            "CHK Insert starter (" + portNumber + ')',
            stats.requestOutputThrottle,
            stats.requestInputThrottle,
            stats.localChkInsertBytesSentAverage,
            stats.localChkInsertBytesReceivedAverage,
            true,
            false,
            true);
    chkPutSchedulerBulk =
        new ClientRequestScheduler(
            true, false, false, random, chkInsertStarterBulk, node, core, "CHKinserter", ctx);
    chkPutSchedulerRT =
        new ClientRequestScheduler(
            true, false, true, random, chkInsertStarterRT, node, core, "CHKinserter", ctx);
    chkInsertStarterBulk.setScheduler(chkPutSchedulerBulk);
    chkInsertStarterRT.setScheduler(chkPutSchedulerRT);

    registerSchedulerConfig(
        schedulerConfig, "CHKinserter", chkPutSchedulerBulk, chkPutSchedulerRT, false, true);

    sskRequestThrottleBulk =
        new MyRequestThrottle(
            5000, "SSK Request", fs == null ? null : fs.subset("SSKRequestThrottle"), 1024, false);
    sskRequestThrottleRT =
        new MyRequestThrottle(
            5000,
            "SSK Request (RT)",
            fs == null ? null : fs.subset("SSKRequestThrottleRT"),
            1024,
            true);
    sskRequestStarterBulk =
        new RequestStarter(
            core,
            sskRequestThrottleBulk,
            "SSK Request starter (" + portNumber + ')',
            stats.requestOutputThrottle,
            stats.requestInputThrottle,
            stats.localSskFetchBytesSentAverage,
            stats.localSskFetchBytesReceivedAverage,
            false,
            true,
            false);
    sskRequestStarterRT =
        new RequestStarter(
            core,
            sskRequestThrottleRT,
            "SSK Request starter (" + portNumber + ')',
            stats.requestOutputThrottle,
            stats.requestInputThrottle,
            stats.localSskFetchBytesSentAverage,
            stats.localSskFetchBytesReceivedAverage,
            false,
            true,
            true);
    sskFetchSchedulerBulk =
        new ClientRequestScheduler(
            false, true, false, random, sskRequestStarterBulk, node, core, "SSKrequester", ctx);
    sskFetchSchedulerRT =
        new ClientRequestScheduler(
            false, true, true, random, sskRequestStarterRT, node, core, "SSKrequester", ctx);
    sskRequestStarterBulk.setScheduler(sskFetchSchedulerBulk);
    sskRequestStarterRT.setScheduler(sskFetchSchedulerRT);

    registerSchedulerConfig(
        schedulerConfig, "SSKrequester", sskFetchSchedulerBulk, sskFetchSchedulerRT, true, false);

    // insertThrottle = new ChainedRequestThrottle(10000, 2.0F, requestThrottle);
    // FIXME reenable the above
    sskInsertThrottleBulk =
        new MyRequestThrottle(
            20000, "SSK Insert", fs == null ? null : fs.subset("SSKInsertThrottle"), 1024, false);
    sskInsertThrottleRT =
        new MyRequestThrottle(
            20000, "SSK Insert", fs == null ? null : fs.subset("SSKInsertThrottleRT"), 1024, true);
    sskInsertStarterBulk =
        new RequestStarter(
            core,
            sskInsertThrottleBulk,
            "SSK Insert starter (" + portNumber + ')',
            stats.requestOutputThrottle,
            stats.requestInputThrottle,
            stats.localSskInsertBytesSentAverage,
            stats.localSskFetchBytesReceivedAverage,
            true,
            true,
            false);
    sskInsertStarterRT =
        new RequestStarter(
            core,
            sskInsertThrottleRT,
            "SSK Insert starter (" + portNumber + ')',
            stats.requestOutputThrottle,
            stats.requestInputThrottle,
            stats.localSskInsertBytesSentAverage,
            stats.localSskFetchBytesReceivedAverage,
            true,
            true,
            true);
    sskPutSchedulerBulk =
        new ClientRequestScheduler(
            true, true, false, random, sskInsertStarterBulk, node, core, "SSKinserter", ctx);
    sskPutSchedulerRT =
        new ClientRequestScheduler(
            true, true, true, random, sskInsertStarterRT, node, core, "SSKinserter", ctx);
    sskInsertStarterBulk.setScheduler(sskPutSchedulerBulk);
    sskInsertStarterRT.setScheduler(sskPutSchedulerRT);

    registerSchedulerConfig(
        schedulerConfig, "SSKinserter", sskPutSchedulerBulk, sskPutSchedulerRT, true, true);

    schedulerConfig.finishedInitialization();
  }
Ejemplo n.º 4
0
 public DelayedFreeBucket(SimpleFieldSet fs, RandomSource random, PersistentFileTracker f)
     throws CannotCreateFromFieldSetException {
   factory = f;
   freed = false;
   bucket = SerializableToFieldSetBucketUtil.create(fs.subset("Underlying"), random, f);
 }