public MyRequestThrottle( int rtt, String string, SimpleFieldSet fs, int size, boolean realTime) { roundTripTime = new BootstrappingDecayingRunningAverage( rtt, 10, MINUTES.toMillis(5), 10, fs == null ? null : fs.subset("RoundTripTime")); this.size = size; this.realTime = realTime; }
/** * Create a ClientGet from a request serialized to a SimpleFieldSet. Can throw, and does minimal * verification, as is dealing with data supposedly serialized out by the node. * * @throws IOException * @throws FetchException */ public ClientGet(SimpleFieldSet fs, FCPClient client2, FCPServer server) throws IOException, FetchException { super(fs, client2); returnType = ClientGetMessage.parseValidReturnType(fs.get("ReturnType")); String f = fs.get("Filename"); if (f != null) targetFile = new File(f); else targetFile = null; f = fs.get("TempFilename"); if (f != null) tempFile = new File(f); else tempFile = null; boolean ignoreDS = Fields.stringToBool(fs.get("IgnoreDS"), false); boolean dsOnly = Fields.stringToBool(fs.get("DSOnly"), false); int maxRetries = Integer.parseInt(fs.get("MaxRetries")); fctx = new FetchContext(server.defaultFetchContext, FetchContext.IDENTICAL_MASK, false, null); fctx.eventProducer.addEventListener(this); // ignoreDS fctx.localRequestOnly = dsOnly; fctx.ignoreStore = ignoreDS; fctx.maxNonSplitfileRetries = maxRetries; fctx.maxSplitfileBlockRetries = maxRetries; binaryBlob = Fields.stringToBool(fs.get("BinaryBlob"), false); succeeded = Fields.stringToBool(fs.get("Succeeded"), false); if (finished) { if (succeeded) { foundDataLength = Long.parseLong(fs.get("FoundDataLength")); foundDataMimeType = fs.get("FoundDataMimeType"); SimpleFieldSet fs1 = fs.subset("PostFetchProtocolError"); if (fs1 != null) postFetchProtocolErrorMessage = new ProtocolErrorMessage(fs1); } else { getFailedMessage = new GetFailedMessage(fs.subset("GetFailed"), false); } } Bucket ret = null; if (returnType == ClientGetMessage.RETURN_TYPE_DISK) { if (succeeded) { ret = new FileBucket(targetFile, false, true, false, false, false); } else { ret = new FileBucket(tempFile, false, true, false, false, false); } } else if (returnType == ClientGetMessage.RETURN_TYPE_NONE) { ret = new NullBucket(); } else if (returnType == ClientGetMessage.RETURN_TYPE_DIRECT) { try { ret = SerializableToFieldSetBucketUtil.create( fs.subset("ReturnBucket"), server.core.random, server.core.persistentTempBucketFactory); if (ret == null) throw new CannotCreateFromFieldSetException("ret == null"); } catch (CannotCreateFromFieldSetException e) { Logger.error(this, "Cannot read: " + this + " : " + e, e); try { // Create a new temp bucket if (persistenceType == PERSIST_FOREVER) ret = server.core.persistentTempBucketFactory.makeBucket(fctx.maxOutputLength); else ret = server.core.tempBucketFactory.makeBucket(fctx.maxOutputLength); } catch (IOException e1) { Logger.error(this, "Cannot create bucket for temp storage: " + e, e); getter = null; returnBucket = null; throw new FetchException(FetchException.BUCKET_ERROR, e); } } } else { throw new IllegalArgumentException(); } if (succeeded) { if (foundDataLength < ret.size()) { Logger.error(this, "Failing " + identifier + " because lost data"); succeeded = false; } } if (ret == null) Logger.error( this, "Impossible: ret = null in SFS constructor for " + this, new Exception("debug")); returnBucket = ret; String[] allowed = fs.getAll("AllowedMIMETypes"); if (allowed != null) { fctx.allowedMIMETypes = new HashSet<String>(); for (String a : allowed) fctx.allowedMIMETypes.add(a); } getter = new ClientGetter( this, uri, fctx, priorityClass, lowLevelClient, binaryBlob ? new NullBucket() : returnBucket, binaryBlob ? returnBucket : null); if (finished && succeeded) allDataPending = new AllDataMessage( returnBucket, identifier, global, startupTime, completionTime, this.foundDataMimeType); }
RequestStarterGroup( Node node, NodeClientCore core, int portNumber, RandomSource random, Config config, SimpleFieldSet fs, ClientContext ctx, long dbHandle) throws InvalidConfigValueException { SubConfig schedulerConfig = config.createSubConfig("node.scheduler"); this.stats = core.nodeStats; throttleWindowBulk = new ThrottleWindowManager(2.0, fs == null ? null : fs.subset("ThrottleWindow"), node); throttleWindowRT = new ThrottleWindowManager(2.0, fs == null ? null : fs.subset("ThrottleWindowRT"), node); throttleWindowCHK = new ThrottleWindowManager(2.0, fs == null ? null : fs.subset("ThrottleWindowCHK"), node); throttleWindowSSK = new ThrottleWindowManager(2.0, fs == null ? null : fs.subset("ThrottleWindowSSK"), node); throttleWindowInsert = new ThrottleWindowManager(2.0, fs == null ? null : fs.subset("ThrottleWindowInsert"), node); throttleWindowRequest = new ThrottleWindowManager( 2.0, fs == null ? null : fs.subset("ThrottleWindowRequest"), node); chkRequestThrottleBulk = new MyRequestThrottle( 5000, "CHK Request", fs == null ? null : fs.subset("CHKRequestThrottle"), 32768, false); chkRequestThrottleRT = new MyRequestThrottle( 5000, "CHK Request (RT)", fs == null ? null : fs.subset("CHKRequestThrottleRT"), 32768, true); chkRequestStarterBulk = new RequestStarter( core, chkRequestThrottleBulk, "CHK Request starter (" + portNumber + ')', stats.requestOutputThrottle, stats.requestInputThrottle, stats.localChkFetchBytesSentAverage, stats.localChkFetchBytesReceivedAverage, false, false, false); chkRequestStarterRT = new RequestStarter( core, chkRequestThrottleRT, "CHK Request starter (" + portNumber + ')', stats.requestOutputThrottle, stats.requestInputThrottle, stats.localChkFetchBytesSentAverage, stats.localChkFetchBytesReceivedAverage, false, false, true); chkFetchSchedulerBulk = new ClientRequestScheduler( false, false, false, random, chkRequestStarterBulk, node, core, "CHKrequester", ctx); chkFetchSchedulerRT = new ClientRequestScheduler( false, false, true, random, chkRequestStarterRT, node, core, "CHKrequester", ctx); chkRequestStarterBulk.setScheduler(chkFetchSchedulerBulk); chkRequestStarterRT.setScheduler(chkFetchSchedulerRT); registerSchedulerConfig( schedulerConfig, "CHKrequester", chkFetchSchedulerBulk, chkFetchSchedulerRT, false, false); // insertThrottle = new ChainedRequestThrottle(10000, 2.0F, requestThrottle); // FIXME reenable the above chkInsertThrottleBulk = new MyRequestThrottle( 20000, "CHK Insert", fs == null ? null : fs.subset("CHKInsertThrottle"), 32768, false); chkInsertThrottleRT = new MyRequestThrottle( 20000, "CHK Insert (RT)", fs == null ? null : fs.subset("CHKInsertThrottleRT"), 32768, true); chkInsertStarterBulk = new RequestStarter( core, chkInsertThrottleBulk, "CHK Insert starter (" + portNumber + ')', stats.requestOutputThrottle, stats.requestInputThrottle, stats.localChkInsertBytesSentAverage, stats.localChkInsertBytesReceivedAverage, true, false, false); chkInsertStarterRT = new RequestStarter( core, chkInsertThrottleRT, "CHK Insert starter (" + portNumber + ')', stats.requestOutputThrottle, stats.requestInputThrottle, stats.localChkInsertBytesSentAverage, stats.localChkInsertBytesReceivedAverage, true, false, true); chkPutSchedulerBulk = new ClientRequestScheduler( true, false, false, random, chkInsertStarterBulk, node, core, "CHKinserter", ctx); chkPutSchedulerRT = new ClientRequestScheduler( true, false, true, random, chkInsertStarterRT, node, core, "CHKinserter", ctx); chkInsertStarterBulk.setScheduler(chkPutSchedulerBulk); chkInsertStarterRT.setScheduler(chkPutSchedulerRT); registerSchedulerConfig( schedulerConfig, "CHKinserter", chkPutSchedulerBulk, chkPutSchedulerRT, false, true); sskRequestThrottleBulk = new MyRequestThrottle( 5000, "SSK Request", fs == null ? null : fs.subset("SSKRequestThrottle"), 1024, false); sskRequestThrottleRT = new MyRequestThrottle( 5000, "SSK Request (RT)", fs == null ? null : fs.subset("SSKRequestThrottleRT"), 1024, true); sskRequestStarterBulk = new RequestStarter( core, sskRequestThrottleBulk, "SSK Request starter (" + portNumber + ')', stats.requestOutputThrottle, stats.requestInputThrottle, stats.localSskFetchBytesSentAverage, stats.localSskFetchBytesReceivedAverage, false, true, false); sskRequestStarterRT = new RequestStarter( core, sskRequestThrottleRT, "SSK Request starter (" + portNumber + ')', stats.requestOutputThrottle, stats.requestInputThrottle, stats.localSskFetchBytesSentAverage, stats.localSskFetchBytesReceivedAverage, false, true, true); sskFetchSchedulerBulk = new ClientRequestScheduler( false, true, false, random, sskRequestStarterBulk, node, core, "SSKrequester", ctx); sskFetchSchedulerRT = new ClientRequestScheduler( false, true, true, random, sskRequestStarterRT, node, core, "SSKrequester", ctx); sskRequestStarterBulk.setScheduler(sskFetchSchedulerBulk); sskRequestStarterRT.setScheduler(sskFetchSchedulerRT); registerSchedulerConfig( schedulerConfig, "SSKrequester", sskFetchSchedulerBulk, sskFetchSchedulerRT, true, false); // insertThrottle = new ChainedRequestThrottle(10000, 2.0F, requestThrottle); // FIXME reenable the above sskInsertThrottleBulk = new MyRequestThrottle( 20000, "SSK Insert", fs == null ? null : fs.subset("SSKInsertThrottle"), 1024, false); sskInsertThrottleRT = new MyRequestThrottle( 20000, "SSK Insert", fs == null ? null : fs.subset("SSKInsertThrottleRT"), 1024, true); sskInsertStarterBulk = new RequestStarter( core, sskInsertThrottleBulk, "SSK Insert starter (" + portNumber + ')', stats.requestOutputThrottle, stats.requestInputThrottle, stats.localSskInsertBytesSentAverage, stats.localSskFetchBytesReceivedAverage, true, true, false); sskInsertStarterRT = new RequestStarter( core, sskInsertThrottleRT, "SSK Insert starter (" + portNumber + ')', stats.requestOutputThrottle, stats.requestInputThrottle, stats.localSskInsertBytesSentAverage, stats.localSskFetchBytesReceivedAverage, true, true, true); sskPutSchedulerBulk = new ClientRequestScheduler( true, true, false, random, sskInsertStarterBulk, node, core, "SSKinserter", ctx); sskPutSchedulerRT = new ClientRequestScheduler( true, true, true, random, sskInsertStarterRT, node, core, "SSKinserter", ctx); sskInsertStarterBulk.setScheduler(sskPutSchedulerBulk); sskInsertStarterRT.setScheduler(sskPutSchedulerRT); registerSchedulerConfig( schedulerConfig, "SSKinserter", sskPutSchedulerBulk, sskPutSchedulerRT, true, true); schedulerConfig.finishedInitialization(); }
public DelayedFreeBucket(SimpleFieldSet fs, RandomSource random, PersistentFileTracker f) throws CannotCreateFromFieldSetException { factory = f; freed = false; bucket = SerializableToFieldSetBucketUtil.create(fs.subset("Underlying"), random, f); }