public MyRequestThrottle(
int rtt, String string, SimpleFieldSet fs, int size, boolean realTime) {
roundTripTime =
new BootstrappingDecayingRunningAverage(
rtt, 10, MINUTES.toMillis(5), 10, fs == null ? null : fs.subset("RoundTripTime"));
this.size = size;
this.realTime = realTime;
}
/**
* Create a ClientGet from a request serialized to a SimpleFieldSet. Can throw, and does minimal
* verification, as is dealing with data supposedly serialized out by the node.
*
* @throws IOException
* @throws FetchException
*/
public ClientGet(SimpleFieldSet fs, FCPClient client2, FCPServer server)
throws IOException, FetchException {
super(fs, client2);
returnType = ClientGetMessage.parseValidReturnType(fs.get("ReturnType"));
String f = fs.get("Filename");
if (f != null) targetFile = new File(f);
else targetFile = null;
f = fs.get("TempFilename");
if (f != null) tempFile = new File(f);
else tempFile = null;
boolean ignoreDS = Fields.stringToBool(fs.get("IgnoreDS"), false);
boolean dsOnly = Fields.stringToBool(fs.get("DSOnly"), false);
int maxRetries = Integer.parseInt(fs.get("MaxRetries"));
fctx = new FetchContext(server.defaultFetchContext, FetchContext.IDENTICAL_MASK, false, null);
fctx.eventProducer.addEventListener(this);
// ignoreDS
fctx.localRequestOnly = dsOnly;
fctx.ignoreStore = ignoreDS;
fctx.maxNonSplitfileRetries = maxRetries;
fctx.maxSplitfileBlockRetries = maxRetries;
binaryBlob = Fields.stringToBool(fs.get("BinaryBlob"), false);
succeeded = Fields.stringToBool(fs.get("Succeeded"), false);
if (finished) {
if (succeeded) {
foundDataLength = Long.parseLong(fs.get("FoundDataLength"));
foundDataMimeType = fs.get("FoundDataMimeType");
SimpleFieldSet fs1 = fs.subset("PostFetchProtocolError");
if (fs1 != null) postFetchProtocolErrorMessage = new ProtocolErrorMessage(fs1);
} else {
getFailedMessage = new GetFailedMessage(fs.subset("GetFailed"), false);
}
}
Bucket ret = null;
if (returnType == ClientGetMessage.RETURN_TYPE_DISK) {
if (succeeded) {
ret = new FileBucket(targetFile, false, true, false, false, false);
} else {
ret = new FileBucket(tempFile, false, true, false, false, false);
}
} else if (returnType == ClientGetMessage.RETURN_TYPE_NONE) {
ret = new NullBucket();
} else if (returnType == ClientGetMessage.RETURN_TYPE_DIRECT) {
try {
ret =
SerializableToFieldSetBucketUtil.create(
fs.subset("ReturnBucket"),
server.core.random,
server.core.persistentTempBucketFactory);
if (ret == null) throw new CannotCreateFromFieldSetException("ret == null");
} catch (CannotCreateFromFieldSetException e) {
Logger.error(this, "Cannot read: " + this + " : " + e, e);
try {
// Create a new temp bucket
if (persistenceType == PERSIST_FOREVER)
ret = server.core.persistentTempBucketFactory.makeBucket(fctx.maxOutputLength);
else ret = server.core.tempBucketFactory.makeBucket(fctx.maxOutputLength);
} catch (IOException e1) {
Logger.error(this, "Cannot create bucket for temp storage: " + e, e);
getter = null;
returnBucket = null;
throw new FetchException(FetchException.BUCKET_ERROR, e);
}
}
} else {
throw new IllegalArgumentException();
}
if (succeeded) {
if (foundDataLength < ret.size()) {
Logger.error(this, "Failing " + identifier + " because lost data");
succeeded = false;
}
}
if (ret == null)
Logger.error(
this, "Impossible: ret = null in SFS constructor for " + this, new Exception("debug"));
returnBucket = ret;
String[] allowed = fs.getAll("AllowedMIMETypes");
if (allowed != null) {
fctx.allowedMIMETypes = new HashSet<String>();
for (String a : allowed) fctx.allowedMIMETypes.add(a);
}
getter =
new ClientGetter(
this,
uri,
fctx,
priorityClass,
lowLevelClient,
binaryBlob ? new NullBucket() : returnBucket,
binaryBlob ? returnBucket : null);
if (finished && succeeded)
allDataPending =
new AllDataMessage(
returnBucket,
identifier,
global,
startupTime,
completionTime,
this.foundDataMimeType);
}
RequestStarterGroup(
Node node,
NodeClientCore core,
int portNumber,
RandomSource random,
Config config,
SimpleFieldSet fs,
ClientContext ctx,
long dbHandle)
throws InvalidConfigValueException {
SubConfig schedulerConfig = config.createSubConfig("node.scheduler");
this.stats = core.nodeStats;
throttleWindowBulk =
new ThrottleWindowManager(2.0, fs == null ? null : fs.subset("ThrottleWindow"), node);
throttleWindowRT =
new ThrottleWindowManager(2.0, fs == null ? null : fs.subset("ThrottleWindowRT"), node);
throttleWindowCHK =
new ThrottleWindowManager(2.0, fs == null ? null : fs.subset("ThrottleWindowCHK"), node);
throttleWindowSSK =
new ThrottleWindowManager(2.0, fs == null ? null : fs.subset("ThrottleWindowSSK"), node);
throttleWindowInsert =
new ThrottleWindowManager(2.0, fs == null ? null : fs.subset("ThrottleWindowInsert"), node);
throttleWindowRequest =
new ThrottleWindowManager(
2.0, fs == null ? null : fs.subset("ThrottleWindowRequest"), node);
chkRequestThrottleBulk =
new MyRequestThrottle(
5000, "CHK Request", fs == null ? null : fs.subset("CHKRequestThrottle"), 32768, false);
chkRequestThrottleRT =
new MyRequestThrottle(
5000,
"CHK Request (RT)",
fs == null ? null : fs.subset("CHKRequestThrottleRT"),
32768,
true);
chkRequestStarterBulk =
new RequestStarter(
core,
chkRequestThrottleBulk,
"CHK Request starter (" + portNumber + ')',
stats.requestOutputThrottle,
stats.requestInputThrottle,
stats.localChkFetchBytesSentAverage,
stats.localChkFetchBytesReceivedAverage,
false,
false,
false);
chkRequestStarterRT =
new RequestStarter(
core,
chkRequestThrottleRT,
"CHK Request starter (" + portNumber + ')',
stats.requestOutputThrottle,
stats.requestInputThrottle,
stats.localChkFetchBytesSentAverage,
stats.localChkFetchBytesReceivedAverage,
false,
false,
true);
chkFetchSchedulerBulk =
new ClientRequestScheduler(
false, false, false, random, chkRequestStarterBulk, node, core, "CHKrequester", ctx);
chkFetchSchedulerRT =
new ClientRequestScheduler(
false, false, true, random, chkRequestStarterRT, node, core, "CHKrequester", ctx);
chkRequestStarterBulk.setScheduler(chkFetchSchedulerBulk);
chkRequestStarterRT.setScheduler(chkFetchSchedulerRT);
registerSchedulerConfig(
schedulerConfig, "CHKrequester", chkFetchSchedulerBulk, chkFetchSchedulerRT, false, false);
// insertThrottle = new ChainedRequestThrottle(10000, 2.0F, requestThrottle);
// FIXME reenable the above
chkInsertThrottleBulk =
new MyRequestThrottle(
20000, "CHK Insert", fs == null ? null : fs.subset("CHKInsertThrottle"), 32768, false);
chkInsertThrottleRT =
new MyRequestThrottle(
20000,
"CHK Insert (RT)",
fs == null ? null : fs.subset("CHKInsertThrottleRT"),
32768,
true);
chkInsertStarterBulk =
new RequestStarter(
core,
chkInsertThrottleBulk,
"CHK Insert starter (" + portNumber + ')',
stats.requestOutputThrottle,
stats.requestInputThrottle,
stats.localChkInsertBytesSentAverage,
stats.localChkInsertBytesReceivedAverage,
true,
false,
false);
chkInsertStarterRT =
new RequestStarter(
core,
chkInsertThrottleRT,
"CHK Insert starter (" + portNumber + ')',
stats.requestOutputThrottle,
stats.requestInputThrottle,
stats.localChkInsertBytesSentAverage,
stats.localChkInsertBytesReceivedAverage,
true,
false,
true);
chkPutSchedulerBulk =
new ClientRequestScheduler(
true, false, false, random, chkInsertStarterBulk, node, core, "CHKinserter", ctx);
chkPutSchedulerRT =
new ClientRequestScheduler(
true, false, true, random, chkInsertStarterRT, node, core, "CHKinserter", ctx);
chkInsertStarterBulk.setScheduler(chkPutSchedulerBulk);
chkInsertStarterRT.setScheduler(chkPutSchedulerRT);
registerSchedulerConfig(
schedulerConfig, "CHKinserter", chkPutSchedulerBulk, chkPutSchedulerRT, false, true);
sskRequestThrottleBulk =
new MyRequestThrottle(
5000, "SSK Request", fs == null ? null : fs.subset("SSKRequestThrottle"), 1024, false);
sskRequestThrottleRT =
new MyRequestThrottle(
5000,
"SSK Request (RT)",
fs == null ? null : fs.subset("SSKRequestThrottleRT"),
1024,
true);
sskRequestStarterBulk =
new RequestStarter(
core,
sskRequestThrottleBulk,
"SSK Request starter (" + portNumber + ')',
stats.requestOutputThrottle,
stats.requestInputThrottle,
stats.localSskFetchBytesSentAverage,
stats.localSskFetchBytesReceivedAverage,
false,
true,
false);
sskRequestStarterRT =
new RequestStarter(
core,
sskRequestThrottleRT,
"SSK Request starter (" + portNumber + ')',
stats.requestOutputThrottle,
stats.requestInputThrottle,
stats.localSskFetchBytesSentAverage,
stats.localSskFetchBytesReceivedAverage,
false,
true,
true);
sskFetchSchedulerBulk =
new ClientRequestScheduler(
false, true, false, random, sskRequestStarterBulk, node, core, "SSKrequester", ctx);
sskFetchSchedulerRT =
new ClientRequestScheduler(
false, true, true, random, sskRequestStarterRT, node, core, "SSKrequester", ctx);
sskRequestStarterBulk.setScheduler(sskFetchSchedulerBulk);
sskRequestStarterRT.setScheduler(sskFetchSchedulerRT);
registerSchedulerConfig(
schedulerConfig, "SSKrequester", sskFetchSchedulerBulk, sskFetchSchedulerRT, true, false);
// insertThrottle = new ChainedRequestThrottle(10000, 2.0F, requestThrottle);
// FIXME reenable the above
sskInsertThrottleBulk =
new MyRequestThrottle(
20000, "SSK Insert", fs == null ? null : fs.subset("SSKInsertThrottle"), 1024, false);
sskInsertThrottleRT =
new MyRequestThrottle(
20000, "SSK Insert", fs == null ? null : fs.subset("SSKInsertThrottleRT"), 1024, true);
sskInsertStarterBulk =
new RequestStarter(
core,
sskInsertThrottleBulk,
"SSK Insert starter (" + portNumber + ')',
stats.requestOutputThrottle,
stats.requestInputThrottle,
stats.localSskInsertBytesSentAverage,
stats.localSskFetchBytesReceivedAverage,
true,
true,
false);
sskInsertStarterRT =
new RequestStarter(
core,
sskInsertThrottleRT,
"SSK Insert starter (" + portNumber + ')',
stats.requestOutputThrottle,
stats.requestInputThrottle,
stats.localSskInsertBytesSentAverage,
stats.localSskFetchBytesReceivedAverage,
true,
true,
true);
sskPutSchedulerBulk =
new ClientRequestScheduler(
true, true, false, random, sskInsertStarterBulk, node, core, "SSKinserter", ctx);
sskPutSchedulerRT =
new ClientRequestScheduler(
true, true, true, random, sskInsertStarterRT, node, core, "SSKinserter", ctx);
sskInsertStarterBulk.setScheduler(sskPutSchedulerBulk);
sskInsertStarterRT.setScheduler(sskPutSchedulerRT);
registerSchedulerConfig(
schedulerConfig, "SSKinserter", sskPutSchedulerBulk, sskPutSchedulerRT, true, true);
schedulerConfig.finishedInitialization();
}
public DelayedFreeBucket(SimpleFieldSet fs, RandomSource random, PersistentFileTracker f)
throws CannotCreateFromFieldSetException {
factory = f;
freed = false;
bucket = SerializableToFieldSetBucketUtil.create(fs.subset("Underlying"), random, f);
}