private void addRequest(FreenetURI uri) {
if (isRunning) {
FetchContext mFetchContext = mFetcher.getFetchContext();
mFetchContext.allowSplitfiles = true; // FIXME: disable as soon as its fixed!
mFetchContext.canWriteClientCache = true;
mFetchContext.dontEnterImplicitArchives = true; // ?
mFetchContext.filterData = false; // ?
mFetchContext.followRedirects = false;
mFetchContext.ignoreStore = false;
// final? mFetchContext.ignoreTooManyPathComponents = false;
mFetchContext.ignoreUSKDatehints = true; // ?
mFetchContext.localRequestOnly = false;
mFetchContext.maxArchiveLevels = 0; // ?
mFetchContext.maxArchiveRestarts = 0; // ?
mFetchContext.maxCheckBlocksPerSegment = 0; // ?
mFetchContext.maxDataBlocksPerSegment = 0; // ?
// mFetchContext.maxMetadataSize = ?
// cooldown for 30 minutes, wtf? this is a real time chat plugin.
// mFetchContext.maxNonSplitfileRetries = -1;
mFetchContext.maxNonSplitfileRetries = 2;
// mFetchContext.maxOutputLength = 1024 ?
mFetchContext.maxRecursionLevel = 0; // ?
mFetchContext.maxSplitfileBlockRetries = 0;
// mFetchContext.maxTempLength = ?
// final? mFetchContext.maxUSKRetries = -1; //?
// mFetchContext.overrideMIME = "text/plain"; //?
// mFetchContext.prefetchHook = ?
// mFetchContext.returnZIPManifests = true ?
// mFetchContext.tagReplacer = ?
// mFetchContext.setCooldownRetries(cooldownRetries);
// mFetchContext.setCooldownTime(cooldownTime);
try {
mFetcher.fetch(uri, this, this, mFetchContext, (short) 1);
} catch (FetchException e) {
System.err.println(
"[Async_AnnounceFetcher]::addRequest() FetchException: " + e.getMessage());
}
}
}
/**
* Create one for a global-queued request not made by FCP.
*
* @throws IdentifierCollisionException
* @throws NotAllowedException
* @throws IOException
*/
public ClientGet(
FCPClient globalClient,
FreenetURI uri,
boolean dsOnly,
boolean ignoreDS,
int maxSplitfileRetries,
int maxNonSplitfileRetries,
long maxOutputLength,
short returnType,
boolean persistRebootOnly,
String identifier,
int verbosity,
short prioClass,
File returnFilename,
File returnTempFilename,
boolean writeToClientCache,
FCPServer server,
ObjectContainer container)
throws IdentifierCollisionException, NotAllowedException, IOException {
super(
uri,
identifier,
verbosity,
null,
globalClient,
prioClass,
(persistRebootOnly ? ClientRequest.PERSIST_REBOOT : ClientRequest.PERSIST_FOREVER),
null,
true,
container);
fctx = new FetchContext(server.defaultFetchContext, FetchContext.IDENTICAL_MASK, false, null);
fctx.eventProducer.addEventListener(this);
fctx.localRequestOnly = dsOnly;
fctx.ignoreStore = ignoreDS;
fctx.maxNonSplitfileRetries = maxNonSplitfileRetries;
fctx.maxSplitfileBlockRetries = maxSplitfileRetries;
fctx.maxOutputLength = maxOutputLength;
fctx.maxTempLength = maxOutputLength;
fctx.canWriteClientCache = writeToClientCache;
Bucket ret = null;
this.returnType = returnType;
binaryBlob = false;
if (returnType == ClientGetMessage.RETURN_TYPE_DISK) {
this.targetFile = returnFilename;
this.tempFile = returnTempFilename;
if (!(server.core.allowDownloadTo(returnTempFilename)
&& server.core.allowDownloadTo(returnFilename))) throw new NotAllowedException();
ret = new FileBucket(returnTempFilename, false, true, false, false, false);
} else if (returnType == ClientGetMessage.RETURN_TYPE_NONE) {
targetFile = null;
tempFile = null;
ret = new NullBucket();
} else {
targetFile = null;
tempFile = null;
if (persistenceType == PERSIST_FOREVER)
ret = server.core.persistentTempBucketFactory.makeBucket(maxOutputLength);
else ret = server.core.tempBucketFactory.makeBucket(maxOutputLength);
}
returnBucket = ret;
getter = new ClientGetter(this, uri, fctx, priorityClass, lowLevelClient, returnBucket, null);
}
/**
* Create a ClientGet from a request serialized to a SimpleFieldSet. Can throw, and does minimal
* verification, as is dealing with data supposedly serialized out by the node.
*
* @throws IOException
* @throws FetchException
*/
public ClientGet(SimpleFieldSet fs, FCPClient client2, FCPServer server)
throws IOException, FetchException {
super(fs, client2);
returnType = ClientGetMessage.parseValidReturnType(fs.get("ReturnType"));
String f = fs.get("Filename");
if (f != null) targetFile = new File(f);
else targetFile = null;
f = fs.get("TempFilename");
if (f != null) tempFile = new File(f);
else tempFile = null;
boolean ignoreDS = Fields.stringToBool(fs.get("IgnoreDS"), false);
boolean dsOnly = Fields.stringToBool(fs.get("DSOnly"), false);
int maxRetries = Integer.parseInt(fs.get("MaxRetries"));
fctx = new FetchContext(server.defaultFetchContext, FetchContext.IDENTICAL_MASK, false, null);
fctx.eventProducer.addEventListener(this);
// ignoreDS
fctx.localRequestOnly = dsOnly;
fctx.ignoreStore = ignoreDS;
fctx.maxNonSplitfileRetries = maxRetries;
fctx.maxSplitfileBlockRetries = maxRetries;
binaryBlob = Fields.stringToBool(fs.get("BinaryBlob"), false);
succeeded = Fields.stringToBool(fs.get("Succeeded"), false);
if (finished) {
if (succeeded) {
foundDataLength = Long.parseLong(fs.get("FoundDataLength"));
foundDataMimeType = fs.get("FoundDataMimeType");
SimpleFieldSet fs1 = fs.subset("PostFetchProtocolError");
if (fs1 != null) postFetchProtocolErrorMessage = new ProtocolErrorMessage(fs1);
} else {
getFailedMessage = new GetFailedMessage(fs.subset("GetFailed"), false);
}
}
Bucket ret = null;
if (returnType == ClientGetMessage.RETURN_TYPE_DISK) {
if (succeeded) {
ret = new FileBucket(targetFile, false, true, false, false, false);
} else {
ret = new FileBucket(tempFile, false, true, false, false, false);
}
} else if (returnType == ClientGetMessage.RETURN_TYPE_NONE) {
ret = new NullBucket();
} else if (returnType == ClientGetMessage.RETURN_TYPE_DIRECT) {
try {
ret =
SerializableToFieldSetBucketUtil.create(
fs.subset("ReturnBucket"),
server.core.random,
server.core.persistentTempBucketFactory);
if (ret == null) throw new CannotCreateFromFieldSetException("ret == null");
} catch (CannotCreateFromFieldSetException e) {
Logger.error(this, "Cannot read: " + this + " : " + e, e);
try {
// Create a new temp bucket
if (persistenceType == PERSIST_FOREVER)
ret = server.core.persistentTempBucketFactory.makeBucket(fctx.maxOutputLength);
else ret = server.core.tempBucketFactory.makeBucket(fctx.maxOutputLength);
} catch (IOException e1) {
Logger.error(this, "Cannot create bucket for temp storage: " + e, e);
getter = null;
returnBucket = null;
throw new FetchException(FetchException.BUCKET_ERROR, e);
}
}
} else {
throw new IllegalArgumentException();
}
if (succeeded) {
if (foundDataLength < ret.size()) {
Logger.error(this, "Failing " + identifier + " because lost data");
succeeded = false;
}
}
if (ret == null)
Logger.error(
this, "Impossible: ret = null in SFS constructor for " + this, new Exception("debug"));
returnBucket = ret;
String[] allowed = fs.getAll("AllowedMIMETypes");
if (allowed != null) {
fctx.allowedMIMETypes = new HashSet<String>();
for (String a : allowed) fctx.allowedMIMETypes.add(a);
}
getter =
new ClientGetter(
this,
uri,
fctx,
priorityClass,
lowLevelClient,
binaryBlob ? new NullBucket() : returnBucket,
binaryBlob ? returnBucket : null);
if (finished && succeeded)
allDataPending =
new AllDataMessage(
returnBucket,
identifier,
global,
startupTime,
completionTime,
this.foundDataMimeType);
}
public ClientGet(
FCPConnectionHandler handler,
ClientGetMessage message,
FCPServer server,
ObjectContainer container)
throws IdentifierCollisionException, MessageInvalidException {
super(
message.uri,
message.identifier,
message.verbosity,
handler,
message.priorityClass,
message.persistenceType,
message.clientToken,
message.global,
container);
// Create a Fetcher directly in order to get more fine-grained control,
// since the client may override a few context elements.
fctx = new FetchContext(server.defaultFetchContext, FetchContext.IDENTICAL_MASK, false, null);
fctx.eventProducer.addEventListener(this);
// ignoreDS
fctx.localRequestOnly = message.dsOnly;
fctx.ignoreStore = message.ignoreDS;
fctx.maxNonSplitfileRetries = message.maxRetries;
fctx.maxSplitfileBlockRetries = message.maxRetries;
// FIXME do something with verbosity !!
// Has already been checked
fctx.maxOutputLength = message.maxSize;
fctx.maxTempLength = message.maxTempSize;
fctx.canWriteClientCache = message.writeToClientCache;
if (message.allowedMIMETypes != null) {
fctx.allowedMIMETypes = new HashSet<String>();
for (String mime : message.allowedMIMETypes) fctx.allowedMIMETypes.add(mime);
}
this.returnType = message.returnType;
this.binaryBlob = message.binaryBlob;
Bucket ret = null;
if (returnType == ClientGetMessage.RETURN_TYPE_DISK) {
this.targetFile = message.diskFile;
this.tempFile = message.tempFile;
if (!(server.core.allowDownloadTo(tempFile) && server.core.allowDownloadTo(targetFile)))
throw new MessageInvalidException(
ProtocolErrorMessage.ACCESS_DENIED,
"Not allowed to download to " + tempFile + " or " + targetFile,
identifier,
global);
else if (!(handler.allowDDAFrom(tempFile, true) && handler.allowDDAFrom(targetFile, true)))
throw new MessageInvalidException(
ProtocolErrorMessage.DIRECT_DISK_ACCESS_DENIED,
"Not allowed to download to "
+ tempFile
+ " or "
+ targetFile
+ ". You might need to do a "
+ TestDDARequestMessage.NAME
+ " first.",
identifier,
global);
ret = new FileBucket(message.tempFile, false, true, false, false, false);
} else if (returnType == ClientGetMessage.RETURN_TYPE_NONE) {
targetFile = null;
tempFile = null;
ret = new NullBucket();
} else {
targetFile = null;
tempFile = null;
try {
if (persistenceType == PERSIST_FOREVER)
ret = server.core.persistentTempBucketFactory.makeBucket(fctx.maxOutputLength);
else ret = server.core.tempBucketFactory.makeBucket(fctx.maxOutputLength);
} catch (IOException e) {
Logger.error(this, "Cannot create bucket for temp storage: " + e, e);
getter = null;
returnBucket = null;
// This is *not* a FetchException since we don't register it: it's a protocol error.
throw new MessageInvalidException(
ProtocolErrorMessage.INTERNAL_ERROR,
"Cannot create bucket for temporary storage (out of disk space???): " + e,
identifier,
global);
}
}
if (ret == null)
Logger.error(
this, "Impossible: ret = null in FCP constructor for " + this, new Exception("debug"));
returnBucket = ret;
getter =
new ClientGetter(
this,
uri,
fctx,
priorityClass,
lowLevelClient,
binaryBlob ? new NullBucket() : returnBucket,
binaryBlob ? returnBucket : null);
}
public static void main(String[] args) {
if (args.length < 1 || args.length > 2) {
System.err.println(
"Usage: java freenet.node.simulator.LongTermPushPullTest <unique identifier>");
System.exit(1);
}
String uid = args[0];
List<String> csvLine = new ArrayList<String>();
System.out.println("DATE:" + dateFormat.format(today.getTime()));
csvLine.add(dateFormat.format(today.getTime()));
System.out.println("Version:" + Version.buildNumber());
csvLine.add(String.valueOf(Version.buildNumber()));
int exitCode = 0;
Node node = null;
Node node2 = null;
FileInputStream fis = null;
File file = new File("many-single-blocks-test-" + uid + ".csv");
long t1, t2;
try {
// INSERT STUFF
final File dir = new File("longterm-mhk-test-" + uid);
FileUtil.removeAll(dir);
RandomSource random =
NodeStarter.globalTestInit(dir.getPath(), false, LogLevel.ERROR, "", false);
File seednodes = new File("seednodes.fref");
if (!seednodes.exists() || seednodes.length() == 0 || !seednodes.canRead()) {
System.err.println("Unable to read seednodes.fref, it doesn't exist, or is empty");
System.exit(EXIT_NO_SEEDNODES);
}
final File innerDir = new File(dir, Integer.toString(DARKNET_PORT1));
innerDir.mkdir();
fis = new FileInputStream(seednodes);
FileUtil.writeTo(fis, new File(innerDir, "seednodes.fref"));
fis.close();
// Create one node
node =
NodeStarter.createTestNode(
DARKNET_PORT1,
OPENNET_PORT1,
dir.getPath(),
false,
Node.DEFAULT_MAX_HTL,
0,
random,
new PooledExecutor(),
1000,
4 * 1024 * 1024,
true,
true,
true,
true,
true,
true,
true,
12 * 1024,
true,
true,
false,
false,
null);
Logger.getChain().setThreshold(LogLevel.ERROR);
// Start it
node.start(true);
t1 = System.currentTimeMillis();
if (!TestUtil.waitForNodes(node)) {
exitCode = EXIT_FAILED_TARGET;
return;
}
t2 = System.currentTimeMillis();
System.out.println("SEED-TIME:" + (t2 - t1));
csvLine.add(String.valueOf(t2 - t1));
HighLevelSimpleClient client = node.clientCore.makeClient((short) 0, false, false);
int successes = 0;
long startInsertsTime = System.currentTimeMillis();
InsertBatch batch = new InsertBatch(client);
// Inserts are sloooooow so do them in parallel.
for (int i = 0; i < INSERTED_BLOCKS; i++) {
System.err.println("Inserting block " + i);
RandomAccessBucket single = randomData(node);
InsertBlock block = new InsertBlock(single, new ClientMetadata(), FreenetURI.EMPTY_CHK_URI);
batch.startInsert(block);
}
batch.waitUntilFinished();
FreenetURI[] uris = batch.getURIs();
long[] times = batch.getTimes();
InsertException[] errors = batch.getErrors();
for (int i = 0; i < INSERTED_BLOCKS; i++) {
if (uris[i] != null) {
csvLine.add(String.valueOf(times[i]));
csvLine.add(uris[i].toASCIIString());
System.out.println("Pushed block " + i + " : " + uris[i] + " in " + times[i]);
successes++;
} else {
csvLine.add(InsertException.getShortMessage(errors[i].getMode()));
csvLine.add("N/A");
System.out.println("Failed to push block " + i + " : " + errors[i]);
}
}
long endInsertsTime = System.currentTimeMillis();
System.err.println(
"Succeeded inserts: "
+ successes
+ " of "
+ INSERTED_BLOCKS
+ " in "
+ (endInsertsTime - startInsertsTime)
+ "ms");
FetchContext fctx = client.getFetchContext();
fctx.maxNonSplitfileRetries = 0;
fctx.maxSplitfileBlockRetries = 0;
RequestClient requestContext = new RequestClientBuilder().build();
// PARSE FILE AND FETCH OLD STUFF IF APPROPRIATE
FreenetURI[] mhkURIs = new FreenetURI[3];
fis = new FileInputStream(file);
BufferedReader br = new BufferedReader(new InputStreamReader(fis, ENCODING));
String line = null;
GregorianCalendar target = (GregorianCalendar) today.clone();
target.set(Calendar.HOUR_OF_DAY, 0);
target.set(Calendar.MINUTE, 0);
target.set(Calendar.MILLISECOND, 0);
target.set(Calendar.SECOND, 0);
GregorianCalendar[] targets = new GregorianCalendar[MAX_N + 1];
for (int i = 0; i < targets.length; i++) {
targets[i] = ((GregorianCalendar) target.clone());
targets[i].add(Calendar.DAY_OF_MONTH, -((1 << i) - 1));
targets[i].getTime();
}
int[] totalFetchesByDelta = new int[MAX_N + 1];
int[] totalSuccessfulFetchesByDelta = new int[MAX_N + 1];
long[] totalFetchTimeByDelta = new long[MAX_N + 1];
loopOverLines:
while ((line = br.readLine()) != null) {
for (int i = 0; i < mhkURIs.length; i++) mhkURIs[i] = null;
// System.out.println("LINE: "+line);
String[] split = line.split("!");
Date date = dateFormat.parse(split[0]);
GregorianCalendar calendar = new GregorianCalendar(TimeZone.getTimeZone("GMT"));
calendar.setTime(date);
System.out.println("Date: " + dateFormat.format(calendar.getTime()));
calendar.set(Calendar.HOUR_OF_DAY, 0);
calendar.set(Calendar.MINUTE, 0);
calendar.set(Calendar.MILLISECOND, 0);
calendar.set(Calendar.SECOND, 0);
calendar.getTime();
FreenetURI[] insertedURIs = new FreenetURI[INSERTED_BLOCKS];
int[] insertTimes = new int[INSERTED_BLOCKS];
if (split.length < 3) continue;
int seedTime = Integer.parseInt(split[2]);
System.out.println("Seed time: " + seedTime);
if (split.length < 4) continue;
int token = 3;
if (split.length < token + INSERTED_BLOCKS * 2) continue;
for (int i = 0; i < INSERTED_BLOCKS; i++) {
try {
insertTimes[i] = Integer.parseInt(split[token]);
} catch (NumberFormatException e) {
insertTimes[i] = -1;
}
token++;
try {
insertedURIs[i] = new FreenetURI(split[token]);
} catch (MalformedURLException e) {
insertedURIs[i] = null;
}
token++;
System.out.println("Key insert " + i + " : " + insertedURIs[i] + " in " + insertTimes[i]);
}
for (int i = 0; i < targets.length; i++) {
if (Math.abs(targets[i].getTimeInMillis() - calendar.getTimeInMillis())
< HOURS.toMillis(12)) {
System.out.println("Found row for target date " + ((1 << i) - 1) + " days ago.");
System.out.println("Version: " + split[1]);
csvLine.add(Integer.toString(i));
int pulled = 0;
int inserted = 0;
for (int j = 0; j < INSERTED_BLOCKS; j++) {
if (insertedURIs[j] == null) {
csvLine.add("INSERT FAILED");
continue;
}
inserted++;
try {
t1 = System.currentTimeMillis();
FetchWaiter fw = new FetchWaiter(requestContext);
client.fetch(insertedURIs[j], 32768, fw, fctx);
fw.waitForCompletion();
t2 = System.currentTimeMillis();
System.out.println("PULL-TIME FOR BLOCK " + j + ": " + (t2 - t1));
csvLine.add(String.valueOf(t2 - t1));
pulled++;
} catch (FetchException e) {
if (e.getMode() != FetchExceptionMode.ALL_DATA_NOT_FOUND
&& e.getMode() != FetchExceptionMode.DATA_NOT_FOUND) e.printStackTrace();
csvLine.add(FetchException.getShortMessage(e.getMode()));
System.err.println("FAILED PULL FOR BLOCK " + j + ": " + e);
}
}
System.out.println(
"Pulled "
+ pulled
+ " blocks of "
+ inserted
+ " from "
+ ((1 << i) - 1)
+ " days ago.");
}
}
while (split.length > token + INSERTED_BLOCKS) {
int delta;
try {
delta = Integer.parseInt(split[token]);
} catch (NumberFormatException e) {
System.err.println("Unable to parse token " + token + " = \"" + token + "\"");
System.err.println("This is supposed to be a delta");
System.err.println(
"Skipping the rest of the line for date " + dateFormat.format(calendar.getTime()));
continue loopOverLines;
}
System.out.println("Delta: " + ((1 << delta) - 1) + " days");
token++;
int totalFetchTime = 0;
int totalSuccesses = 0;
int totalFetches = 0;
for (int i = 0; i < INSERTED_BLOCKS; i++) {
if (split[token].equals("")) continue;
int mhkFetchTime = -1;
totalFetches++;
try {
mhkFetchTime = Integer.parseInt(split[token]);
System.out.println(
"Fetched block #" + i + " on " + date + " in " + mhkFetchTime + "ms");
totalSuccesses++;
totalFetchTime += mhkFetchTime;
} catch (NumberFormatException e) {
System.out.println("Failed block #" + i + " on " + date + " : " + split[token]);
}
token++;
}
totalFetchesByDelta[delta] += totalFetches;
totalSuccessfulFetchesByDelta[delta] += totalSuccesses;
totalFetchTimeByDelta[delta] += totalFetchTime;
System.err.println(
"Succeeded: "
+ totalSuccesses
+ " of "
+ totalFetches
+ " average "
+ ((double) totalFetchTime) / ((double) totalSuccesses)
+ "ms for delta "
+ delta
+ " on "
+ dateFormat.format(date));
}
}
System.out.println();
System.out.println();
for (int i = 0; i < MAX_N + 1; i++) {
System.out.println(
"DELTA: "
+ i
+ " days: Total fetches: "
+ totalFetchesByDelta[i]
+ " total successes "
+ totalSuccessfulFetchesByDelta[i]
+ " = "
+ ((totalSuccessfulFetchesByDelta[i] * 100.0) / totalFetchesByDelta[i])
+ "% in "
+ (totalFetchTimeByDelta[i] * 1.0) / totalSuccessfulFetchesByDelta[i]
+ "ms");
}
fis.close();
fis = null;
} catch (Throwable t) {
t.printStackTrace();
exitCode = EXIT_THREW_SOMETHING;
} finally {
try {
if (node != null) node.park();
} catch (Throwable tt) {
}
try {
if (node2 != null) node2.park();
} catch (Throwable tt) {
}
Closer.close(fis);
writeToStatusLog(file, csvLine);
System.out.println("Exiting with status " + exitCode);
System.exit(exitCode);
}
}